First version of DefaultDataFilter

requirements.txt

@@ -1,36 +1,57 @@
+annotated-types==0.7.0
 anyio==4.6.0
+apsw==3.50.2.0
+apswutils==0.1.0
 beautifulsoup4==4.12.3
 certifi==2024.8.30
+charset-normalizer==3.4.2
 click==8.1.7
-fastcore==1.7.8
-fastlite==0.0.11
+et-xmlfile==1.1.0
+fastcore==1.8.5
+fastlite==0.2.1
 h11==0.14.0
 httpcore==1.0.5
 httptools==0.6.1
 httpx==0.27.2
+httpx-sse==0.4.0
 idna==3.10
 iniconfig==2.0.0
 itsdangerous==2.2.0
+markdown-it-py==3.0.0
+mcp==1.9.2
+mdurl==0.1.2
+numpy==2.1.1
 oauthlib==3.2.2
+openpyxl==3.1.5
 packaging==24.1
+pandas==2.2.3
 pluggy==1.5.0
+pydantic==2.11.5
+pydantic-settings==2.9.1
+pydantic_core==2.33.2
+Pygments==2.19.1
 pytest==8.3.3
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
-python-fasthtml==0.6.4
+python-fasthtml==0.12.21
 python-multipart==0.0.10
+pytz==2024.2
 PyYAML==6.0.2
+requests==2.32.3
+rich==14.0.0
+shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.1
 soupsieve==2.6
 sqlite-minutils==3.37.0.post3
+sse-starlette==2.3.6
 starlette==0.38.5
+typer==0.16.0
+typing-inspection==0.4.1
+typing_extensions==4.13.2
+tzdata==2024.1
+urllib3==2.4.0
 uvicorn==0.30.6
 uvloop==0.20.0
 watchfiles==0.24.0
 websockets==13.1
-
-pandas~=2.2.3
-numpy~=2.1.1
-requests~=2.32.3
-mcp~=1.9.2

src/components/workflows/components/WorkflowPlayer.py

@@ -7,8 +7,9 @@ from components.datagrid_new.settings import DataGridSettings
 from components.workflows.commands import WorkflowPlayerCommandManager
 from components.workflows.constants import WORKFLOW_PLAYER_INSTANCE_ID, ProcessorTypes
 from components.workflows.db_management import WorkflowsPlayerSettings
+from core.instance_manager import InstanceManager
 from core.utils import get_unique_id, make_safe_id
-from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter
+from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter, DefaultDataFilter
 
 grid_settings = DataGridSettings(
   header_visible=True,
@@ -32,17 +33,22 @@ class WorkflowPlayer(BaseComponent):
     self._player_settings = player_settings
     self._boundaries = boundaries
     self.commands = WorkflowPlayerCommandManager(self)
-    self._datagrid = DataGrid(self._session,
-                              DataGrid.create_component_id(session),
-                              self.key,
-                              grid_settings=grid_settings,
-                              boundaries=boundaries)
+    self._datagrid = InstanceManager.get(self._session,
+                                         DataGrid.create_component_id(session),
+                                         DataGrid,
+                                         key=self.key,
+                                         grid_settings=grid_settings,
+                                         boundaries=boundaries)
   
   def run(self):
     engine = WorkflowEngine()
     for component in self._player_settings.components:
       if component.type == ProcessorTypes.Producer and component.properties["processor_name"] == "Repository":
-        engine.add_processor(TableDataProducer(self._session, self._settings_manager, component.properties["repository"], component.properties["table"]))
+        engine.add_processor(
+          TableDataProducer(self._session, self._settings_manager, component.properties["repository"],
+                            component.properties["table"]))
+      elif component.type == ProcessorTypes.Filter and component.properties["processor_name"] == "Default":
+        engine.add_processor(DefaultDataFilter(component.properties["filter"]))
       elif component.type == ProcessorTypes.Presenter and component.properties["processor_name"] == "Default":
         engine.add_processor(DefaultDataPresenter(component.properties["columns"]))
     

src/core/Expando.py

@@ -50,6 +50,9 @@ class Expando:
   def to_dict(self, mappings: dict) -> dict:
     return {prop_name: self.get(path) for path, prop_name in mappings.items() if prop_name is not None}
   
+  def __hasattr__(self, item):
+    return item in self._props
+  
   def __repr__(self):
     if "key" in self._props:
       return f"Expando(key={self._props["key"]})"

src/core/utils.py

@@ -1,3 +1,4 @@
+import ast
 import base64
 import hashlib
 import importlib
@@ -417,3 +418,50 @@ def split_host_port(url):
       port = None
   
   return host, port
+
+
+class UnreferencedNamesVisitor(ast.NodeVisitor):
+  """
+  Try to find symbols that will be requested by the ast
+  It can be variable names, but also function names
+  """
+  
+  def __init__(self):
+    self.names = set()
+  
+  def get_names(self, node):
+    self.visit(node)
+    return self.names
+  
+  def visit_Name(self, node):
+    self.names.add(node.id)
+  
+  def visit_For(self, node: ast.For):
+    self.visit_selected(node, ["body", "orelse"])
+  
+  def visit_selected(self, node, to_visit):
+    """Called if no explicit visitor function exists for a node."""
+    for field in to_visit:
+      value = getattr(node, field)
+      if isinstance(value, list):
+        for item in value:
+          if isinstance(item, ast.AST):
+            self.visit(item)
+      elif isinstance(value, ast.AST):
+        self.visit(value)
+  
+  def visit_Call(self, node: ast.Call):
+    self.visit_selected(node, ["args", "keywords"])
+  
+  def visit_keyword(self, node: ast.keyword):
+    """
+    Keywords are parameters that are defined with a double star (**) in function / method definition
+    ex: def fun(positional, *args, **keywords)
+    :param node:
+    :type node:
+    :return:
+    :rtype:
+    """
+    self.names.add(node.arg)
+    self.visit_selected(node, ["value"])
+

src/workflow/engine.py

@@ -1,7 +1,9 @@
+import ast
 from abc import ABC, abstractmethod
 from typing import Any, Generator
 
 from core.Expando import Expando
+from core.utils import UnreferencedNamesVisitor
 from utils.Datahelper import DataHelper
 
 
@@ -88,11 +90,21 @@ class DefaultDataPresenter(DataPresenter):
     
     return Expando(data.to_dict(self.mappings))
 
+
 class DefaultDataFilter(DataFilter):
+  def __init__(self, filter_expression: str):
+    super().__init__()
+    self.filter_expression = filter_expression
+    self._ast_tree = ast.parse(filter_expression, "<user input>", 'eval')
+    self._compiled = compile(self._ast_tree, "<string>", "eval")
+    visitor = UnreferencedNamesVisitor()
+    self._unreferenced_names = visitor.get_names(self._ast_tree)
+  
   """Default data filter that returns True for all data items."""
   
   def filter(self, data: Any) -> bool:
-    return True
+    my_locals = {name: data.get(name) for name in self._unreferenced_names if hasattr(data, name)}
+    return eval(self._compiled, globals(), my_locals)
 
 
 class WorkflowEngine:

tests/test_utils.py

@@ -5,10 +5,13 @@
 #     assert column_to_number("A") == 1
 #     assert column_to_number("AA") == 27
 #     assert column_to_number("ZZZ") == 475254
+import ast
+
 import pytest
 from fasthtml.components import Div
 
-from core.utils import make_html_id, update_elements, snake_case_to_capitalized_words, merge_classes
+from core.utils import make_html_id, update_elements, snake_case_to_capitalized_words, merge_classes, \
+  UnreferencedNamesVisitor
 
 
 @pytest.mark.parametrize("string, expected", [
@@ -127,4 +130,21 @@ def test_i_can_merge_cls():
   assert merge_classes("class1", ("class2", "class3")) == "class1 class2 class3"
   
   # values are unique
-  assert merge_classes("class2", "class1", ("class1", ), {"cls": "class1"}) == "class2 class1"
+  assert merge_classes("class2", "class1", ("class1",), {"cls": "class1"}) == "class2 class1"
+
+
+@pytest.mark.parametrize("source, expected", [
+    ("a,b", {"a", "b"}),
+    ("isinstance(a, int)", {"a", "int"}),
+    ("date.today()", set()),
+    ("test()", set()),
+    ("sheerka.test()", set()),
+    ("for i in range(10): pass", set()),
+    ("func(x=a, y=b)", {"a", "b", "x", "y"}),
+])
+def test_i_can_get_unreferenced_variables_from_simple_expressions(source, expected):
+  ast_ = ast.parse(source)
+  visitor = UnreferencedNamesVisitor()
+  visitor.visit(ast_)
+  
+  assert visitor.names == expected