diff --git a/requirements.txt b/requirements.txt index 22eec3f..e76f404 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,36 +1,57 @@ +annotated-types==0.7.0 anyio==4.6.0 +apsw==3.50.2.0 +apswutils==0.1.0 beautifulsoup4==4.12.3 certifi==2024.8.30 +charset-normalizer==3.4.2 click==8.1.7 -fastcore==1.7.8 -fastlite==0.0.11 +et-xmlfile==1.1.0 +fastcore==1.8.5 +fastlite==0.2.1 h11==0.14.0 httpcore==1.0.5 httptools==0.6.1 httpx==0.27.2 +httpx-sse==0.4.0 idna==3.10 iniconfig==2.0.0 itsdangerous==2.2.0 +markdown-it-py==3.0.0 +mcp==1.9.2 +mdurl==0.1.2 +numpy==2.1.1 oauthlib==3.2.2 +openpyxl==3.1.5 packaging==24.1 +pandas==2.2.3 pluggy==1.5.0 +pydantic==2.11.5 +pydantic-settings==2.9.1 +pydantic_core==2.33.2 +Pygments==2.19.1 pytest==8.3.3 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 -python-fasthtml==0.6.4 +python-fasthtml==0.12.21 python-multipart==0.0.10 +pytz==2024.2 PyYAML==6.0.2 +requests==2.32.3 +rich==14.0.0 +shellingham==1.5.4 six==1.16.0 sniffio==1.3.1 soupsieve==2.6 sqlite-minutils==3.37.0.post3 +sse-starlette==2.3.6 starlette==0.38.5 +typer==0.16.0 +typing-inspection==0.4.1 +typing_extensions==4.13.2 +tzdata==2024.1 +urllib3==2.4.0 uvicorn==0.30.6 uvloop==0.20.0 watchfiles==0.24.0 websockets==13.1 - -pandas~=2.2.3 -numpy~=2.1.1 -requests~=2.32.3 -mcp~=1.9.2 \ No newline at end of file diff --git a/src/components/workflows/components/WorkflowPlayer.py b/src/components/workflows/components/WorkflowPlayer.py index 0e9d5f1..2298c9a 100644 --- a/src/components/workflows/components/WorkflowPlayer.py +++ b/src/components/workflows/components/WorkflowPlayer.py @@ -7,8 +7,9 @@ from components.datagrid_new.settings import DataGridSettings from components.workflows.commands import WorkflowPlayerCommandManager from components.workflows.constants import WORKFLOW_PLAYER_INSTANCE_ID, ProcessorTypes from components.workflows.db_management import WorkflowsPlayerSettings +from core.instance_manager import InstanceManager from core.utils import get_unique_id, make_safe_id -from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter +from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter, DefaultDataFilter grid_settings = DataGridSettings( header_visible=True, @@ -32,17 +33,22 @@ class WorkflowPlayer(BaseComponent): self._player_settings = player_settings self._boundaries = boundaries self.commands = WorkflowPlayerCommandManager(self) - self._datagrid = DataGrid(self._session, - DataGrid.create_component_id(session), - self.key, - grid_settings=grid_settings, - boundaries=boundaries) + self._datagrid = InstanceManager.get(self._session, + DataGrid.create_component_id(session), + DataGrid, + key=self.key, + grid_settings=grid_settings, + boundaries=boundaries) def run(self): engine = WorkflowEngine() for component in self._player_settings.components: if component.type == ProcessorTypes.Producer and component.properties["processor_name"] == "Repository": - engine.add_processor(TableDataProducer(self._session, self._settings_manager, component.properties["repository"], component.properties["table"])) + engine.add_processor( + TableDataProducer(self._session, self._settings_manager, component.properties["repository"], + component.properties["table"])) + elif component.type == ProcessorTypes.Filter and component.properties["processor_name"] == "Default": + engine.add_processor(DefaultDataFilter(component.properties["filter"])) elif component.type == ProcessorTypes.Presenter and component.properties["processor_name"] == "Default": engine.add_processor(DefaultDataPresenter(component.properties["columns"])) diff --git a/src/core/Expando.py b/src/core/Expando.py index 641eceb..d8622ca 100644 --- a/src/core/Expando.py +++ b/src/core/Expando.py @@ -50,6 +50,9 @@ class Expando: def to_dict(self, mappings: dict) -> dict: return {prop_name: self.get(path) for path, prop_name in mappings.items() if prop_name is not None} + def __hasattr__(self, item): + return item in self._props + def __repr__(self): if "key" in self._props: return f"Expando(key={self._props["key"]})" diff --git a/src/core/utils.py b/src/core/utils.py index 73b300d..f466a5b 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,3 +1,4 @@ +import ast import base64 import hashlib import importlib @@ -417,3 +418,50 @@ def split_host_port(url): port = None return host, port + + +class UnreferencedNamesVisitor(ast.NodeVisitor): + """ + Try to find symbols that will be requested by the ast + It can be variable names, but also function names + """ + + def __init__(self): + self.names = set() + + def get_names(self, node): + self.visit(node) + return self.names + + def visit_Name(self, node): + self.names.add(node.id) + + def visit_For(self, node: ast.For): + self.visit_selected(node, ["body", "orelse"]) + + def visit_selected(self, node, to_visit): + """Called if no explicit visitor function exists for a node.""" + for field in to_visit: + value = getattr(node, field) + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + self.visit(item) + elif isinstance(value, ast.AST): + self.visit(value) + + def visit_Call(self, node: ast.Call): + self.visit_selected(node, ["args", "keywords"]) + + def visit_keyword(self, node: ast.keyword): + """ + Keywords are parameters that are defined with a double star (**) in function / method definition + ex: def fun(positional, *args, **keywords) + :param node: + :type node: + :return: + :rtype: + """ + self.names.add(node.arg) + self.visit_selected(node, ["value"]) + diff --git a/src/workflow/engine.py b/src/workflow/engine.py index c595e08..a1704df 100644 --- a/src/workflow/engine.py +++ b/src/workflow/engine.py @@ -1,7 +1,9 @@ +import ast from abc import ABC, abstractmethod from typing import Any, Generator from core.Expando import Expando +from core.utils import UnreferencedNamesVisitor from utils.Datahelper import DataHelper @@ -88,11 +90,21 @@ class DefaultDataPresenter(DataPresenter): return Expando(data.to_dict(self.mappings)) + class DefaultDataFilter(DataFilter): + def __init__(self, filter_expression: str): + super().__init__() + self.filter_expression = filter_expression + self._ast_tree = ast.parse(filter_expression, "", 'eval') + self._compiled = compile(self._ast_tree, "", "eval") + visitor = UnreferencedNamesVisitor() + self._unreferenced_names = visitor.get_names(self._ast_tree) + """Default data filter that returns True for all data items.""" def filter(self, data: Any) -> bool: - return True + my_locals = {name: data.get(name) for name in self._unreferenced_names if hasattr(data, name)} + return eval(self._compiled, globals(), my_locals) class WorkflowEngine: diff --git a/tests/test_utils.py b/tests/test_utils.py index 9b171ce..b89e574 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,10 +5,13 @@ # assert column_to_number("A") == 1 # assert column_to_number("AA") == 27 # assert column_to_number("ZZZ") == 475254 +import ast + import pytest from fasthtml.components import Div -from core.utils import make_html_id, update_elements, snake_case_to_capitalized_words, merge_classes +from core.utils import make_html_id, update_elements, snake_case_to_capitalized_words, merge_classes, \ + UnreferencedNamesVisitor @pytest.mark.parametrize("string, expected", [ @@ -110,7 +113,7 @@ def test_i_can_merge_cls(): kwargs = {} assert merge_classes("class1", kwargs) == "class1" assert kwargs == {} - + kwargs = {"foo": "bar"} assert merge_classes("class1", kwargs) == "class1" assert kwargs == {"foo": "bar"} @@ -127,4 +130,21 @@ def test_i_can_merge_cls(): assert merge_classes("class1", ("class2", "class3")) == "class1 class2 class3" # values are unique - assert merge_classes("class2", "class1", ("class1", ), {"cls": "class1"}) == "class2 class1" + assert merge_classes("class2", "class1", ("class1",), {"cls": "class1"}) == "class2 class1" + + +@pytest.mark.parametrize("source, expected", [ + ("a,b", {"a", "b"}), + ("isinstance(a, int)", {"a", "int"}), + ("date.today()", set()), + ("test()", set()), + ("sheerka.test()", set()), + ("for i in range(10): pass", set()), + ("func(x=a, y=b)", {"a", "b", "x", "y"}), +]) +def test_i_can_get_unreferenced_variables_from_simple_expressions(source, expected): + ast_ = ast.parse(source) + visitor = UnreferencedNamesVisitor() + visitor.visit(ast_) + + assert visitor.names == expected