From 14be07720fc243aa1332d7268d82a5af5e499b3b Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Sun, 6 Jul 2025 22:53:18 +0200 Subject: [PATCH] I can run simple workflow --- .../workflows/components/WorkflowDesigner.py | 12 +-- .../workflows/components/WorkflowPlayer.py | 18 ++++- src/components/workflows/db_management.py | 1 + src/core/Expando.py | 70 +++++++++++++++++ src/core/settings_management.py | 2 +- src/utils/Datahelper.py | 3 + src/workflow/engine.py | 43 ++++++++++- tests/test_data_helper.py | 66 ++++++++++++++++ tests/test_expando.py | 75 +++++++++++++++++++ 9 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 src/core/Expando.py create mode 100644 tests/test_data_helper.py create mode 100644 tests/test_expando.py diff --git a/src/components/workflows/components/WorkflowDesigner.py b/src/components/workflows/components/WorkflowDesigner.py index a4dc46a..9fb8bf7 100644 --- a/src/components/workflows/components/WorkflowDesigner.py +++ b/src/components/workflows/components/WorkflowDesigner.py @@ -184,11 +184,13 @@ class WorkflowDesigner(BaseComponent): WorkflowPlayer, settings_manager=self._settings_manager, tabs_manager=self.tabs_manager, - player_settings=WorkflowsPlayerSettings(workflow_name), + player_settings=WorkflowsPlayerSettings(workflow_name, + list(self._state.components.values())), boundaries=boundaries) + player.run() + self.tabs_manager.add_tab(f"Workflow {workflow_name}", player, player.key) - # player.start(self._state.selected_component_id) return self.tabs_manager.refresh() def on_processor_details_event(self, component_id: str, event_name: str, details: dict): @@ -436,9 +438,9 @@ class WorkflowDesigner(BaseComponent): Fieldset( Legend("Presenter", cls="fieldset-legend"), Input(type="text", - name="presenter", - value=component.properties.get("filter", ""), - placeholder="Enter filter expression", + name="columns", + value=component.properties.get("columns", ""), + placeholder="Columns to display, separated by comma", cls="input w-full"), P("Comma separated list of columns to display. Use * to display all columns, source=dest to rename columns."), cls="fieldset bg-base-200 border-base-300 rounded-box border p-4" diff --git a/src/components/workflows/components/WorkflowPlayer.py b/src/components/workflows/components/WorkflowPlayer.py index 0694575..0e9d5f1 100644 --- a/src/components/workflows/components/WorkflowPlayer.py +++ b/src/components/workflows/components/WorkflowPlayer.py @@ -1,12 +1,14 @@ +import pandas as pd from fasthtml.components import * from components.BaseComponent import BaseComponent from components.datagrid_new.components.DataGrid import DataGrid from components.datagrid_new.settings import DataGridSettings from components.workflows.commands import WorkflowPlayerCommandManager -from components.workflows.constants import WORKFLOW_PLAYER_INSTANCE_ID +from components.workflows.constants import WORKFLOW_PLAYER_INSTANCE_ID, ProcessorTypes from components.workflows.db_management import WorkflowsPlayerSettings from core.utils import get_unique_id, make_safe_id +from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter grid_settings = DataGridSettings( header_visible=True, @@ -36,6 +38,20 @@ class WorkflowPlayer(BaseComponent): grid_settings=grid_settings, boundaries=boundaries) + def run(self): + engine = WorkflowEngine() + for component in self._player_settings.components: + if component.type == ProcessorTypes.Producer and component.properties["processor_name"] == "Repository": + engine.add_processor(TableDataProducer(self._session, self._settings_manager, component.properties["repository"], component.properties["table"])) + elif component.type == ProcessorTypes.Presenter and component.properties["processor_name"] == "Default": + engine.add_processor(DefaultDataPresenter(component.properties["columns"])) + + res = engine.run_to_list() + + data = [row.as_dict() for row in res] + df = pd.DataFrame(data) + self._datagrid.init_from_dataframe(df) + def __ft__(self): return Div( self._datagrid, diff --git a/src/components/workflows/db_management.py b/src/components/workflows/db_management.py index a77c2cc..c4b9ca5 100644 --- a/src/components/workflows/db_management.py +++ b/src/components/workflows/db_management.py @@ -44,6 +44,7 @@ class WorkflowsDesignerState: @dataclass class WorkflowsPlayerSettings: workflow_name: str = "No Name" + components: list[WorkflowComponent] = None @dataclass diff --git a/src/core/Expando.py b/src/core/Expando.py new file mode 100644 index 0000000..641eceb --- /dev/null +++ b/src/core/Expando.py @@ -0,0 +1,70 @@ +class Expando: + """ + Readonly dynamic class that eases the access to attributes and sub attributes + It is initialized with a dict + You can then access the property using dot '.' (ex. obj.prop1.prop2) + """ + + def __init__(self, props): + self._props = props + + def __getattr__(self, item): + if item not in self._props: + raise AttributeError(item) + + current = self._props[item] + return Expando(current) if isinstance(current, dict) else current + + def __setitem__(self, key, value): + self._props[key] = value + + def get(self, path): + """ + returns the value, from a string with represents the path + :param path: + :return: + """ + current = self._props + for attr in path.split("."): + if isinstance(current, list): + temp = [] + for value in current: + if value and attr in value: + temp.append(value[attr]) + current = temp + + else: + if current is None or attr not in current: + return None + current = current[attr] + + return current + + def as_dict(self): + """ + Return the information as a dictionary + :return: + """ + return self._props.copy() + + def to_dict(self, mappings: dict) -> dict: + return {prop_name: self.get(path) for path, prop_name in mappings.items() if prop_name is not None} + + def __repr__(self): + if "key" in self._props: + return f"Expando(key={self._props["key"]})" + + props_as_str = str(self._props) + if len(props_as_str) > 50: + props_as_str = props_as_str[:50] + "..." + + return f"Expando({props_as_str})" + + def __eq__(self, other): + if not isinstance(other, Expando): + return False + + return self._props == other._props + + def __hash__(self): + return hash(tuple(sorted(self._props.items()))) diff --git a/src/core/settings_management.py b/src/core/settings_management.py index 9726c5d..58dd7e3 100644 --- a/src/core/settings_management.py +++ b/src/core/settings_management.py @@ -87,7 +87,7 @@ class MemoryDbEngine: obj.update(items) def exists(self, user_id: str, entry: str): - return user_id in entry and entry in self.db[user_id] + return user_id in self.db and entry in self.db[user_id] class SettingsManager: diff --git a/src/utils/Datahelper.py b/src/utils/Datahelper.py index 0e91f1d..0fedc62 100644 --- a/src/utils/Datahelper.py +++ b/src/utils/Datahelper.py @@ -1,6 +1,7 @@ from dataclasses import is_dataclass from components.datagrid_new.db_management import DataGridDbManager +from core.Expando import Expando class DataHelper: @@ -16,6 +17,8 @@ class DataHelper: if object_type: if is_dataclass(object_type): return [object_type(**row) for row in dataframe.to_dict(orient="records")] + elif object_type is Expando: + return [Expando(row) for row in dataframe.to_dict(orient="records")] else: raise ValueError("object_type must be a dataclass type") diff --git a/src/workflow/engine.py b/src/workflow/engine.py index 19317fa..5650987 100644 --- a/src/workflow/engine.py +++ b/src/workflow/engine.py @@ -1,6 +1,9 @@ from abc import ABC, abstractmethod from typing import Any, Generator +from core.Expando import Expando +from utils.Datahelper import DataHelper + class DataProcessor(ABC): """Base class for all data processing components.""" @@ -47,6 +50,45 @@ class DataPresenter(DataProcessor): yield self.present(data) +class TableDataProducer(DataProducer): + """Base class for data producers that emit data from a repository.""" + + def __init__(self, session, settings_manager, repository_name, table_name): + self._session = session + self.settings_manager = settings_manager + self.repository_name = repository_name + self.table_name = table_name + + def emit(self, data: Any = None) -> Generator[Any, None, None]: + yield from DataHelper.get(self._session, self.settings_manager, self.repository_name, self.table_name, Expando) + + +class DefaultDataPresenter(DataPresenter): + """Default data presenter that returns the input data unchanged.""" + + def __init__(self, columns_as_str: str): + super().__init__() + if not columns_as_str or columns_as_str == "*": + self.mappings = None + + else: + + self.mappings = {} + temp_mappings = [col.strip() for col in columns_as_str.split(",")] + for mapping in temp_mappings: + if "=" in mapping: + key, value = mapping.split("=") + self.mappings[key] = value + else: + self.mappings[mapping] = mapping + + def present(self, data: Any) -> Any: + if self.mappings is None: + return data + + return Expando(data.to_dict(self.mappings)) + + class WorkflowEngine: """Orchestrates the data processing pipeline using generators.""" @@ -93,4 +135,3 @@ class WorkflowEngine: Use this method when you need all results at once. """ return list(self.run()) - diff --git a/tests/test_data_helper.py b/tests/test_data_helper.py new file mode 100644 index 0000000..9a6fdd1 --- /dev/null +++ b/tests/test_data_helper.py @@ -0,0 +1,66 @@ +from dataclasses import dataclass + +import pandas as pd +import pytest + +from components.datagrid_new.components.DataGrid import DataGrid +from core.Expando import Expando +from core.settings_management import SettingsManager, MemoryDbEngine +from utils.Datahelper import DataHelper + +TEST_GRID_ID = "testing_grid_id" +TEST_GRID_KEY = ("RepoName", "TableName") + + +@pytest.fixture() +def settings_manager(): + return SettingsManager(MemoryDbEngine()) + + +@pytest.fixture() +def datagrid(session, settings_manager): + dg = DataGrid(session, + _id=TEST_GRID_ID, + settings_manager=settings_manager, + key=TEST_GRID_KEY, + boundaries={"height": 500, "width": 800}) + + df = pd.DataFrame({ + 'Name': ['Alice', 'Bob'], + 'Age': [20, 25], + 'Is Student': [True, False], + }) + + dg.init_from_dataframe(df, save_state=True) + return dg + + +def test_i_can_get_data_as_dataframe(session, settings_manager, datagrid): + res = DataHelper.get(session, settings_manager, "RepoName", "TableName") + assert isinstance(res, pd.DataFrame) + assert res.equals(datagrid.get_dataframe()) + + +def test_i_can_get_data_as_dataclass(session, settings_manager, datagrid): + @dataclass + class DataclassTestClass: + name: str + age: int + is_student: bool + + res = DataHelper.get(session, settings_manager, "RepoName", "TableName", DataclassTestClass) + assert isinstance(res, list) + assert res == [ + DataclassTestClass("Alice", 20, True), + DataclassTestClass("Bob", 25, False), + ] + + +def test_i_can_get_data_as_expando(session, settings_manager, datagrid): + res = DataHelper.get(session, settings_manager, "RepoName", "TableName", Expando) + assert isinstance(res, list) + assert res == [ + Expando({"name": "Alice", "age": 20, "is_student": True}), + Expando({"name": "Bob", "age": 25, "is_student": False}) + ] + \ No newline at end of file diff --git a/tests/test_expando.py b/tests/test_expando.py new file mode 100644 index 0000000..8d35b66 --- /dev/null +++ b/tests/test_expando.py @@ -0,0 +1,75 @@ +import pytest + +from core.Expando import Expando + + +def test_i_can_get_properties(): + props = {"a": 10, + "b": { + "c": "value", + "d": 20 + }} + dynamic = Expando(props) + + assert dynamic.a == 10 + assert dynamic.b.c == "value" + + with pytest.raises(AttributeError): + assert dynamic.unknown == "some_value" + + +def test_i_can_get(): + props = {"a": 10, + "b": { + "c": "value", + "d": 20 + }} + dynamic = Expando(props) + + assert dynamic.get("a") == 10 + assert dynamic.get("b.c") == "value" + assert dynamic.get("unknown") is None + + +def test_i_can_get_from_list(): + props = {"a": [{"c": "value1", "d": 1}, {"c": "value2", "d": 2}]} + dynamic = Expando(props) + + assert dynamic.get("a.c") == ["value1", "value2"] + + +def test_none_is_returned_when_get_from_list_and_property_does_not_exist(): + props = {"a": [{"c": "value1", "d": 1}, + {"a": "value2", "d": 2} # 'c' does not exist in the second row + ]} + dynamic = Expando(props) + + assert dynamic.get("a.c") == ["value1"] + + +def test_i_can_manage_none_values(): + props = {"a": 10, + "b": None} + dynamic = Expando(props) + + assert dynamic.get("b.c") is None + + +def test_i_can_manage_none_values_in_list(): + props = {"a": [{"b": {"c": "value"}}, + {"b": None} + ]} + dynamic = Expando(props) + + assert dynamic.get("a.b.c") == ["value"] + + +def test_i_can_add_new_properties(): + props = {"a": 10, + "b": 20} + dynamic = Expando(props) + dynamic["c"] = 30 + + assert dynamic.a == 10 + assert dynamic.b == 20 + assert dynamic.c == 30