Added DataServicesManager and DataService

tests/core/data/conftest.py

@@ -0,0 +1,44 @@
+import shutil
+
+import pytest
+from dbengine.handlers import handlers
+
+from myfasthtml.core.data.DataServicesManager import DataServicesManager
+from myfasthtml.core.dbengine_utils import DataFrameHandler
+from myfasthtml.core.dbmanager import DbManager
+from myfasthtml.core.instances import SingleInstance, InstancesManager
+
+
+@pytest.fixture(scope="session")
+def session():
+  handlers.register_handler(DataFrameHandler())
+  return {
+      "user_info": {
+          "id": "test_tenant_id",
+          "email": "test@email.com",
+          "username": "test user",
+          "role": [],
+      }
+  }
+
+
+@pytest.fixture
+def parent(session):
+  instance = SingleInstance(session=session, _id="test_parent_id")
+  return instance
+
+
+@pytest.fixture
+def db_manager(parent):
+  shutil.rmtree("TestDb", ignore_errors=True)
+  db_manager_instance = DbManager(parent, root="TestDb", auto_register=True)
+
+  yield db_manager_instance
+
+  shutil.rmtree("TestDb", ignore_errors=True)
+  InstancesManager.reset()
+
+
+@pytest.fixture
+def dsm(parent, db_manager):
+  return DataServicesManager(parent, parent._session)

tests/core/data/test_dataservice.py

@@ -0,0 +1,213 @@
+"""Unit tests for DataService."""
+import pandas as pd
+import pytest
+
+from myfasthtml.core.constants import ColumnType
+from myfasthtml.core.data.ColumnDefinition import ColumnDefinition
+
+
+class TestDataInitialisation:
+  """Tests for the Data initialisation section of DataService."""
+  
+  @pytest.fixture
+  def service(self, dsm):
+    return dsm.create_service("ns.tbl", save_state=False)
+  
+  def test_i_can_load_a_dataframe(self, service):
+    """load_dataframe() populates the store and column definitions."""
+    df = pd.DataFrame({"name": ["Alice", "Bob"], "age": [30, 25]})
+    
+    service.load_dataframe(df)
+    
+    assert service.get_store().ne_df is not None
+    assert service.get_store().ns_total_rows == 2
+    assert len(service.columns) == 2
+  
+  def test_i_can_load_an_empty_dataframe(self, service):
+    """load_dataframe() with empty DataFrame sets total_rows to 0."""
+    service.load_dataframe(pd.DataFrame())
+    
+    assert service.get_store().ns_total_rows == 0
+    assert service.columns == []
+  
+  def test_i_can_load_dataframe_without_reinitializing_columns(self, service):
+    """load_dataframe(init_columns=False) preserves existing column definitions."""
+    df = pd.DataFrame({"a": [1]})
+    service.load_dataframe(df)
+    original_columns = list(service.columns)
+    
+    df2 = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    service.load_dataframe(df2, init_columns=False)
+    
+    assert service.columns == original_columns
+  
+  def test_i_can_load_none_dataframe_without_error(self, service):
+    """load_dataframe(None) is a no-op and does not raise.
+
+    Why this matters:
+    - Early return on None protects against uninitialized callers.
+    - ne_df must remain None (no side effects on the store).
+    """
+    service.load_dataframe(None)
+    
+    assert service.get_store().ne_df is None
+  
+  def test_i_can_load_dataframe_with_column_name_normalization(self, service):
+    """load_dataframe() normalizes column names to safe IDs via make_safe_id.
+
+    Why this matters:
+    - Columns with spaces or special characters must be accessible as safe IDs.
+    - make_safe_id lowercases and replaces non-safe characters with underscores.
+    """
+    df = pd.DataFrame({"First Name": ["Alice"], "Last Name": ["Smith"]})
+    
+    service.load_dataframe(df)
+    
+    col_ids = [c.col_id for c in service.columns]
+    assert col_ids == ["first_name", "last_name"]
+
+
+class TestMutations:
+  """Tests for the Mutations section of DataService."""
+  
+  @pytest.fixture
+  def service(self, dsm):
+    svc = dsm.create_service("ns.mutations", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"value": [1, 2, 3]}))
+    return svc
+  
+  def test_i_can_add_a_row(self, service):
+    """add_row() appends a row with default values and updates the caches."""
+    service.add_row()
+    
+    assert service.get_store().ns_total_rows == 4
+    assert len(service.get_store().ne_df) == 4
+  
+  def test_i_can_add_a_row_with_custom_data(self, service):
+    """add_row() with explicit data stores the provided values."""
+    service.add_row(row_data={"value": 99})
+    
+    assert service.get_store().ne_df.iloc[-1]["value"] == 99
+  
+  def test_i_can_set_data(self, service):
+    """set_data() updates the cell in the DataFrame, fast-access cache, and row data."""
+    service.set_data("value", 1, 99)
+    
+    assert service.get_store().ne_df.at[1, "value"] == 99
+    assert service.get_store().ns_fast_access["value"][1] == 99
+    assert service.get_store().ns_row_data[1]["value"] == 99
+  
+  @pytest.mark.parametrize("col_type, expected_default", [
+      (ColumnType.Text, ""),
+      (ColumnType.Number, 0),
+      (ColumnType.Bool, False),
+      (ColumnType.Datetime, pd.NaT),
+      (ColumnType.Choice, ""),
+      (ColumnType.Enum, ""),
+      (ColumnType.RowSelection_, ""),
+  ])
+  def test_i_can_add_column_with_correct_default_value(self, service, col_type, expected_default):
+    """add_column() creates a DataFrame column with the type-appropriate default value.
+
+    Why these assertions matter:
+    - col_id in ne_df.columns: Confirms the column is materialized in the DataFrame.
+    - len(columns) == 2: Confirms the column is registered in the metadata.
+    - default value: Each type has a specific sentinel value; wrong defaults corrupt data.
+    - pd.isna() for Datetime: pd.NaT does not support equality comparison.
+    """
+    col_def = ColumnDefinition(col_id="__new__", col_index=-1, title="New Col", type=col_type)
+    service.add_column(col_def)
+    
+    assert col_def.col_id in service.get_store().ne_df.columns
+    assert len(service.columns) == 2
+    actual = service.get_store().ne_df[col_def.col_id].iloc[0]
+    if pd.isna(expected_default):
+      assert pd.isna(actual)
+    else:
+      assert actual == expected_default
+  
+  @pytest.mark.parametrize("col_type", [ColumnType.Formula, ColumnType.RowIndex])
+  def test_i_can_add_virtual_column_without_dataframe_column(self, service, col_type):
+    """add_column() with virtual types does not create a DataFrame column.
+
+    Why these assertions matter:
+    - col_id not in ne_df.columns: Virtual columns are computed, not stored in the DataFrame.
+    - col_index == -1: Sentinel value marking virtual columns.
+    - len(columns) == 2: Column is registered in the state metadata despite being virtual.
+    """
+    col_def = ColumnDefinition(col_id="__new__", col_index=-1, title="Virtual", type=col_type)
+    service.add_column(col_def)
+    
+    assert col_def.col_id not in service.get_store().ne_df.columns
+    assert col_def.col_index == -1
+    assert len(service.columns) == 2
+  
+  def test_i_can_add_row_without_loaded_dataframe_without_error(self, dsm):
+    """add_row() is a no-op and does not raise when no DataFrame is loaded."""
+    service = dsm.create_service("ns.nodf_row", save_state=False)
+    
+    service.add_row()
+    
+    assert service.get_store().ne_df is None
+  
+  def test_i_can_set_data_without_loaded_dataframe_without_error(self, dsm):
+    """set_data() is a no-op and does not raise when no DataFrame is loaded."""
+    service = dsm.create_service("ns.nodf_set", save_state=False)
+    
+    service.set_data("x", 0, 42)
+    
+    assert service.get_store().ne_df is None
+
+
+class TestFormulaManagement:
+  """Tests for the Formula management section of DataService."""
+  
+  @pytest.fixture
+  def service(self, dsm):
+    svc = dsm.create_service("ns.formula", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"a": [1, 2, 3]}))
+    return svc
+  
+  def test_i_can_get_table_name(self, service):
+    """table_name property returns the value set at creation."""
+    assert service.table_name == "ns.formula"
+  
+  def test_i_can_update_table_name(self, service):
+    """set_table_name() updates the table name."""
+    service.set_table_name("ns.new_name")
+    
+    assert service.table_name == "ns.new_name"
+  
+  def test_i_can_register_formula(self, service):
+    """register_formula() registers a formula in the shared FormulaEngine.
+
+    Why these assertions matter:
+    - has_formula: Confirms the formula was registered in the engine's DAG.
+    - get_formula_text: Confirms the source expression is stored as-is.
+    """
+    service.register_formula("computed", "{a} + 1")
+    
+    engine = service.get_formula_engine()
+    assert engine.has_formula("ns.formula", "computed")
+    assert engine.get_formula_text("ns.formula", "computed") == "{a} + 1"
+  
+  def test_i_can_remove_formula(self, service):
+    """remove_formula() unregisters a formula from the FormulaEngine."""
+    service.register_formula("computed", "{a} + 1")
+    service.remove_formula("computed")
+
+    engine = service.get_formula_engine()
+    assert not engine.has_formula("ns.formula", "computed")
+
+  def test_i_cannot_register_invalid_formula(self, service):
+    """register_formula() with invalid DSL syntax does not register the formula.
+
+    Why this matters:
+    - parse_formula() raises DSLSyntaxError when it cannot parse the expression.
+    - register_formula() catches the exception to protect the caller, but the
+      formula must remain absent from the engine — not silently removed.
+    """
+    service.register_formula("computed", "invalid syntax without braces")
+
+    engine = service.get_formula_engine()
+    assert not engine.has_formula("ns.formula", "computed")

tests/core/data/test_dataservice_formula_integration.py

@@ -0,0 +1,210 @@
+"""Integration tests for DataService formula evaluation through DataServicesManager.
+
+These tests exercise the full stack: DataServicesManager owns the FormulaEngine
+and provides the registry_resolver that enables cross-table formula resolution.
+Each test uses real DataService instances and real DataStore objects — no fakes.
+"""
+import pytest
+import pandas as pd
+
+from myfasthtml.core.constants import ColumnType
+from myfasthtml.core.data.ColumnDefinition import ColumnDefinition
+
+
+class TestIntraTableFormula:
+  """Single-table formula evaluation through the DSM/DataService stack."""
+
+  @pytest.fixture
+  def service(self, dsm):
+    svc = dsm.create_service("ns.sales", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"price": [10, 20, 30], "qty": [2, 3, 4]}))
+    return svc
+
+  def test_i_can_evaluate_formula_on_single_table(self, service):
+    """register_formula() + ensure_ready() computes the column for all rows.
+
+    Why these assertions matter:
+    - ns_fast_access["total"]: ensure_ready() writes results to the cache used by rendering.
+    - All three rows: verifies the formula is applied to every row, not just the first.
+    """
+    service.register_formula("total", "{price} * {qty}")
+    service.ensure_ready()
+
+    result = service.get_store().ns_fast_access["total"]
+    assert result[0] == 20
+    assert result[1] == 60
+    assert result[2] == 120
+
+  def test_i_can_reevaluate_formula_after_data_change(self, service):
+    """set_data() marks dependent formula columns dirty; ensure_ready() recomputes them.
+
+    Why these assertions matter:
+    - result[0] updated: confirms the dirty flag propagated and row 0 was recomputed.
+    - result[1] unchanged: confirms only affected rows are recomputed (no unnecessary work).
+    """
+    service.register_formula("total", "{price} * {qty}")
+    service.ensure_ready()
+
+    service.set_data("price", 0, 100)
+    service.ensure_ready()
+
+    result = service.get_store().ns_fast_access["total"]
+    assert result[0] == 200
+    assert result[1] == 60
+
+
+class TestCrossTableFormula:
+  """Cross-table formula resolution via DataServicesManager.registry_resolver.
+
+  Table names use namespace notation (e.g. "ns.products"). The DSL grammar
+  now supports multiple dots in TABLE_COL_REF; the transformer splits on the
+  last dot to separate the table name from the column name.
+  """
+
+  @pytest.fixture
+  def orders_service(self, dsm):
+    svc = dsm.create_service("ns.orders", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"qty": [2, 3]}))
+    return svc
+
+  @pytest.fixture
+  def products_service(self, dsm):
+    svc = dsm.create_service("ns.products", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"price": [10, 20]}))
+    return svc
+
+  def test_i_can_evaluate_cross_table_formula(self, orders_service, products_service):
+    """A formula in one service can reference a namespaced table from another service.
+
+    Why these assertions matter:
+    - result[0] and result[1]: confirms the registry_resolver resolved "ns.products"
+      correctly and combined its data with the orders data row by row.
+    """
+    orders_service.register_formula("total", "{ns.products.price} * {qty}")
+    orders_service.ensure_ready()
+
+    result = orders_service.get_store().ns_fast_access["total"]
+    assert result[0] == 20
+    assert result[1] == 60
+
+  def test_i_cannot_resolve_cross_table_formula_for_unknown_table(self, orders_service):
+    """A formula referencing an unregistered table resolves to None without raising.
+
+    Why this matters:
+    - result is None: confirms the engine degrades gracefully when the resolver
+      returns None, instead of raising or producing corrupt values.
+    """
+    orders_service.register_formula("total", "{ns.unknown_table.price} * {qty}")
+    orders_service.ensure_ready()
+
+    result = orders_service.get_store().ns_fast_access["total"]
+    assert result[0] is None
+    assert result[1] is None
+
+
+class TestCrossTableFormulaWhere:
+  """Cross-table formula resolution using an explicit WHERE clause.
+
+  The WHERE clause scans the remote table for a row where remote_column == local_value,
+  enabling correct lookups regardless of row ordering between tables.
+  """
+
+  @pytest.fixture
+  def orders_service(self, dsm):
+    svc = dsm.create_service("ns.orders", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"product_id": [2, 1], "qty": [3, 5]}))
+    return svc
+
+  @pytest.fixture
+  def products_service(self, dsm):
+    svc = dsm.create_service("ns.products", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"product_id": [1, 2], "price": [10, 20]}))
+    return svc
+
+  def test_i_can_lookup_value_with_where_clause_non_sequential(self, orders_service, products_service):
+    """WHERE resolves the correct remote row even when tables are not aligned by position.
+
+    Why these assertions matter:
+    - result[0] == 60: order row 0 has product_id=2, products row 1 has price=20 → 20*3=60.
+    - result[1] == 50: order row 1 has product_id=1, products row 0 has price=10 → 10*5=50.
+    Row-index fallback would return 10*3=30 and 20*5=100 — both wrong.
+    """
+    orders_service.register_formula(
+      "total",
+      "{ns.products.price where ns.products.product_id = product_id} * {qty}"
+    )
+    orders_service.ensure_ready()
+
+    result = orders_service.get_store().ns_fast_access["total"]
+    assert result[0] == 60
+    assert result[1] == 50
+    
+
+  def test_i_can_lookup_returns_none_when_no_match(self, orders_service, products_service):
+    """WHERE returns None when the local value has no matching row in the remote table.
+
+    Why this matters:
+    - result[0] is None: product_id=2 exists in products, but product_id=99 does not.
+    - No exception is raised: the engine must degrade gracefully on missing lookups.
+    """
+    orders_service_no_match = orders_service
+    orders_service_no_match.get_store().ns_fast_access["product_id"][0] = 99
+
+    orders_service_no_match.register_formula(
+      "total",
+      "{ns.products.price where ns.products.product_id = product_id} * {qty}"
+    )
+    orders_service_no_match.ensure_ready()
+
+    result = orders_service_no_match.get_store().ns_fast_access["total"]
+    assert result[0] is None
+
+
+class TestFormulaLifecycle:
+  """End-to-end formula lifecycle: column creation, registration, and evaluation."""
+
+  @pytest.fixture
+  def service(self, dsm):
+    svc = dsm.create_service("ns.lifecycle", save_state=False)
+    svc.load_dataframe(pd.DataFrame({"a": [1, 2, 3]}))
+    return svc
+
+  def test_i_can_add_formula_column_and_evaluate(self, service):
+    """add_column(Formula) + register_formula() + ensure_ready() produces computed values.
+
+    Why these assertions matter:
+    - col_id in ns_fast_access: ensure_ready() must write the formula column into the cache.
+    - Values [2, 4, 6]: validates the formula expression is correctly applied to all rows.
+    """
+    col_def = ColumnDefinition(col_id="__new__", col_index=-1,
+                               title="Doubled", type=ColumnType.Formula,
+                               formula="{a} * 2")
+    service.add_column(col_def)
+    service.register_formula(col_def.col_id, col_def.formula)
+    service.ensure_ready()
+
+    result = service.get_store().ns_fast_access[col_def.col_id]
+    assert list(result) == [2, 4, 6]
+
+  def test_i_can_evaluate_formula_after_adding_row(self, service):
+    """add_row() marks formula columns dirty; ensure_ready() computes the new row.
+
+    Why these assertions matter:
+    - len(result) == 4: confirms the new row was appended and the cache extended.
+    - result[3] == 20: confirms the formula was recalculated for the new row (a=10, * 2).
+    - result[0] == 2: confirms existing rows are not corrupted by the recalculation.
+    """
+    col_def = ColumnDefinition(col_id="__new__", col_index=-1,
+                               title="Doubled", type=ColumnType.Formula,
+                               formula="{a} * 2")
+    service.add_column(col_def)
+    service.register_formula(col_def.col_id, col_def.formula)
+    service.ensure_ready()
+
+    service.add_row(row_data={"a": 10})
+    service.ensure_ready()
+
+    result = service.get_store().ns_fast_access[col_def.col_id]
+    assert len(result) == 4
+    assert result[0] == 2
+    assert result[3] == 20

tests/core/data/test_dataservicesmanager.py

@@ -0,0 +1,121 @@
+"""Unit tests for DataServicesManager."""
+import pandas as pd
+
+
+class TestDataServicesManagerServiceLifecycle:
+  def test_i_can_create_a_service(self, dsm):
+    """create_service() returns a DataService accessible by grid_id."""
+    service = dsm.create_service("ns.tbl", save_state=False)
+    
+    assert service is not None
+    assert service.get_id() is not None
+    assert dsm.get_service(service.get_id()) is service
+  
+  def test_i_can_create_service_with_correct_table_name(self, dsm):
+    """create_service() sets the table_name on the returned DataService.
+  
+    create_service() calls service.set_table_name() internally.
+    This test verifies the side effect is applied before returning the service.
+    """
+    service = dsm.create_service("ns.my_table", save_state=False)
+    
+    assert service.table_name == "ns.my_table"
+  
+  def test_i_can_create_service_forcing_the_id(self, dsm):
+    """create_service() sets the table_name on the returned DataService.
+  
+    create_service() calls service.set_table_name() internally.
+    This test verifies the side effect is applied before returning the service.
+    """
+    service = dsm.create_service("ns.my_table", _id="grid_id", save_state=False)
+    
+    assert service.get_id() == "grid_id"
+  
+  def test_i_can_get_a_service_by_grid_id(self, dsm):
+    """get_service() returns the correct service."""
+    svc1 = dsm.create_service("ns.t1", _id="g1", save_state=False)
+    svc2 = dsm.create_service("ns.t2", _id="g2", save_state=False)
+    
+    assert dsm.get_service("g1") is svc1
+    assert dsm.get_service("g2") is svc2
+  
+  def test_i_cannot_get_a_nonexistent_service(self, dsm):
+    """get_service() returns None for unknown grid_id."""
+    assert dsm.get_service("does_not_exist") is None
+  
+  def test_i_can_remove_a_service(self, dsm):
+    """remove_service() unregisters the service."""
+    service = dsm.create_service("ns.rm", save_state=False)
+    dsm.remove_service(service.get_id())
+    
+    assert dsm.get_service(service.get_id()) is None
+  
+  def test_i_can_remove_a_nonexistent_service_without_error(self, dsm):
+    """remove_service() on unknown grid_id does not raise."""
+    dsm.remove_service("ghost")  # should not raise
+  
+  def test_i_can_restore_a_service(self, dsm):
+    """restore_service() creates and registers a service if not already present."""
+    service = dsm.restore_service("grid_restore")
+    
+    assert service is not None
+    assert dsm.get_service("grid_restore") is service
+    assert service.get_id() == "grid_restore"
+  
+  def test_i_can_restore_existing_service(self, dsm):
+    """restore_service() returns the existing service when already registered."""
+    original = dsm.create_service("ns.e", _id="grid_exist", save_state=False)
+    restored = dsm.restore_service("grid_exist")
+    
+    assert restored is original
+
+
+class TestDataServicesManagerFormulaEngine:
+  
+  def test_i_can_get_formula_engine(self, dsm):
+    """get_formula_engine() returns the shared FormulaEngine instance."""
+    engine = dsm.get_formula_engine()
+    assert engine is not None
+  
+  def test_i_can_verify_shared_formula_engine(self, dsm):
+    """All services share the same FormulaEngine from DataServicesManager."""
+    svc1 = dsm.create_service("ns.fe1", save_state=False)
+    svc2 = dsm.create_service("ns.fe2", save_state=False)
+    
+    assert svc1.get_formula_engine() is svc2.get_formula_engine()
+    assert svc1.get_formula_engine() is dsm.get_formula_engine()
+  
+  def test_i_can_resolve_store_by_table_name(self, dsm):
+    """FormulaEngine resolver finds the DataStore for a given table name."""
+    service = dsm.create_service("ns.resolver", save_state=False)
+    df = pd.DataFrame({"a": [1, 2]})
+    service.load_dataframe(df)
+    
+    store = dsm._resolve_store_for_table("ns.resolver")
+    
+    assert store is service.get_store()
+  
+  def test_i_can_resolve_correct_store_among_multiple_services(self, dsm):
+    """_resolve_store_for_table() identifies the right store when multiple services are registered.
+  
+    The resolver iterates over all registered services and must return the store
+    whose service has a matching table_name, not another service's store.
+    """
+    svc_a = dsm.create_service("ns.table_a", save_state=False)
+    svc_b = dsm.create_service("ns.table_b", save_state=False)
+    
+    df = pd.DataFrame({"x": [10, 20]})
+    svc_a.load_dataframe(df)
+    svc_b.load_dataframe(df.copy())
+    
+    store_a = dsm._resolve_store_for_table("ns.table_a")
+    store_b = dsm._resolve_store_for_table("ns.table_b")
+    
+    assert store_a is svc_a.get_store()
+    assert store_b is svc_b.get_store()
+    assert store_a is not store_b
+  
+  def test_i_cannot_resolve_unknown_table(self, dsm):
+    """FormulaEngine resolver returns None for an unknown table name."""
+    result = dsm._resolve_store_for_table("unknown.table")
+    assert result is None