diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..4069cea --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..cab4603 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/components/datagrid_new/DataGridApp.py b/src/components/datagrid_new/DataGridApp.py index 5493530..47ef8c2 100644 --- a/src/components/datagrid_new/DataGridApp.py +++ b/src/components/datagrid_new/DataGridApp.py @@ -1,6 +1,8 @@ +import asyncio import json import logging +from fasthtml.components import Div, sse_message from fasthtml.core import EventStream from fasthtml.fastapp import fast_app from starlette.datastructures import UploadFile @@ -138,9 +140,21 @@ def post(session, _id: str, state: str, args: str = None): instance = InstanceManager.get(session, _id) return instance.manage_state_changed(state, args) + @rt(Routes.YieldRow) async def get(session, _id: str): logger.debug(f"Entering {Routes.YieldRow} with args {_id=}") instance = InstanceManager.get(session, _id) - return EventStream(instance.mk_async_lasy_body_content()) - \ No newline at end of file + return EventStream(instance.mk_lazy_body_content()) + + +async def number_generator2(): + for i in range(20): + yield sse_message(Div(i * 5 + 1)) + yield sse_message(Div(i * 5 + 2)) + yield sse_message(Div(i * 5 + 3)) + yield sse_message(Div(i * 5 + 4)) + yield sse_message(Div(i * 5 + 5)) + await asyncio.sleep(0.1) + + yield f"event: close\ndata: \n\n" diff --git a/src/components/datagrid_new/assets/Datagrid.js b/src/components/datagrid_new/assets/Datagrid.js index ca9c43a..2df4c05 100644 --- a/src/components/datagrid_new/assets/Datagrid.js +++ b/src/components/datagrid_new/assets/Datagrid.js @@ -1,6 +1,10 @@ function bindDatagrid(datagridId, allowColumnsReordering) { bindScrollbars(datagridId); makeResizable(datagridId) + + document.body.addEventListener('htmx:sseBeforeMessage', function (e) { + console.log("htmx:sseBeforeMessage", e) + }) } function bindScrollbars(datagridId) { @@ -494,4 +498,5 @@ function onAfterSettle(datagridId, event) { if (response.includes("hx-on::before-settle")) { bindDatagrid(datagridId) } -} \ No newline at end of file +} + diff --git a/src/components/datagrid_new/components/DataGrid.py b/src/components/datagrid_new/components/DataGrid.py index a13af75..332e4fe 100644 --- a/src/components/datagrid_new/components/DataGrid.py +++ b/src/components/datagrid_new/components/DataGrid.py @@ -1,3 +1,4 @@ +import asyncio import copy import logging from io import BytesIO @@ -23,7 +24,7 @@ from components_helpers import mk_icon, mk_ellipsis from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon from core.instance_manager import InstanceManager from core.settings_management import SettingsManager -from core.utils import get_unique_id, make_safe_id, timed, profile_function +from core.utils import get_unique_id, make_safe_id, timed logger = logging.getLogger("DataGrid") @@ -60,6 +61,7 @@ class DataGrid(BaseComponent): self._state: DataGridState = self._db.load_state() self._settings: DataGridSettings = grid_settings or self._db.load_settings() self._df: DataFrame | None = self._db.load_dataframe() + self._fast_access = self._init_fast_access(self._df) # update boundaries if possible self.set_boundaries(boundaries) @@ -127,6 +129,7 @@ class DataGrid(BaseComponent): col_id, _get_column_type(self._df[make_safe_id(col_id)].dtype)) for col_index, col_id in enumerate(df.columns)] + self._fast_access = self._init_fast_access(self._df) if save_state: self._db.save_all(None, self._state, self._df) @@ -387,7 +390,7 @@ class DataGrid(BaseComponent): id=f"scb_{self._id}", ) - @profile_function + @timed def mk_table(self, oob=False): htmx_extra_params = { "hx-on::before-settle": f"onAfterSettle('{self._id}', event);", @@ -441,8 +444,8 @@ class DataGrid(BaseComponent): _mk_keyboard_management(), Div( self.mk_table_header(), - # self.mk_table_body(), - self.mk_table_body_lasy(), + #self.mk_table_body(), + self.mk_table_body_lazy(), self.mk_table_footer(), cls="dt2-inner-table"), cls="dt2-table", @@ -477,25 +480,25 @@ class DataGrid(BaseComponent): header_class = "dt2-row dt2-header" + "" if self._settings.header_visible else " hidden" return Div( + Div(sse_swap="message"), *[_mk_header(col_def) for col_def in self._state.columns], cls=header_class, id=f"th_{self._id}" ) - def mk_table_body_lasy(self): - df = self._get_filtered_df() + def mk_table_body_lazy(self): + max_height = self._compute_body_max_height() return Div( - # *self.mk_lasy_body_content(df), hx_ext="sse", - sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}", - hx_swap="beforeend show:bottom", + sse_connect=f"{ROUTE_ROOT}{Routes.YieldRow}?_id={self._id}", + sse_close='close', sse_swap="message", + hx_swap="beforeend", cls="dt2-body", style=f"max-height:{max_height}px;", id=f"tb_{self._id}", - hx_vals=f'{{"_id": "{self._id}"}}', ) def mk_table_body(self): @@ -526,22 +529,21 @@ class DataGrid(BaseComponent): id=f"tf_{self._id}" ) - async def mk_async_lasy_body_content(self, df): - for row_index in df.index: + async def mk_lazy_body_content(self): + df = self._get_filtered_df() + for i, row_index in enumerate(df.index): yield sse_message(Div( *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)], cls="dt2-row", data_row=f"{row_index}", id=f"tr_{self._id}-{row_index}", )) - - def mk_lasy_body_content(self, df): - return [Div( - *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)], - cls="dt2-row", - data_row=f"{row_index}", - id=f"tr_{self._id}-{row_index}", - ) for row_index in df.index] + if i % 50 == 0: + await asyncio.sleep(0.01) + logger.debug(f"yielding row {i}") + + logger.debug(f"yielding close event") + yield f"event: close\ndata: \n\n" def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState): if not col_def.usable: @@ -557,21 +559,20 @@ class DataGrid(BaseComponent): style=f"width:{col_def.width}px;", cls="dt2-cell") - @profile_function def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState): - def mk_bool(value): - return MyDiv(mk_my_icon(icon_checked if value else icon_unchecked, can_select=False), + def mk_bool(_value): + return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False), cls="dt2-cell-content-checkbox") - def mk_text(value): - return mk_my_ellipsis(value, cls="dt2-cell-content-text") + def mk_text(_value): + return mk_my_ellipsis(_value, cls="dt2-cell-content-text") - def mk_number(value): - return mk_my_ellipsis(value, cls="dt2-cell-content-number") + def mk_number(_value): + return mk_my_ellipsis(_value, cls="dt2-cell-content-number") - def process_cell_content(value): - value_str = str(value) + def process_cell_content(_value): + value_str = str(_value) if FILTER_INPUT_CID not in self._state.filtered or ( keyword := self._state.filtered[FILTER_INPUT_CID]) is None: @@ -588,15 +589,17 @@ class DataGrid(BaseComponent): return tuple(res) column_type = col_def.type + # value = self._df.iloc[row_index, col_def.col_index] + value = self._fast_access[col_def.col_id][row_index] if column_type == ColumnType.Bool: - content = mk_bool(self._df.iloc[row_index, col_def.col_index]) + content = mk_bool(value) elif column_type == ColumnType.Number: - content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index])) + content = mk_number(process_cell_content(value)) elif column_type == ColumnType.RowIndex: content = mk_number(row_index) else: - content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index])) + content = mk_text(process_cell_content(value)) return content @@ -859,6 +862,25 @@ class DataGrid(BaseComponent): return True + @staticmethod + def _init_fast_access(df): + """ + Generates a fast-access dictionary for a DataFrame. + + This method converts the columns of the provided DataFrame into NumPy arrays + and stores them as values in a dictionary, using the column names as keys. + This allows for efficient access to the data stored in the DataFrame. + + Args: + df (DataFrame): The input pandas DataFrame whose columns are to be converted + into a dictionary of NumPy arrays. + + Returns: + dict: A dictionary where the keys are the column names of the input DataFrame + and the values are the corresponding column values as NumPy arrays. + """ + return {col: df[col].to_numpy() for col in df.columns} + @timed def __ft__(self): return Div( diff --git a/src/components/repositories/RepositoriesApp.py b/src/components/repositories/RepositoriesApp.py index e146657..e43351b 100644 --- a/src/components/repositories/RepositoriesApp.py +++ b/src/components/repositories/RepositoriesApp.py @@ -20,7 +20,7 @@ def get(session): @rt(Routes.AddRepository) -def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries:str): +def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries: str): logger.debug( f"Entering {Routes.AddRepository} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository=}, {table=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) # Repository @@ -34,8 +34,9 @@ def get(session, _id: str, repository_name: str): @rt(Routes.AddTable) -def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries:str): - logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}") +def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str): + logger.debug( + f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries)) @@ -48,7 +49,8 @@ def put(session, _id: str, repository: str): @rt(Routes.ShowTable) -def get(session, _id: str, repository: str, table: str, tab_boundaries:str): - logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}") +def get(session, _id: str, repository: str, table: str, tab_boundaries: str): + logger.debug( + f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) return instance.show_table(repository, table, json.loads(tab_boundaries)) diff --git a/src/core/fasthtml_helper.py b/src/core/fasthtml_helper.py index eb0c633..c40bb17 100644 --- a/src/core/fasthtml_helper.py +++ b/src/core/fasthtml_helper.py @@ -26,12 +26,12 @@ def to_html(item): class MyFt: def __init__(self, name, *args, **kwargs): self.name = name - self.args = args - self.kwargs = kwargs + self.children = args + self.attrs = kwargs def to_html(self): - body_items = [to_html(item) for item in self.args] - return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.kwargs.items())}>{' '.join(body_items)}" + body_items = [to_html(item) for item in self.children] + return f"<{self.name} {' '.join(f'{attr_map.get(k, k)}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}" def __ft__(self): return NotStr(self.to_html()) diff --git a/src/core/utils.py b/src/core/utils.py index 72a56ac..0b93161 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,6 +1,7 @@ import ast import base64 import cProfile +import functools import hashlib import importlib import inspect @@ -424,6 +425,7 @@ def split_host_port(url): def timed(func): + @functools.wraps(func) def wrapper(*args, **kwargs): start = time.perf_counter() result = func(*args, **kwargs) @@ -449,11 +451,14 @@ def timed(func): def profile_function(func): + @functools.wraps(func) def wrapper(*args, **kwargs): profiler = cProfile.Profile() - profiler.enable() - result = func(*args, **kwargs) - profiler.disable() + try: + profiler.enable() + result = func(*args, **kwargs) + finally: + profiler.disable() # Determine class name if any class_name = None @@ -522,4 +527,4 @@ class UnreferencedNamesVisitor(ast.NodeVisitor): :rtype: """ self.names.add(node.arg) - self.visit_selected(node, ["value"]) + self.visit_selected(node, ["value"]) \ No newline at end of file diff --git a/src/main.py b/src/main.py index 7141fcb..814ba1c 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,4 @@ # global layout -import asyncio import logging.config import random from asyncio import sleep @@ -216,21 +215,23 @@ app, rt = fast_app( pico=False, ) + # ------------------------- # Profiling middleware # ------------------------- -# @app.middleware("http") -# async def timing_middleware(request, call_next): -# start_total = time.perf_counter() -# -# # Call the next middleware or route handler -# response = await call_next(request) -# -# end_total = time.perf_counter() -# elapsed = end_total - start_total -# -# print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}") -# return response +@app.middleware("http") +async def timing_middleware(request, call_next): + import time + start_total = time.perf_counter() + + # Call the next middleware or route handler + response = await call_next(request) + + end_total = time.perf_counter() + elapsed = end_total - start_total + + print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}") + return response settings_manager = SettingsManager() @@ -279,7 +280,7 @@ shutdown_event = signal_shutdown() async def number_generator(): - while True: # not shutdown_event.is_set(): + while True: # not shutdown_event.is_set(): data = Article(random.randint(1, 100)) print(data) yield sse_message(data) @@ -333,7 +334,7 @@ def not_found(path: str, session=None): setup_toasts(app) -async def main(): +def main(): logger.info(f" Starting FastHTML server on http://localhost:{APP_PORT}") serve(port=APP_PORT) @@ -341,9 +342,4 @@ async def main(): if __name__ == "__main__": # Start your application logger.info("Application starting...") - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("\nStopping application...") - except Exception as e: - logger.error(f"Error: {e}") + main()