From 33970c9c97fe1fc3e283b724b4203773898e9d0a Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Sat, 23 Aug 2025 22:26:14 +0200 Subject: [PATCH] Added lazy loading when showing grid to improve performance --- .gitignore | 1 + Makefile | 4 +- README.md | 7 + src/assets/main.js | 12 +- src/components/datagrid_new/DataGridApp.py | 7 + .../datagrid_new/assets/Datagrid.js | 227 +++++++++++++++++- .../datagrid_new/components/DataGrid.py | 133 +++++++--- .../datagrid_new/components/commands.py | 45 +--- src/components/datagrid_new/constants.py | 8 +- src/components/datagrid_new/settings.py | 1 + .../repositories/RepositoriesApp.py | 12 +- src/core/fasthtml_helper.py | 76 ++++++ src/core/utils.py | 67 +++++- src/main.py | 51 ++-- tests/helpers.py | 11 +- tests/test_datagrid_new.py | 15 ++ 16 files changed, 567 insertions(+), 110 deletions(-) create mode 100644 src/core/fasthtml_helper.py diff --git a/.gitignore b/.gitignore index 142f5be..2eb4f17 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ tools.db .mytools_db .idea/MyManagingTools.iml .idea/misc.xml +**/*.prof # Created by .ignore support plugin (hsz.mobi) ### Python template diff --git a/Makefile b/Makefile index 85ef873..e885a8a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,9 @@ clean: rm -rf Untitled*.ipynb rm -rf .ipynb_checkpoints rm -rf src/tools.db + rm -rf src/*.out + rm -rf src/*.prof find . -name '.sesskey' -exec rm -rf {} + find . -name '.pytest_cache' -exec rm -rf {} + find . -name '__pycache__' -exec rm -rf {} + - find . -name 'debug.txt' -exec rm -rf {} \ No newline at end of file + find . -name 'debug.txt' -exec rm -rf {} diff --git a/README.md b/README.md index 4664d32..0eb3c4f 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,11 @@ docker-compose down 1. **Rebuild**: ```shell docker-compose build +``` + +# Profiling +```shell +cd src +python -m cProfile -o profile.out main.py +snakeviz profile.out # 'pip install snakeviz' if snakeviz is not installed ``` \ No newline at end of file diff --git a/src/assets/main.js b/src/assets/main.js index 33e777d..97a8be6 100644 --- a/src/assets/main.js +++ b/src/assets/main.js @@ -25,11 +25,19 @@ function bindTooltipsWithDelegation() { // Add a single mouseenter and mouseleave listener to the parent element element.addEventListener("mouseenter", (event) => { + //console.debug("Entering element", event.target) + const cell = event.target.closest("[data-tooltip]"); - if (!cell) return; + if (!cell) { + // console.debug(" No 'data-tooltip' attribute found. Stopping."); + return; + } const no_tooltip = element.hasAttribute("mmt-no-tooltip"); - if (no_tooltip) return; + if (no_tooltip) { + // console.debug(" Attribute 'mmt-no-tooltip' found. Cancelling."); + return; + } const content = cell.querySelector(".truncate") || cell; const isOverflowing = content.scrollWidth > content.clientWidth; diff --git a/src/components/datagrid_new/DataGridApp.py b/src/components/datagrid_new/DataGridApp.py index fda1ed0..0625ba6 100644 --- a/src/components/datagrid_new/DataGridApp.py +++ b/src/components/datagrid_new/DataGridApp.py @@ -136,3 +136,10 @@ def post(session, _id: str, state: str, args: str = None): logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}") instance = InstanceManager.get(session, _id) return instance.manage_state_changed(state, args) + + +@rt(Routes.GetPage) +def get(session, _id: str, page_index: int): + logger.debug(f"Entering {Routes.GetPage} with args {_id=}, {page_index=}") + instance = InstanceManager.get(session, _id) + return instance.mk_body_content_page(page_index) diff --git a/src/components/datagrid_new/assets/Datagrid.js b/src/components/datagrid_new/assets/Datagrid.js index ca9c43a..479dba7 100644 --- a/src/components/datagrid_new/assets/Datagrid.js +++ b/src/components/datagrid_new/assets/Datagrid.js @@ -1,6 +1,6 @@ function bindDatagrid(datagridId, allowColumnsReordering) { - bindScrollbars(datagridId); - makeResizable(datagridId) + manageScrollbars(datagridId, true); + makeResizable(datagridId); } function bindScrollbars(datagridId) { @@ -21,7 +21,7 @@ function bindScrollbars(datagridId) { const table = datagrid.querySelector(".dt2-table"); if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) { - console.error("Essential scrollbar or content elements are missing in the datagrid."); + console.error("Essential scrollbars or content elements are missing in the datagrid."); return; } @@ -176,6 +176,224 @@ function bindScrollbars(datagridId) { }); } +function manageScrollbars(datagridId, binding) { + console.debug("manageScrollbars on element " + datagridId + " with binding=" + binding); + + const datagrid = document.getElementById(datagridId); + + if (!datagrid) { + console.error(`Datagrid with id "${datagridId}" not found.`); + return; + } + + const verticalScrollbar = datagrid.querySelector(".dt2-scrollbars-vertical"); + const verticalWrapper = datagrid.querySelector(".dt2-scrollbars-vertical-wrapper"); + const horizontalScrollbar = datagrid.querySelector(".dt2-scrollbars-horizontal"); + const horizontalWrapper = datagrid.querySelector(".dt2-scrollbars-horizontal-wrapper"); + const body = datagrid.querySelector(".dt2-body"); + const table = datagrid.querySelector(".dt2-table"); + + if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) { + console.error("Essential scrollbars or content elements are missing in the datagrid."); + return; + } + + const computeScrollbarVisibility = () => { + // Determine if the content is clipped + const isVerticalRequired = body.scrollHeight > body.clientHeight; + const isHorizontalRequired = table.scrollWidth > table.clientWidth; + + // Show or hide the scrollbar wrappers + requestAnimationFrame(() => { + verticalWrapper.style.display = isVerticalRequired ? "block" : "none"; + horizontalWrapper.style.display = isHorizontalRequired ? "block" : "none"; + }); + }; + + const computeScrollbarSize = () => { + // Vertical scrollbar height + const visibleHeight = body.clientHeight; + const totalHeight = body.scrollHeight; + const wrapperHeight = verticalWrapper.offsetHeight; + + let scrollbarHeight = 0; + if (totalHeight > 0) { + scrollbarHeight = (visibleHeight / totalHeight) * wrapperHeight; + } + + // Horizontal scrollbar width + const visibleWidth = table.clientWidth; + const totalWidth = table.scrollWidth; + const wrapperWidth = horizontalWrapper.offsetWidth; + + let scrollbarWidth = 0; + if (totalWidth > 0) { + scrollbarWidth = (visibleWidth / totalWidth) * wrapperWidth; + } + + requestAnimationFrame(() => { + verticalScrollbar.style.height = `${scrollbarHeight}px`; + horizontalScrollbar.style.width = `${scrollbarWidth}px`; + }); + }; + + const updateVerticalScrollbarForMouseWheel = () => { + const maxScrollTop = body.scrollHeight - body.clientHeight; + const wrapperHeight = verticalWrapper.offsetHeight; + + if (maxScrollTop > 0) { + const scrollRatio = wrapperHeight / body.scrollHeight; + verticalScrollbar.style.top = `${body.scrollTop * scrollRatio}px`; + } + }; + + if (binding) { + // Clean up existing managers if they exist + if (datagrid._managers) { + // Remove drag events + if (datagrid._managers.dragManager) { + verticalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.verticalMouseDown); + horizontalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.horizontalMouseDown); + document.removeEventListener("mousemove", datagrid._managers.dragManager.mouseMove); + document.removeEventListener("mouseup", datagrid._managers.dragManager.mouseUp); + } + + // Remove wheel events + if (datagrid._managers.wheelManager) { + body.removeEventListener("wheel", datagrid._managers.wheelManager.handleWheelScrolling); + } + + // Remove resize events + if (datagrid._managers.resizeManager) { + window.removeEventListener("resize", datagrid._managers.resizeManager.handleResize); + } + } + + // Create managers + const dragManager = { + isDragging: false, + startY: 0, + startX: 0, + + updateVerticalScrollbar: (deltaX, deltaY) => { + const wrapperHeight = verticalWrapper.offsetHeight; + const scrollbarHeight = verticalScrollbar.offsetHeight; + const maxScrollTop = body.scrollHeight - body.clientHeight; + const scrollRatio = maxScrollTop / (wrapperHeight - scrollbarHeight); + + let newTop = parseFloat(verticalScrollbar.style.top || "0") + deltaY; + newTop = Math.max(0, Math.min(newTop, wrapperHeight - scrollbarHeight)); + + verticalScrollbar.style.top = `${newTop}px`; + body.scrollTop = newTop * scrollRatio; + }, + + updateHorizontalScrollbar: (deltaX, deltaY) => { + const wrapperWidth = horizontalWrapper.offsetWidth; + const scrollbarWidth = horizontalScrollbar.offsetWidth; + const maxScrollLeft = table.scrollWidth - table.clientWidth; + const scrollRatio = maxScrollLeft / (wrapperWidth - scrollbarWidth); + + let newLeft = parseFloat(horizontalScrollbar.style.left || "0") + deltaX; + newLeft = Math.max(0, Math.min(newLeft, wrapperWidth - scrollbarWidth)); + + horizontalScrollbar.style.left = `${newLeft}px`; + table.scrollLeft = newLeft * scrollRatio; + }, + + verticalMouseDown: (e) => { + disableTooltip(); + dragManager.isDragging = true; + dragManager.startY = e.clientY; + dragManager.startX = e.clientX; + document.body.style.userSelect = "none"; + verticalScrollbar.classList.add("dt2-dragging"); + }, + + horizontalMouseDown: (e) => { + disableTooltip(); + dragManager.isDragging = true; + dragManager.startY = e.clientY; + dragManager.startX = e.clientX; + document.body.style.userSelect = "none"; + horizontalScrollbar.classList.add("dt2-dragging"); + }, + + mouseMove: (e) => { + if (dragManager.isDragging) { + const deltaY = e.clientY - dragManager.startY; + const deltaX = e.clientX - dragManager.startX; + + // Determine which scrollbar is being dragged + if (verticalScrollbar.classList.contains("dt2-dragging")) { + dragManager.updateVerticalScrollbar(deltaX, deltaY); + } else if (horizontalScrollbar.classList.contains("dt2-dragging")) { + dragManager.updateHorizontalScrollbar(deltaX, deltaY); + } + + // Reset start points for next update + dragManager.startY = e.clientY; + dragManager.startX = e.clientX; + } + }, + + mouseUp: () => { + dragManager.isDragging = false; + document.body.style.userSelect = ""; + verticalScrollbar.classList.remove("dt2-dragging"); + horizontalScrollbar.classList.remove("dt2-dragging"); + enableTooltip(); + } + }; + + const wheelManager = { + handleWheelScrolling: (event) => { + const deltaX = event.deltaX; + const deltaY = event.deltaY; + + // Scroll the body and table content + body.scrollTop += deltaY; // Vertical scrolling + table.scrollLeft += deltaX; // Horizontal scrolling + + // Update the vertical scrollbar position + updateVerticalScrollbarForMouseWheel(); + + // Prevent default behavior to fully manage the scroll + event.preventDefault(); + } + }; + + const resizeManager = { + handleResize: () => { + computeScrollbarVisibility(); + computeScrollbarSize(); + updateVerticalScrollbarForMouseWheel(); + } + }; + + // Store managers on datagrid for cleanup + datagrid._managers = { + dragManager, + wheelManager, + resizeManager + }; + + // Bind events + verticalScrollbar.addEventListener("mousedown", dragManager.verticalMouseDown); + horizontalScrollbar.addEventListener("mousedown", dragManager.horizontalMouseDown); + document.addEventListener("mousemove", dragManager.mouseMove); + document.addEventListener("mouseup", dragManager.mouseUp); + + body.addEventListener("wheel", wheelManager.handleWheelScrolling, {passive: false}); + + window.addEventListener("resize", resizeManager.handleResize); + } + + // Always execute computations + computeScrollbarVisibility(); + computeScrollbarSize(); +} + function makeResizable(datagridId) { console.debug("makeResizable on element " + datagridId); @@ -494,4 +712,5 @@ function onAfterSettle(datagridId, event) { if (response.includes("hx-on::before-settle")) { bindDatagrid(datagridId) } -} \ No newline at end of file +} + diff --git a/src/components/datagrid_new/components/DataGrid.py b/src/components/datagrid_new/components/DataGrid.py index 4709ac6..16d5926 100644 --- a/src/components/datagrid_new/components/DataGrid.py +++ b/src/components/datagrid_new/components/DataGrid.py @@ -1,4 +1,5 @@ import copy +import html import logging from io import BytesIO from typing import Literal, Any @@ -20,9 +21,10 @@ from components.datagrid_new.db_management import DataGridDbManager from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \ DataGridFooterConf, DataGridState, DataGridSettings, DatagridView from components_helpers import mk_icon, mk_ellipsis +from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon from core.instance_manager import InstanceManager from core.settings_management import SettingsManager -from core.utils import get_unique_id, make_safe_id +from core.utils import get_unique_id, make_safe_id, timed logger = logging.getLogger("DataGrid") @@ -59,6 +61,8 @@ class DataGrid(BaseComponent): self._state: DataGridState = self._db.load_state() self._settings: DataGridSettings = grid_settings or self._db.load_settings() self._df: DataFrame | None = self._db.load_dataframe() + self._fast_access = self._init_fast_access(self._df) + self._total_rows = len(self._df) if self._df is not None else 0 # update boundaries if possible self.set_boundaries(boundaries) @@ -118,14 +122,23 @@ class DataGrid(BaseComponent): else: return ColumnType.Text # Default to Text if no match + def _init_columns(_df): + columns = [DataGridColumnState(make_safe_id(col_id), + col_index, + col_id, + _get_column_type(self._df[make_safe_id(col_id)].dtype)) + for col_index, col_id in enumerate(_df.columns)] + if self._state.row_index: + columns.insert(0, DataGridColumnState(make_safe_id(ROW_INDEX_ID), -1, " ", ColumnType.RowIndex)) + + return columns + self._df = df.copy() self._df.columns = self._df.columns.map(make_safe_id) # make sure column names are trimmed self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index] - self._state.columns = [DataGridColumnState(make_safe_id(col_id), - col_index, - col_id, - _get_column_type(self._df[make_safe_id(col_id)].dtype)) - for col_index, col_id in enumerate(df.columns)] + self._state.columns = _init_columns(df) # use df not self._df to keep the original title + self._fast_access = self._init_fast_access(self._df) + self._total_rows = len(self._df) if self._df is not None else 0 if save_state: self._db.save_all(None, self._state, self._df) @@ -205,6 +218,7 @@ class DataGrid(BaseComponent): self._state.columns = new_columns_states + self._fast_access = self._init_fast_access(self._df) self._views.recompute_need_save() self._db.save_all(self._settings, self._state, self._df if new_column else None) @@ -439,7 +453,7 @@ class DataGrid(BaseComponent): _mk_keyboard_management(), Div( self.mk_table_header(), - self.mk_table_body(), + self.mk_table_body_page(), self.mk_table_footer(), cls="dt2-inner-table"), cls="dt2-table", @@ -479,20 +493,18 @@ class DataGrid(BaseComponent): id=f"th_{self._id}" ) - def mk_table_body(self): - df = self._get_filtered_df() + def mk_table_body_page(self): + """ + This function is used to update the table body when the vertical scrollbar reaches the end + A new page is added when requested + """ max_height = self._compute_body_max_height() return Div( - *[Div( - *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)], - cls="dt2-row", - data_row=f"{row_index}", - id=f"tr_{self._id}-{row_index}", - ) for row_index in df.index], + *self.mk_body_content_page(0), cls="dt2-body", style=f"max-height:{max_height}px;", - id=f"tb_{self._id}" + id=f"tb_{self._id}", ) def mk_table_footer(self): @@ -507,34 +519,55 @@ class DataGrid(BaseComponent): id=f"tf_{self._id}" ) + def mk_body_content_page(self, page_index: int): + df = self._get_filtered_df() + start = page_index * DATAGRID_PAGE_SIZE + end = start + DATAGRID_PAGE_SIZE + if self._total_rows > end: + last_row = df.index[end - 1] + else: + last_row = None + + rows = [Div( + *[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)], + cls="dt2-row", + data_row=f"{row_index}", + id=f"tr_{self._id}-{row_index}", + **self.commands.get_page(page_index + 1) if row_index == last_row else {} + ) for row_index in df.index[start:end]] + + rows.append(Script(f"manageScrollbars('{self._id}', false);"), ) + + return rows + def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState): if not col_def.usable: return None if not col_def.visible: - return Div(cls="dt2-col-hidden") + return MyDiv(cls="dt2-col-hidden") content = self.mk_body_cell_content(col_pos, row_index, col_def) - return Div(content, - data_col=col_def.col_id, - style=f"width:{col_def.width}px;", - cls="dt2-cell") + return MyDiv(content, + data_col=col_def.col_id, + style=f"width:{col_def.width}px;", + cls="dt2-cell") def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState): - def mk_bool(value): - return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False), - cls="dt2-cell-content-checkbox") + def mk_bool(_value): + return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False), + cls="dt2-cell-content-checkbox") - def mk_text(value): - return mk_ellipsis(value, cls="dt2-cell-content-text") + def mk_text(_value): + return mk_my_ellipsis(_value, cls="dt2-cell-content-text") - def mk_number(value): - return mk_ellipsis(value, cls="dt2-cell-content-number") + def mk_number(_value): + return mk_my_ellipsis(_value, cls="dt2-cell-content-number") - def process_cell_content(value): - value_str = str(value) + def process_cell_content(_value): + value_str = html.escape(str(_value)) if FILTER_INPUT_CID not in self._state.filtered or ( keyword := self._state.filtered[FILTER_INPUT_CID]) is None: @@ -545,21 +578,22 @@ class DataGrid(BaseComponent): return value_str len_keyword = len(keyword) - res = [Span(value_str[:index])] if index > 0 else [] - res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")] - res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else [] + res = [MySpan(value_str[:index])] if index > 0 else [] + res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")] + res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else [] return tuple(res) column_type = col_def.type + value = self._fast_access[col_def.col_id][row_index] if column_type == ColumnType.Bool: - content = mk_bool(self._df.iloc[row_index, col_def.col_index]) + content = mk_bool(value) elif column_type == ColumnType.Number: - content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index])) + content = mk_number(process_cell_content(value)) elif column_type == ColumnType.RowIndex: content = mk_number(row_index) else: - content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index])) + content = mk_text(process_cell_content(value)) return content @@ -822,6 +856,31 @@ class DataGrid(BaseComponent): return True + @staticmethod + def _init_fast_access(df): + """ + Generates a fast-access dictionary for a DataFrame. + + This method converts the columns of the provided DataFrame into NumPy arrays + and stores them as values in a dictionary, using the column names as keys. + This allows for efficient access to the data stored in the DataFrame. + + Args: + df (DataFrame): The input pandas DataFrame whose columns are to be converted + into a dictionary of NumPy arrays. + + Returns: + dict: A dictionary where the keys are the column names of the input DataFrame + and the values are the corresponding column values as NumPy arrays. + """ + if df is None: + return {} + + res = {col: df[col].to_numpy() for col in df.columns} + res[ROW_INDEX_ID] = df.index.to_numpy() + return res + + @timed def __ft__(self): return Div( Div( @@ -844,7 +903,7 @@ class DataGrid(BaseComponent): @staticmethod def new(session, data, index=None): datagrid = DataGrid(session, DataGrid.create_component_id(session)) - #dataframe = DataFrame(data, index=index) + # dataframe = DataFrame(data, index=index) dataframe = DataFrame(data) datagrid.init_from_dataframe(dataframe) return datagrid diff --git a/src/components/datagrid_new/components/commands.py b/src/components/datagrid_new/components/commands.py index 1cf010c..7ddef46 100644 --- a/src/components/datagrid_new/components/commands.py +++ b/src/components/datagrid_new/components/commands.py @@ -91,12 +91,21 @@ class DataGridCommandManager(BaseCommandManager): return { "hx-post": f"{ROUTE_ROOT}{Routes.OnClick}", "hx-target": f"#tsm_{self._id}", - "hx-trigger" : "click", + "hx-trigger": "click", "hx-swap": "outerHTML", "hx-vals": f'js:{{_id: "{self._id}", cell_id:getCellId(event), modifier:getClickModifier(event), boundaries: getCellBoundaries(event)}}', "hx-on::before-request": f'validateOnClickRequest("{self._id}", event)', } + def get_page(self, page_index=0): + return { + "hx-get": f"{ROUTE_ROOT}{Routes.GetPage}", + "hx-target": f"#tb_{self._id}", + "hx-swap": "beforeend", + "hx-vals": f'{{"_id": "{self._id}", "page_index": "{page_index}"}}', + "hx-trigger": f"intersect root:#tb_{self._id} once", + } + def _get_hide_show_columns_attrs(self, mode, col_defs: list, new_value, cls=""): str_col_names = ", ".join(f"'{col_def.title}'" for col_def in col_defs) tooltip_msg = f"{mode} column{'s' if len(col_defs) > 1 else ''} {str_col_names}" @@ -109,38 +118,6 @@ class DataGridCommandManager(BaseCommandManager): "data_tooltip": tooltip_msg, "cls": self.merge_class(cls, "mmt-tooltip") } - # - # @staticmethod - # def merge(*items): - # """ - # Merges multiple dictionaries into a single dictionary by combining their key-value pairs. - # If a key exists in multiple dictionaries and its value is a string, the values are concatenated. - # If the key's value is not a string, an error is raised. - # - # :param items: dictionaries to be merged. If all items are None, None is returned. - # :return: A single dictionary containing the merged key-value pairs from all input dictionaries. - # :raises NotImplementedError: If a key's value is not a string and exists in multiple input dictionaries. - # """ - # if all(item is None for item in items): - # return None - # - # res = {} - # for item in [item for item in items if item is not None]: - # - # for key, value in item.items(): - # if not key in res: - # res[key] = value - # else: - # if isinstance(res[key], str): - # res[key] += " " + value - # else: - # raise NotImplementedError("") - # - # return res - # - # @staticmethod - # def merge_class(cls1, cls2): - # return (cls1 + " " + cls2) if cls2 else cls1 class FilterAllCommands(BaseCommandManager): @@ -165,4 +142,4 @@ class FilterAllCommands(BaseCommandManager): "hx_vals": f'{{"_id": "{self._id}", "col_id":"{FILTER_INPUT_CID}"}}', "data_tooltip": "Reset filter", "cls": self.merge_class(cls, "mmt-tooltip"), - } \ No newline at end of file + } diff --git a/src/components/datagrid_new/constants.py b/src/components/datagrid_new/constants.py index e6d8ad7..678d0d4 100644 --- a/src/components/datagrid_new/constants.py +++ b/src/components/datagrid_new/constants.py @@ -17,6 +17,9 @@ CONTAINER_HEIGHT = "container_height" DATAGRID_STATE_FOOTER = "footer" +DATAGRID_PAGE_SIZE = 50 + +ROW_INDEX_ID = "__row_index__" class Routes: Filter = "/filter" # request the filtering in the grid @@ -33,6 +36,7 @@ class Routes: UpdateView = "/update_view" ShowFooterMenu = "/show_footer_menu" UpdateState = "/update_state" + GetPage = "/page" class ColumnType(Enum): @@ -44,11 +48,13 @@ class ColumnType(Enum): Choice = "Choice" List = "List" + class ViewType(Enum): Table = "Table" Chart = "Chart" Form = "Form" + class FooterAggregation(Enum): Sum = "Sum" Mean = "Mean" @@ -59,4 +65,4 @@ class FooterAggregation(Enum): FilteredMean = "FilteredMean" FilteredMin = "FilteredMin" FilteredMax = "FilteredMax" - FilteredCount = "FilteredCount" \ No newline at end of file + FilteredCount = "FilteredCount" diff --git a/src/components/datagrid_new/settings.py b/src/components/datagrid_new/settings.py index 4a036af..1971fdd 100644 --- a/src/components/datagrid_new/settings.py +++ b/src/components/datagrid_new/settings.py @@ -69,6 +69,7 @@ class DataGridSettings: class DataGridState: sidebar_visible: bool = False selected_view: str = None + row_index: bool = False columns: list[DataGridColumnState] = dataclasses.field(default_factory=list) rows: list[DataGridRowState] = dataclasses.field(default_factory=list) # only the rows that have a specific state footers: list[DataGridFooterConf] = dataclasses.field(default_factory=list) diff --git a/src/components/repositories/RepositoriesApp.py b/src/components/repositories/RepositoriesApp.py index e146657..e43351b 100644 --- a/src/components/repositories/RepositoriesApp.py +++ b/src/components/repositories/RepositoriesApp.py @@ -20,7 +20,7 @@ def get(session): @rt(Routes.AddRepository) -def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries:str): +def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries: str): logger.debug( f"Entering {Routes.AddRepository} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository=}, {table=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) # Repository @@ -34,8 +34,9 @@ def get(session, _id: str, repository_name: str): @rt(Routes.AddTable) -def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries:str): - logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}") +def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str): + logger.debug( + f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries)) @@ -48,7 +49,8 @@ def put(session, _id: str, repository: str): @rt(Routes.ShowTable) -def get(session, _id: str, repository: str, table: str, tab_boundaries:str): - logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}") +def get(session, _id: str, repository: str, table: str, tab_boundaries: str): + logger.debug( + f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}") instance = InstanceManager.get(session, _id) return instance.show_table(repository, table, json.loads(tab_boundaries)) diff --git a/src/core/fasthtml_helper.py b/src/core/fasthtml_helper.py new file mode 100644 index 0000000..016f8a2 --- /dev/null +++ b/src/core/fasthtml_helper.py @@ -0,0 +1,76 @@ +from fastcore.basics import NotStr + +from core.utils import merge_classes + +attr_map = { + "cls": "class", + "_id": "id", +} + + +def safe_attr(attr_name): + attr_name = attr_name.replace("hx_", "hx-") + attr_name = attr_name.replace("data_", "data-") + return attr_map.get(attr_name, attr_name) + + +def to_html(item): + if item is None: + return "" + elif isinstance(item, str): + return item + elif isinstance(item, (int, float, bool)): + return str(item) + elif isinstance(item, MyFt): + return item.to_html() + elif isinstance(item, NotStr): + return str(item) + else: + raise Exception(f"Unsupported type: {type(item)}, {item=}") + + +class MyFt: + def __init__(self, tag, *args, **kwargs): + self.tag = tag + self.children = args + self.attrs = {safe_attr(k): v for k, v in kwargs.items()} + + def to_html(self): + body_items = [to_html(item) for item in self.children] + return f"<{self.tag} {' '.join(f'{k}="{v}"' for k, v in self.attrs.items())}>{' '.join(body_items)}" + + def __ft__(self): + return NotStr(self.to_html()) + + +class MyDiv(MyFt): + def __init__(self, *args, **kwargs): + super().__init__("div", *args, **kwargs) + + +class MySpan(MyFt): + def __init__(self, *args, **kwargs): + super().__init__("span", *args, **kwargs) + + +def mk_my_ellipsis(txt: str, cls='', **kwargs): + merged_cls = merge_classes("truncate", + cls, + kwargs) + return MyDiv(txt, cls=merged_cls, data_tooltip=txt, **kwargs) + + +def mk_my_icon(icon, size=20, can_select=True, can_hover=False, cls='', tooltip=None, **kwargs): + merged_cls = merge_classes(f"icon-{size}", + 'icon-btn' if can_select else '', + 'mmt-btn' if can_hover else '', + cls, + kwargs) + return mk_my_tooltip(icon, tooltip, cls=merged_cls, **kwargs) if tooltip else MyDiv(icon, cls=merged_cls, **kwargs) + + +def mk_my_tooltip(element, tooltip: str, cls='', **kwargs): + merged_cls = merge_classes("mmt-tooltip", + cls, + kwargs) + return MyDiv(element, cls=merged_cls, data_tooltip=tooltip, **kwargs) diff --git a/src/core/utils.py b/src/core/utils.py index f466a5b..0b93161 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,12 +1,16 @@ import ast import base64 +import cProfile +import functools import hashlib import importlib import inspect import pkgutil import re +import time import types import uuid +from datetime import datetime from enum import Enum from io import BytesIO from urllib.parse import urlparse @@ -420,6 +424,66 @@ def split_host_port(url): return host, port +def timed(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + start = time.perf_counter() + result = func(*args, **kwargs) + end = time.perf_counter() + + # get class name + class_name = None + if args: + # check the first argument to see if it's a class' + if inspect.isclass(args[0]): + class_name = args[0].__name__ # class method + elif hasattr(args[0], "__class__"): + class_name = args[0].__class__.__name__ # instance method + + if class_name: + print(f"[PERF] {class_name}.{func.__name__} took {end - start:.4f} sec") + else: + print(f"[PERF] {func.__name__} took {end - start:.4f} sec") + + return result + + return wrapper + + +def profile_function(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + profiler = cProfile.Profile() + try: + profiler.enable() + result = func(*args, **kwargs) + finally: + profiler.disable() + + # Determine class name if any + class_name = None + if args: + if inspect.isclass(args[0]): + class_name = args[0].__name__ # class method + elif hasattr(args[0], "__class__"): + class_name = args[0].__class__.__name__ # instance method + + # Compose filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + if class_name: + filename = f"{class_name}_{func.__name__}_{timestamp}.prof" + else: + filename = f"{func.__name__}_{timestamp}.prof" + + # Dump stats to file + profiler.dump_stats(filename) + print(f"[PROFILE] Profiling data saved to {filename}") + + return result + + return wrapper + + class UnreferencedNamesVisitor(ast.NodeVisitor): """ Try to find symbols that will be requested by the ast @@ -463,5 +527,4 @@ class UnreferencedNamesVisitor(ast.NodeVisitor): :rtype: """ self.names.add(node.arg) - self.visit_selected(node, ["value"]) - + self.visit_selected(node, ["value"]) \ No newline at end of file diff --git a/src/main.py b/src/main.py index f7df448..377f2a3 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,4 @@ # global layout -import asyncio import logging.config import yaml @@ -211,6 +210,25 @@ app, rt = fast_app( pico=False, ) + +# ------------------------- +# Profiling middleware +# ------------------------- +# @app.middleware("http") +async def timing_middleware(request, call_next): + import time + start_total = time.perf_counter() + + # Call the next middleware or route handler + response = await call_next(request) + + end_total = time.perf_counter() + elapsed = end_total - start_total + + print(f"[PERF] Total server time: {elapsed:.4f} sec - Path: {request.url.path}") + return response + + settings_manager = SettingsManager() import_settings = AdminImportSettings(settings_manager, None) @@ -253,6 +271,17 @@ def get(session): DrawerLayoutOld(pages),) +@rt('/toasting') +def get(session): + # Normally one toast is enough, this allows us to see + # different toast types in action. + add_toast(session, f"Toast is being cooked", "info") + add_toast(session, f"Toast is ready", "success") + add_toast(session, f"Toast is getting a bit crispy", "warning") + add_toast(session, f"Toast is burning!", "error") + return Titled("I like toast") + + # Error Handling @app.get("/{path:path}") def not_found(path: str, session=None): @@ -275,18 +304,7 @@ def not_found(path: str, session=None): setup_toasts(app) -@rt('/toasting') -def get(session): - # Normally one toast is enough, this allows us to see - # different toast types in action. - add_toast(session, f"Toast is being cooked", "info") - add_toast(session, f"Toast is ready", "success") - add_toast(session, f"Toast is getting a bit crispy", "warning") - add_toast(session, f"Toast is burning!", "error") - return Titled("I like toast") - - -async def main(): +def main(): logger.info(f" Starting FastHTML server on http://localhost:{APP_PORT}") serve(port=APP_PORT) @@ -294,9 +312,4 @@ async def main(): if __name__ == "__main__": # Start your application logger.info("Application starting...") - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("\nStopping application...") - except Exception as e: - logger.error(f"Error: {e}") + main() diff --git a/tests/helpers.py b/tests/helpers.py index 9bb5791..4bae796 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -642,10 +642,10 @@ def extract_table_values_new(ft, header=True): # first, get the header if header: - header = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0] + header_element = search_elements_by_name(ft, attrs={"class": "dt2-header"}, comparison_method='contains')[0] header_map = {} res = OrderedDict() - for row in header.children: + for row in header_element.children: col_id = row.attrs["data-col"] title = row.attrs["data-tooltip"] header_map[col_id] = title @@ -654,9 +654,10 @@ def extract_table_values_new(ft, header=True): body = search_elements_by_name(ft, attrs={"class": "dt2-body"}, comparison_method='contains')[0] for row in body.children: for col in row.children: - col_id = col.attrs["data-col"] - cell_value = _get_cell_content_value(col) - res[header_map[col_id]].append(cell_value) + if hasattr(col, "attrs"): + col_id = col.attrs["data-col"] + cell_value = _get_cell_content_value(col) + res[header_map[col_id]].append(cell_value) return res diff --git a/tests/test_datagrid_new.py b/tests/test_datagrid_new.py index 7747d4c..03e2469 100644 --- a/tests/test_datagrid_new.py +++ b/tests/test_datagrid_new.py @@ -509,3 +509,18 @@ def test_i_can_compute_footer_menu_position_when_not_enough_space(dg): ) assert matches(menu, expected) + + +def test_the_content_of_the_cell_is_escaped(empty_dg): + df = pd.DataFrame({ + 'value': ['
My Content
'], + 'value2': ['{My Content}'], + }) + my_dg = empty_dg.init_from_dataframe(df) + + actual = my_dg.__ft__() + table_content = extract_table_values_new(actual, header=True) + + assert table_content == OrderedDict({ + 'value': ['<div> My Content </div>'], + 'value2': ['{My Content}']})