Added lazy loading when showing grid to improve performance

This commit is contained in:
2025-08-23 22:26:14 +02:00
parent 8eca1da3ca
commit 33970c9c97
16 changed files with 567 additions and 110 deletions

View File

@@ -136,3 +136,10 @@ def post(session, _id: str, state: str, args: str = None):
logger.debug(f"Entering on_state_changed with args {_id=}, {state=}, {args=}")
instance = InstanceManager.get(session, _id)
return instance.manage_state_changed(state, args)
@rt(Routes.GetPage)
def get(session, _id: str, page_index: int):
logger.debug(f"Entering {Routes.GetPage} with args {_id=}, {page_index=}")
instance = InstanceManager.get(session, _id)
return instance.mk_body_content_page(page_index)

View File

@@ -1,6 +1,6 @@
function bindDatagrid(datagridId, allowColumnsReordering) {
bindScrollbars(datagridId);
makeResizable(datagridId)
manageScrollbars(datagridId, true);
makeResizable(datagridId);
}
function bindScrollbars(datagridId) {
@@ -21,7 +21,7 @@ function bindScrollbars(datagridId) {
const table = datagrid.querySelector(".dt2-table");
if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
console.error("Essential scrollbar or content elements are missing in the datagrid.");
console.error("Essential scrollbars or content elements are missing in the datagrid.");
return;
}
@@ -176,6 +176,224 @@ function bindScrollbars(datagridId) {
});
}
function manageScrollbars(datagridId, binding) {
console.debug("manageScrollbars on element " + datagridId + " with binding=" + binding);
const datagrid = document.getElementById(datagridId);
if (!datagrid) {
console.error(`Datagrid with id "${datagridId}" not found.`);
return;
}
const verticalScrollbar = datagrid.querySelector(".dt2-scrollbars-vertical");
const verticalWrapper = datagrid.querySelector(".dt2-scrollbars-vertical-wrapper");
const horizontalScrollbar = datagrid.querySelector(".dt2-scrollbars-horizontal");
const horizontalWrapper = datagrid.querySelector(".dt2-scrollbars-horizontal-wrapper");
const body = datagrid.querySelector(".dt2-body");
const table = datagrid.querySelector(".dt2-table");
if (!verticalScrollbar || !verticalWrapper || !horizontalScrollbar || !horizontalWrapper || !body || !table) {
console.error("Essential scrollbars or content elements are missing in the datagrid.");
return;
}
const computeScrollbarVisibility = () => {
// Determine if the content is clipped
const isVerticalRequired = body.scrollHeight > body.clientHeight;
const isHorizontalRequired = table.scrollWidth > table.clientWidth;
// Show or hide the scrollbar wrappers
requestAnimationFrame(() => {
verticalWrapper.style.display = isVerticalRequired ? "block" : "none";
horizontalWrapper.style.display = isHorizontalRequired ? "block" : "none";
});
};
const computeScrollbarSize = () => {
// Vertical scrollbar height
const visibleHeight = body.clientHeight;
const totalHeight = body.scrollHeight;
const wrapperHeight = verticalWrapper.offsetHeight;
let scrollbarHeight = 0;
if (totalHeight > 0) {
scrollbarHeight = (visibleHeight / totalHeight) * wrapperHeight;
}
// Horizontal scrollbar width
const visibleWidth = table.clientWidth;
const totalWidth = table.scrollWidth;
const wrapperWidth = horizontalWrapper.offsetWidth;
let scrollbarWidth = 0;
if (totalWidth > 0) {
scrollbarWidth = (visibleWidth / totalWidth) * wrapperWidth;
}
requestAnimationFrame(() => {
verticalScrollbar.style.height = `${scrollbarHeight}px`;
horizontalScrollbar.style.width = `${scrollbarWidth}px`;
});
};
const updateVerticalScrollbarForMouseWheel = () => {
const maxScrollTop = body.scrollHeight - body.clientHeight;
const wrapperHeight = verticalWrapper.offsetHeight;
if (maxScrollTop > 0) {
const scrollRatio = wrapperHeight / body.scrollHeight;
verticalScrollbar.style.top = `${body.scrollTop * scrollRatio}px`;
}
};
if (binding) {
// Clean up existing managers if they exist
if (datagrid._managers) {
// Remove drag events
if (datagrid._managers.dragManager) {
verticalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.verticalMouseDown);
horizontalScrollbar.removeEventListener("mousedown", datagrid._managers.dragManager.horizontalMouseDown);
document.removeEventListener("mousemove", datagrid._managers.dragManager.mouseMove);
document.removeEventListener("mouseup", datagrid._managers.dragManager.mouseUp);
}
// Remove wheel events
if (datagrid._managers.wheelManager) {
body.removeEventListener("wheel", datagrid._managers.wheelManager.handleWheelScrolling);
}
// Remove resize events
if (datagrid._managers.resizeManager) {
window.removeEventListener("resize", datagrid._managers.resizeManager.handleResize);
}
}
// Create managers
const dragManager = {
isDragging: false,
startY: 0,
startX: 0,
updateVerticalScrollbar: (deltaX, deltaY) => {
const wrapperHeight = verticalWrapper.offsetHeight;
const scrollbarHeight = verticalScrollbar.offsetHeight;
const maxScrollTop = body.scrollHeight - body.clientHeight;
const scrollRatio = maxScrollTop / (wrapperHeight - scrollbarHeight);
let newTop = parseFloat(verticalScrollbar.style.top || "0") + deltaY;
newTop = Math.max(0, Math.min(newTop, wrapperHeight - scrollbarHeight));
verticalScrollbar.style.top = `${newTop}px`;
body.scrollTop = newTop * scrollRatio;
},
updateHorizontalScrollbar: (deltaX, deltaY) => {
const wrapperWidth = horizontalWrapper.offsetWidth;
const scrollbarWidth = horizontalScrollbar.offsetWidth;
const maxScrollLeft = table.scrollWidth - table.clientWidth;
const scrollRatio = maxScrollLeft / (wrapperWidth - scrollbarWidth);
let newLeft = parseFloat(horizontalScrollbar.style.left || "0") + deltaX;
newLeft = Math.max(0, Math.min(newLeft, wrapperWidth - scrollbarWidth));
horizontalScrollbar.style.left = `${newLeft}px`;
table.scrollLeft = newLeft * scrollRatio;
},
verticalMouseDown: (e) => {
disableTooltip();
dragManager.isDragging = true;
dragManager.startY = e.clientY;
dragManager.startX = e.clientX;
document.body.style.userSelect = "none";
verticalScrollbar.classList.add("dt2-dragging");
},
horizontalMouseDown: (e) => {
disableTooltip();
dragManager.isDragging = true;
dragManager.startY = e.clientY;
dragManager.startX = e.clientX;
document.body.style.userSelect = "none";
horizontalScrollbar.classList.add("dt2-dragging");
},
mouseMove: (e) => {
if (dragManager.isDragging) {
const deltaY = e.clientY - dragManager.startY;
const deltaX = e.clientX - dragManager.startX;
// Determine which scrollbar is being dragged
if (verticalScrollbar.classList.contains("dt2-dragging")) {
dragManager.updateVerticalScrollbar(deltaX, deltaY);
} else if (horizontalScrollbar.classList.contains("dt2-dragging")) {
dragManager.updateHorizontalScrollbar(deltaX, deltaY);
}
// Reset start points for next update
dragManager.startY = e.clientY;
dragManager.startX = e.clientX;
}
},
mouseUp: () => {
dragManager.isDragging = false;
document.body.style.userSelect = "";
verticalScrollbar.classList.remove("dt2-dragging");
horizontalScrollbar.classList.remove("dt2-dragging");
enableTooltip();
}
};
const wheelManager = {
handleWheelScrolling: (event) => {
const deltaX = event.deltaX;
const deltaY = event.deltaY;
// Scroll the body and table content
body.scrollTop += deltaY; // Vertical scrolling
table.scrollLeft += deltaX; // Horizontal scrolling
// Update the vertical scrollbar position
updateVerticalScrollbarForMouseWheel();
// Prevent default behavior to fully manage the scroll
event.preventDefault();
}
};
const resizeManager = {
handleResize: () => {
computeScrollbarVisibility();
computeScrollbarSize();
updateVerticalScrollbarForMouseWheel();
}
};
// Store managers on datagrid for cleanup
datagrid._managers = {
dragManager,
wheelManager,
resizeManager
};
// Bind events
verticalScrollbar.addEventListener("mousedown", dragManager.verticalMouseDown);
horizontalScrollbar.addEventListener("mousedown", dragManager.horizontalMouseDown);
document.addEventListener("mousemove", dragManager.mouseMove);
document.addEventListener("mouseup", dragManager.mouseUp);
body.addEventListener("wheel", wheelManager.handleWheelScrolling, {passive: false});
window.addEventListener("resize", resizeManager.handleResize);
}
// Always execute computations
computeScrollbarVisibility();
computeScrollbarSize();
}
function makeResizable(datagridId) {
console.debug("makeResizable on element " + datagridId);
@@ -494,4 +712,5 @@ function onAfterSettle(datagridId, event) {
if (response.includes("hx-on::before-settle")) {
bindDatagrid(datagridId)
}
}
}

View File

@@ -1,4 +1,5 @@
import copy
import html
import logging
from io import BytesIO
from typing import Literal, Any
@@ -20,9 +21,10 @@ from components.datagrid_new.db_management import DataGridDbManager
from components.datagrid_new.settings import DataGridRowState, DataGridColumnState, \
DataGridFooterConf, DataGridState, DataGridSettings, DatagridView
from components_helpers import mk_icon, mk_ellipsis
from core.fasthtml_helper import MyDiv, mk_my_ellipsis, MySpan, mk_my_icon
from core.instance_manager import InstanceManager
from core.settings_management import SettingsManager
from core.utils import get_unique_id, make_safe_id
from core.utils import get_unique_id, make_safe_id, timed
logger = logging.getLogger("DataGrid")
@@ -59,6 +61,8 @@ class DataGrid(BaseComponent):
self._state: DataGridState = self._db.load_state()
self._settings: DataGridSettings = grid_settings or self._db.load_settings()
self._df: DataFrame | None = self._db.load_dataframe()
self._fast_access = self._init_fast_access(self._df)
self._total_rows = len(self._df) if self._df is not None else 0
# update boundaries if possible
self.set_boundaries(boundaries)
@@ -118,14 +122,23 @@ class DataGrid(BaseComponent):
else:
return ColumnType.Text # Default to Text if no match
def _init_columns(_df):
columns = [DataGridColumnState(make_safe_id(col_id),
col_index,
col_id,
_get_column_type(self._df[make_safe_id(col_id)].dtype))
for col_index, col_id in enumerate(_df.columns)]
if self._state.row_index:
columns.insert(0, DataGridColumnState(make_safe_id(ROW_INDEX_ID), -1, " ", ColumnType.RowIndex))
return columns
self._df = df.copy()
self._df.columns = self._df.columns.map(make_safe_id) # make sure column names are trimmed
self._state.rows = [DataGridRowState(row_id) for row_id in self._df.index]
self._state.columns = [DataGridColumnState(make_safe_id(col_id),
col_index,
col_id,
_get_column_type(self._df[make_safe_id(col_id)].dtype))
for col_index, col_id in enumerate(df.columns)]
self._state.columns = _init_columns(df) # use df not self._df to keep the original title
self._fast_access = self._init_fast_access(self._df)
self._total_rows = len(self._df) if self._df is not None else 0
if save_state:
self._db.save_all(None, self._state, self._df)
@@ -205,6 +218,7 @@ class DataGrid(BaseComponent):
self._state.columns = new_columns_states
self._fast_access = self._init_fast_access(self._df)
self._views.recompute_need_save()
self._db.save_all(self._settings, self._state, self._df if new_column else None)
@@ -439,7 +453,7 @@ class DataGrid(BaseComponent):
_mk_keyboard_management(),
Div(
self.mk_table_header(),
self.mk_table_body(),
self.mk_table_body_page(),
self.mk_table_footer(),
cls="dt2-inner-table"),
cls="dt2-table",
@@ -479,20 +493,18 @@ class DataGrid(BaseComponent):
id=f"th_{self._id}"
)
def mk_table_body(self):
df = self._get_filtered_df()
def mk_table_body_page(self):
"""
This function is used to update the table body when the vertical scrollbar reaches the end
A new page is added when requested
"""
max_height = self._compute_body_max_height()
return Div(
*[Div(
*[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
cls="dt2-row",
data_row=f"{row_index}",
id=f"tr_{self._id}-{row_index}",
) for row_index in df.index],
*self.mk_body_content_page(0),
cls="dt2-body",
style=f"max-height:{max_height}px;",
id=f"tb_{self._id}"
id=f"tb_{self._id}",
)
def mk_table_footer(self):
@@ -507,34 +519,55 @@ class DataGrid(BaseComponent):
id=f"tf_{self._id}"
)
def mk_body_content_page(self, page_index: int):
df = self._get_filtered_df()
start = page_index * DATAGRID_PAGE_SIZE
end = start + DATAGRID_PAGE_SIZE
if self._total_rows > end:
last_row = df.index[end - 1]
else:
last_row = None
rows = [Div(
*[self.mk_body_cell(col_pos, row_index, col_def) for col_pos, col_def in enumerate(self._state.columns)],
cls="dt2-row",
data_row=f"{row_index}",
id=f"tr_{self._id}-{row_index}",
**self.commands.get_page(page_index + 1) if row_index == last_row else {}
) for row_index in df.index[start:end]]
rows.append(Script(f"manageScrollbars('{self._id}', false);"), )
return rows
def mk_body_cell(self, col_pos, row_index, col_def: DataGridColumnState):
if not col_def.usable:
return None
if not col_def.visible:
return Div(cls="dt2-col-hidden")
return MyDiv(cls="dt2-col-hidden")
content = self.mk_body_cell_content(col_pos, row_index, col_def)
return Div(content,
data_col=col_def.col_id,
style=f"width:{col_def.width}px;",
cls="dt2-cell")
return MyDiv(content,
data_col=col_def.col_id,
style=f"width:{col_def.width}px;",
cls="dt2-cell")
def mk_body_cell_content(self, col_pos, row_index, col_def: DataGridColumnState):
def mk_bool(value):
return Div(mk_icon(icon_checked if value else icon_unchecked, can_select=False),
cls="dt2-cell-content-checkbox")
def mk_bool(_value):
return MyDiv(mk_my_icon(icon_checked if _value else icon_unchecked, can_select=False),
cls="dt2-cell-content-checkbox")
def mk_text(value):
return mk_ellipsis(value, cls="dt2-cell-content-text")
def mk_text(_value):
return mk_my_ellipsis(_value, cls="dt2-cell-content-text")
def mk_number(value):
return mk_ellipsis(value, cls="dt2-cell-content-number")
def mk_number(_value):
return mk_my_ellipsis(_value, cls="dt2-cell-content-number")
def process_cell_content(value):
value_str = str(value)
def process_cell_content(_value):
value_str = html.escape(str(_value))
if FILTER_INPUT_CID not in self._state.filtered or (
keyword := self._state.filtered[FILTER_INPUT_CID]) is None:
@@ -545,21 +578,22 @@ class DataGrid(BaseComponent):
return value_str
len_keyword = len(keyword)
res = [Span(value_str[:index])] if index > 0 else []
res += [Span(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
res += [Span(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
res = [MySpan(value_str[:index])] if index > 0 else []
res += [MySpan(value_str[index:index + len_keyword], cls="dt2-highlight-1")]
res += [MySpan(value_str[index + len_keyword:])] if len(value_str) > len_keyword else []
return tuple(res)
column_type = col_def.type
value = self._fast_access[col_def.col_id][row_index]
if column_type == ColumnType.Bool:
content = mk_bool(self._df.iloc[row_index, col_def.col_index])
content = mk_bool(value)
elif column_type == ColumnType.Number:
content = mk_number(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
content = mk_number(process_cell_content(value))
elif column_type == ColumnType.RowIndex:
content = mk_number(row_index)
else:
content = mk_text(process_cell_content(self._df.iloc[row_index, col_def.col_index]))
content = mk_text(process_cell_content(value))
return content
@@ -822,6 +856,31 @@ class DataGrid(BaseComponent):
return True
@staticmethod
def _init_fast_access(df):
"""
Generates a fast-access dictionary for a DataFrame.
This method converts the columns of the provided DataFrame into NumPy arrays
and stores them as values in a dictionary, using the column names as keys.
This allows for efficient access to the data stored in the DataFrame.
Args:
df (DataFrame): The input pandas DataFrame whose columns are to be converted
into a dictionary of NumPy arrays.
Returns:
dict: A dictionary where the keys are the column names of the input DataFrame
and the values are the corresponding column values as NumPy arrays.
"""
if df is None:
return {}
res = {col: df[col].to_numpy() for col in df.columns}
res[ROW_INDEX_ID] = df.index.to_numpy()
return res
@timed
def __ft__(self):
return Div(
Div(
@@ -844,7 +903,7 @@ class DataGrid(BaseComponent):
@staticmethod
def new(session, data, index=None):
datagrid = DataGrid(session, DataGrid.create_component_id(session))
#dataframe = DataFrame(data, index=index)
# dataframe = DataFrame(data, index=index)
dataframe = DataFrame(data)
datagrid.init_from_dataframe(dataframe)
return datagrid

View File

@@ -91,12 +91,21 @@ class DataGridCommandManager(BaseCommandManager):
return {
"hx-post": f"{ROUTE_ROOT}{Routes.OnClick}",
"hx-target": f"#tsm_{self._id}",
"hx-trigger" : "click",
"hx-trigger": "click",
"hx-swap": "outerHTML",
"hx-vals": f'js:{{_id: "{self._id}", cell_id:getCellId(event), modifier:getClickModifier(event), boundaries: getCellBoundaries(event)}}',
"hx-on::before-request": f'validateOnClickRequest("{self._id}", event)',
}
def get_page(self, page_index=0):
return {
"hx-get": f"{ROUTE_ROOT}{Routes.GetPage}",
"hx-target": f"#tb_{self._id}",
"hx-swap": "beforeend",
"hx-vals": f'{{"_id": "{self._id}", "page_index": "{page_index}"}}',
"hx-trigger": f"intersect root:#tb_{self._id} once",
}
def _get_hide_show_columns_attrs(self, mode, col_defs: list, new_value, cls=""):
str_col_names = ", ".join(f"'{col_def.title}'" for col_def in col_defs)
tooltip_msg = f"{mode} column{'s' if len(col_defs) > 1 else ''} {str_col_names}"
@@ -109,38 +118,6 @@ class DataGridCommandManager(BaseCommandManager):
"data_tooltip": tooltip_msg,
"cls": self.merge_class(cls, "mmt-tooltip")
}
#
# @staticmethod
# def merge(*items):
# """
# Merges multiple dictionaries into a single dictionary by combining their key-value pairs.
# If a key exists in multiple dictionaries and its value is a string, the values are concatenated.
# If the key's value is not a string, an error is raised.
#
# :param items: dictionaries to be merged. If all items are None, None is returned.
# :return: A single dictionary containing the merged key-value pairs from all input dictionaries.
# :raises NotImplementedError: If a key's value is not a string and exists in multiple input dictionaries.
# """
# if all(item is None for item in items):
# return None
#
# res = {}
# for item in [item for item in items if item is not None]:
#
# for key, value in item.items():
# if not key in res:
# res[key] = value
# else:
# if isinstance(res[key], str):
# res[key] += " " + value
# else:
# raise NotImplementedError("")
#
# return res
#
# @staticmethod
# def merge_class(cls1, cls2):
# return (cls1 + " " + cls2) if cls2 else cls1
class FilterAllCommands(BaseCommandManager):
@@ -165,4 +142,4 @@ class FilterAllCommands(BaseCommandManager):
"hx_vals": f'{{"_id": "{self._id}", "col_id":"{FILTER_INPUT_CID}"}}',
"data_tooltip": "Reset filter",
"cls": self.merge_class(cls, "mmt-tooltip"),
}
}

View File

@@ -17,6 +17,9 @@ CONTAINER_HEIGHT = "container_height"
DATAGRID_STATE_FOOTER = "footer"
DATAGRID_PAGE_SIZE = 50
ROW_INDEX_ID = "__row_index__"
class Routes:
Filter = "/filter" # request the filtering in the grid
@@ -33,6 +36,7 @@ class Routes:
UpdateView = "/update_view"
ShowFooterMenu = "/show_footer_menu"
UpdateState = "/update_state"
GetPage = "/page"
class ColumnType(Enum):
@@ -44,11 +48,13 @@ class ColumnType(Enum):
Choice = "Choice"
List = "List"
class ViewType(Enum):
Table = "Table"
Chart = "Chart"
Form = "Form"
class FooterAggregation(Enum):
Sum = "Sum"
Mean = "Mean"
@@ -59,4 +65,4 @@ class FooterAggregation(Enum):
FilteredMean = "FilteredMean"
FilteredMin = "FilteredMin"
FilteredMax = "FilteredMax"
FilteredCount = "FilteredCount"
FilteredCount = "FilteredCount"

View File

@@ -69,6 +69,7 @@ class DataGridSettings:
class DataGridState:
sidebar_visible: bool = False
selected_view: str = None
row_index: bool = False
columns: list[DataGridColumnState] = dataclasses.field(default_factory=list)
rows: list[DataGridRowState] = dataclasses.field(default_factory=list) # only the rows that have a specific state
footers: list[DataGridFooterConf] = dataclasses.field(default_factory=list)

View File

@@ -20,7 +20,7 @@ def get(session):
@rt(Routes.AddRepository)
def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries:str):
def post(session, _id: str, tab_id: str, form_id: str, repository: str, table: str, tab_boundaries: str):
logger.debug(
f"Entering {Routes.AddRepository} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository=}, {table=}, {tab_boundaries=}")
instance = InstanceManager.get(session, _id) # Repository
@@ -34,8 +34,9 @@ def get(session, _id: str, repository_name: str):
@rt(Routes.AddTable)
def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries:str):
logger.debug(f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
def post(session, _id: str, tab_id: str, form_id: str, repository_name: str, table_name: str, tab_boundaries: str):
logger.debug(
f"Entering {Routes.AddTable} with args {debug_session(session)}, {_id=}, {tab_id=}, {form_id=}, {repository_name=}, {table_name=}, {tab_boundaries=}")
instance = InstanceManager.get(session, _id)
return instance.add_new_table(tab_id, form_id, repository_name, table_name, json.loads(tab_boundaries))
@@ -48,7 +49,8 @@ def put(session, _id: str, repository: str):
@rt(Routes.ShowTable)
def get(session, _id: str, repository: str, table: str, tab_boundaries:str):
logger.debug(f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
def get(session, _id: str, repository: str, table: str, tab_boundaries: str):
logger.debug(
f"Entering {Routes.ShowTable} with args {debug_session(session)}, {_id=}, {repository=}, {table=}, {tab_boundaries=}")
instance = InstanceManager.get(session, _id)
return instance.show_table(repository, table, json.loads(tab_boundaries))