Working on Formating DSL completion

tests/core/dsl/test_base_completion.py

@@ -0,0 +1,137 @@
+"""
+Tests for BaseCompletionEngine.
+
+Uses a mock implementation to test the abstract base class functionality.
+"""
+
+import pytest
+from typing import Any
+
+from myfasthtml.core.dsl.types import Position, Suggestion, CompletionResult
+from myfasthtml.core.dsl.base_completion import BaseCompletionEngine
+from myfasthtml.core.dsl.base_provider import BaseMetadataProvider
+
+
+class MockProvider:
+    """Mock metadata provider for testing."""
+
+    def get_style_presets(self) -> list[str]:
+        return ["custom_highlight"]
+
+    def get_format_presets(self) -> list[str]:
+        return ["CHF"]
+
+
+class MockCompletionEngine(BaseCompletionEngine):
+    """Mock completion engine for testing base class functionality."""
+
+    def __init__(self, provider: BaseMetadataProvider, suggestions: list[Suggestion] = None):
+        super().__init__(provider)
+        self._suggestions = suggestions or []
+        self._scope = None
+        self._context = "test_context"
+
+    def detect_scope(self, text: str, current_line: int) -> Any:
+        return self._scope
+
+    def detect_context(self, text: str, cursor: Position, scope: Any) -> Any:
+        return self._context
+
+    def get_suggestions(self, context: Any, scope: Any, prefix: str) -> list[Suggestion]:
+        return self._suggestions
+
+
+# =============================================================================
+# Filter Suggestions Tests
+# =============================================================================
+
+
+def test_i_can_filter_suggestions_by_prefix():
+    """Test that _filter_suggestions filters case-insensitively."""
+    provider = MockProvider()
+    engine = MockCompletionEngine(provider)
+
+    suggestions = [
+        Suggestion("primary", "Primary color", "preset"),
+        Suggestion("error", "Error color", "preset"),
+        Suggestion("warning", "Warning color", "preset"),
+        Suggestion("Error", "Title case error", "preset"),
+    ]
+
+    # Filter by "err" - should match "error" and "Error" (case-insensitive)
+    filtered = engine._filter_suggestions(suggestions, "err")
+    labels = [s.label for s in filtered]
+    assert "error" in labels
+    assert "Error" in labels
+    assert "primary" not in labels
+    assert "warning" not in labels
+
+
+def test_i_can_filter_suggestions_empty_prefix():
+    """Test that empty prefix returns all suggestions."""
+    provider = MockProvider()
+    engine = MockCompletionEngine(provider)
+
+    suggestions = [
+        Suggestion("a"),
+        Suggestion("b"),
+        Suggestion("c"),
+    ]
+
+    filtered = engine._filter_suggestions(suggestions, "")
+    assert len(filtered) == 3
+
+
+# =============================================================================
+# Empty Result Tests
+# =============================================================================
+
+
+def test_i_can_get_empty_result():
+    """Test that _empty_result returns a CompletionResult with no suggestions."""
+    provider = MockProvider()
+    engine = MockCompletionEngine(provider)
+
+    cursor = Position(line=5, ch=10)
+    result = engine._empty_result(cursor)
+
+    assert result.from_pos == cursor
+    assert result.to_pos == cursor
+    assert result.suggestions == []
+    assert result.is_empty is True
+
+
+# =============================================================================
+# Comment Skipping Tests
+# =============================================================================
+
+
+def test_i_can_skip_completion_in_comment():
+    """Test that get_completions returns empty when cursor is in a comment."""
+    provider = MockProvider()
+    suggestions = [Suggestion("should_not_appear")]
+    engine = MockCompletionEngine(provider, suggestions)
+
+    text = "# This is a comment"
+    cursor = Position(line=0, ch=15)  # Inside the comment
+
+    result = engine.get_completions(text, cursor)
+
+    assert result.is_empty is True
+    assert len(result.suggestions) == 0
+
+
+def test_i_can_get_completions_outside_comment():
+    """Test that get_completions works when cursor is not in a comment."""
+    provider = MockProvider()
+    suggestions = [Suggestion("style"), Suggestion("format")]
+    engine = MockCompletionEngine(provider, suggestions)
+
+    # Cursor at space (ch=5) so prefix is empty and all suggestions are returned
+    text = "text  # comment"
+    cursor = Position(line=0, ch=5)  # At empty space, before comment
+
+    result = engine.get_completions(text, cursor)
+
+    assert result.is_empty is False
+    assert len(result.suggestions) == 2

tests/core/dsl/test_lark_to_lezer.py

@@ -0,0 +1,172 @@
+"""Tests for lark_to_lezer module."""
+
+import pytest
+
+from myfasthtml.core.dsl.lark_to_lezer import (
+  extract_completions_from_grammar,
+  lark_to_lezer_grammar,
+)
+
+# Sample grammars for testing
+SIMPLE_GRAMMAR = r'''
+    start: rule+
+    rule: "if" condition
+    condition: "value" operator literal
+    operator: "==" -> op_eq
+           | "!=" -> op_ne
+           | "contains" -> op_contains
+    literal: QUOTED_STRING -> string_literal
+          | BOOLEAN -> boolean_literal
+    QUOTED_STRING: /"[^"]*"/
+    BOOLEAN: "True" | "False"
+'''
+
+GRAMMAR_WITH_KEYWORDS = r'''
+    start: scope+
+    scope: "column" NAME ":" rule
+        | "row" INTEGER ":" rule
+        | "cell" cell_ref ":" rule
+    rule: style_expr condition?
+    condition: "if" "not"? comparison
+    comparison: operand "and" operand
+             | operand "or" operand
+    style_expr: "style" "(" args ")"
+    operand: "value" | literal
+'''
+
+GRAMMAR_WITH_TYPES = r'''
+    format_type: "number" -> fmt_number
+              | "date" -> fmt_date
+              | "boolean" -> fmt_boolean
+              | "text" -> fmt_text
+              | "enum" -> fmt_enum
+'''
+
+
+class TestExtractCompletions:
+  """Tests for extract_completions_from_grammar function."""
+  
+  def test_i_can_extract_keywords_from_grammar(self):
+    """Test that keywords like if, not, and are extracted."""
+    completions = extract_completions_from_grammar(GRAMMAR_WITH_KEYWORDS)
+    
+    assert "if" in completions["keywords"]
+    assert "not" in completions["keywords"]
+    assert "column" in completions["keywords"]
+    assert "row" in completions["keywords"]
+    assert "cell" in completions["keywords"]
+    assert "value" in completions["keywords"]
+  
+  @pytest.mark.parametrize(
+    "operator",
+    ["==", "!=", "contains"],
+  )
+  def test_i_can_extract_operators_from_grammar(self, operator):
+    """Test that operators are extracted from grammar."""
+    completions = extract_completions_from_grammar(SIMPLE_GRAMMAR)
+    
+    assert operator in completions["operators"]
+  
+  def test_i_can_extract_functions_from_grammar(self):
+    """Test that function-like constructs are extracted."""
+    completions = extract_completions_from_grammar(GRAMMAR_WITH_KEYWORDS)
+    
+    assert "style" in completions["functions"]
+  
+  @pytest.mark.parametrize(
+    "type_name",
+    ["number", "date", "boolean", "text", "enum"],
+  )
+  def test_i_can_extract_types_from_grammar(self, type_name):
+    """Test that type names are extracted from format_type rule."""
+    completions = extract_completions_from_grammar(GRAMMAR_WITH_TYPES)
+    
+    assert type_name in completions["types"]
+  
+  @pytest.mark.parametrize("literal", [
+      "True",
+      "False"
+  ])
+  def test_i_can_extract_literals_from_grammar(self, literal):
+    """Test that literal values like True/False are extracted."""
+    completions = extract_completions_from_grammar(SIMPLE_GRAMMAR)
+    
+    assert literal in completions["literals"]
+  
+  def test_i_can_extract_completions_returns_all_categories(self):
+    """Test that all completion categories are present in result."""
+    completions = extract_completions_from_grammar(SIMPLE_GRAMMAR)
+    
+    assert "keywords" in completions
+    assert "operators" in completions
+    assert "functions" in completions
+    assert "types" in completions
+    assert "literals" in completions
+  
+  def test_i_can_extract_completions_returns_sorted_lists(self):
+    """Test that completion lists are sorted alphabetically."""
+    completions = extract_completions_from_grammar(SIMPLE_GRAMMAR)
+    
+    for category in completions.values():
+      assert category == sorted(category)
+
+
+class TestLarkToLezerConversion:
+  """Tests for lark_to_lezer_grammar function."""
+  
+  def test_i_can_convert_simple_grammar_to_lezer(self):
+    """Test that a simple Lark grammar is converted to Lezer format."""
+    lezer = lark_to_lezer_grammar(SIMPLE_GRAMMAR)
+    
+    # Should have @top directive
+    assert "@top Start" in lezer
+    # Should have @tokens block
+    assert "@tokens {" in lezer
+    # Should have @skip directive
+    assert "@skip {" in lezer
+  
+  def test_i_can_convert_rule_names_to_pascal_case(self):
+    """Test that snake_case rule names become PascalCase."""
+    grammar = r'''
+            my_rule: other_rule
+            other_rule: "test"
+        '''
+    lezer = lark_to_lezer_grammar(grammar)
+    
+    assert "MyRule" in lezer
+    assert "OtherRule" in lezer
+  
+  def test_i_cannot_include_internal_rules_in_lezer(self):
+    """Test that rules starting with _ are not included."""
+    grammar = r'''
+            start: rule _NL
+            rule: "test"
+            _NL: /\n/
+        '''
+    lezer = lark_to_lezer_grammar(grammar)
+    
+    # Internal rules should not appear as Lezer rules
+    assert "Nl {" not in lezer
+  
+  def test_i_can_convert_terminal_regex_to_lezer(self):
+    """Test that terminal regex patterns are converted."""
+    grammar = r'''
+            NAME: /[a-zA-Z_][a-zA-Z0-9_]*/
+        '''
+    lezer = lark_to_lezer_grammar(grammar)
+    
+    assert "NAME" in lezer
+  
+  @pytest.mark.parametrize(
+    "terminal,pattern",
+    [
+        ('BOOLEAN: "True" | "False"', "BOOLEAN"),
+        ('KEYWORD: "if"', "KEYWORD"),
+    ],
+  )
+  def test_i_can_convert_terminal_strings_to_lezer(self, terminal, pattern):
+    """Test that terminal string literals are converted."""
+    grammar = f"start: test\n{terminal}"
+    lezer = lark_to_lezer_grammar(grammar)
+    
+    assert pattern in lezer

tests/core/dsl/test_types.py

@@ -0,0 +1,145 @@
+"""
+Tests for DSL autocompletion types.
+
+Tests for Position, Suggestion, CompletionResult, and WordRange dataclasses.
+"""
+
+import pytest
+from myfasthtml.core.dsl.types import Position, Suggestion, CompletionResult, WordRange
+
+
+# =============================================================================
+# Position Tests
+# =============================================================================
+
+
+def test_i_can_create_position():
+    """Test that a Position can be created with line and ch."""
+    pos = Position(line=0, ch=5)
+    assert pos.line == 0
+    assert pos.ch == 5
+
+
+def test_i_can_convert_position_to_dict():
+    """Test that Position.to_dict() returns correct CodeMirror format."""
+    pos = Position(line=3, ch=12)
+    result = pos.to_dict()
+    assert result == {"line": 3, "ch": 12}
+
+
+# =============================================================================
+# Suggestion Tests
+# =============================================================================
+
+
+def test_i_can_create_suggestion_with_label_only():
+    """Test that a Suggestion can be created with just a label."""
+    suggestion = Suggestion("text")
+    assert suggestion.label == "text"
+    assert suggestion.detail == ""
+    assert suggestion.kind == ""
+
+
+def test_i_can_create_suggestion_with_all_fields():
+    """Test that a Suggestion can be created with label, detail, and kind."""
+    suggestion = Suggestion(label="primary", detail="Primary theme color", kind="preset")
+    assert suggestion.label == "primary"
+    assert suggestion.detail == "Primary theme color"
+    assert suggestion.kind == "preset"
+
+
+def test_i_can_convert_suggestion_to_dict_with_label_only():
+    """Test that Suggestion.to_dict() works with label only."""
+    suggestion = Suggestion("text")
+    result = suggestion.to_dict()
+    assert result == {"label": "text"}
+    assert "detail" not in result
+    assert "kind" not in result
+
+
+def test_i_can_convert_suggestion_to_dict_with_all_fields():
+    """Test that Suggestion.to_dict() includes detail and kind when present."""
+    suggestion = Suggestion(label="error", detail="Error style", kind="preset")
+    result = suggestion.to_dict()
+    assert result == {"label": "error", "detail": "Error style", "kind": "preset"}
+
+
+def test_i_can_convert_suggestion_to_dict_with_partial_fields():
+    """Test that Suggestion.to_dict() includes only non-empty fields."""
+    suggestion = Suggestion(label="text", detail="Description")
+    result = suggestion.to_dict()
+    assert result == {"label": "text", "detail": "Description"}
+    assert "kind" not in result
+
+
+# =============================================================================
+# CompletionResult Tests
+# =============================================================================
+
+
+def test_i_can_create_completion_result():
+    """Test that a CompletionResult can be created with positions and suggestions."""
+    from_pos = Position(line=0, ch=5)
+    to_pos = Position(line=0, ch=10)
+    suggestions = [Suggestion("option1"), Suggestion("option2")]
+
+    result = CompletionResult(from_pos=from_pos, to_pos=to_pos, suggestions=suggestions)
+
+    assert result.from_pos == from_pos
+    assert result.to_pos == to_pos
+    assert len(result.suggestions) == 2
+
+
+def test_i_can_convert_completion_result_to_dict():
+    """Test that CompletionResult.to_dict() returns CodeMirror-compatible format."""
+    from_pos = Position(line=1, ch=4)
+    to_pos = Position(line=1, ch=8)
+    suggestions = [Suggestion("primary", "Primary color", "preset")]
+
+    result = CompletionResult(from_pos=from_pos, to_pos=to_pos, suggestions=suggestions)
+    dict_result = result.to_dict()
+
+    assert dict_result == {
+        "from": {"line": 1, "ch": 4},
+        "to": {"line": 1, "ch": 8},
+        "suggestions": [{"label": "primary", "detail": "Primary color", "kind": "preset"}],
+    }
+
+
+def test_completion_result_is_empty_when_no_suggestions():
+    """Test that is_empty returns True when there are no suggestions."""
+    result = CompletionResult(
+        from_pos=Position(line=0, ch=0),
+        to_pos=Position(line=0, ch=0),
+        suggestions=[],
+    )
+    assert result.is_empty is True
+
+
+def test_completion_result_is_not_empty_when_has_suggestions():
+    """Test that is_empty returns False when there are suggestions."""
+    result = CompletionResult(
+        from_pos=Position(line=0, ch=0),
+        to_pos=Position(line=0, ch=5),
+        suggestions=[Suggestion("text")],
+    )
+    assert result.is_empty is False
+
+
+# =============================================================================
+# WordRange Tests
+# =============================================================================
+
+
+def test_i_can_create_word_range():
+    """Test that a WordRange can be created with start, end, and text."""
+    word_range = WordRange(start=5, end=10, text="hello")
+    assert word_range.start == 5
+    assert word_range.end == 10
+    assert word_range.text == "hello"
+
+
+def test_i_can_create_word_range_with_default_text():
+    """Test that a WordRange has default empty text."""
+    word_range = WordRange(start=0, end=0)
+    assert word_range.text == ""

tests/core/dsl/test_utils.py

@@ -0,0 +1,261 @@
+"""
+Tests for DSL autocompletion utilities.
+
+Tests for line extraction, word boundaries, comment/string detection,
+and indentation functions.
+"""
+
+import pytest
+from myfasthtml.core.dsl.types import Position, WordRange
+from myfasthtml.core.dsl.utils import (
+    get_line_at,
+    get_line_up_to_cursor,
+    get_lines_up_to,
+    find_word_boundaries,
+    get_prefix,
+    is_in_comment,
+    is_in_string,
+    get_indentation,
+    is_indented,
+    strip_quotes,
+)
+
+
+# =============================================================================
+# Line Extraction Tests
+# =============================================================================
+
+
+def test_i_can_get_line_at_valid_index():
+    """Test that get_line_at returns the correct line."""
+    text = "line0\nline1\nline2"
+    assert get_line_at(text, 0) == "line0"
+    assert get_line_at(text, 1) == "line1"
+    assert get_line_at(text, 2) == "line2"
+
+
+def test_i_can_get_line_at_invalid_index():
+    """Test that get_line_at returns empty string for invalid index."""
+    text = "line0\nline1"
+    assert get_line_at(text, -1) == ""
+    assert get_line_at(text, 5) == ""
+
+
+def test_i_can_get_line_up_to_cursor():
+    """Test that get_line_up_to_cursor truncates at cursor position."""
+    text = "hello world\nfoo bar"
+    cursor = Position(line=0, ch=5)
+    assert get_line_up_to_cursor(text, cursor) == "hello"
+
+    cursor = Position(line=1, ch=3)
+    assert get_line_up_to_cursor(text, cursor) == "foo"
+
+
+def test_i_can_get_lines_up_to():
+    """Test that get_lines_up_to returns lines 0..N."""
+    text = "line0\nline1\nline2\nline3"
+    assert get_lines_up_to(text, 0) == ["line0"]
+    assert get_lines_up_to(text, 2) == ["line0", "line1", "line2"]
+
+
+# =============================================================================
+# Word Boundaries Tests
+# =============================================================================
+
+
+def test_i_can_find_word_boundaries_in_middle():
+    """Test word boundaries when cursor is in middle of word."""
+    line = "hello world"
+    result = find_word_boundaries(line, 3)  # hel|lo
+    assert result.start == 0
+    assert result.end == 5
+    assert result.text == "hello"
+
+
+def test_i_can_find_word_boundaries_at_start():
+    """Test word boundaries when cursor is at start of word."""
+    line = "hello world"
+    result = find_word_boundaries(line, 0)  # |hello
+    assert result.start == 0
+    assert result.end == 5
+    assert result.text == "hello"
+
+
+def test_i_can_find_word_boundaries_at_end():
+    """Test word boundaries when cursor is at end of word."""
+    line = "hello world"
+    result = find_word_boundaries(line, 5)  # hello|
+    assert result.start == 0
+    assert result.end == 5
+    assert result.text == "hello"
+
+
+def test_i_can_find_word_boundaries_with_delimiters():
+    """Test word boundaries with delimiter characters like parentheses and quotes."""
+    line = 'style("error")'
+    result = find_word_boundaries(line, 10)  # style("err|or")
+    assert result.start == 7
+    assert result.end == 12
+    assert result.text == "error"
+
+
+def test_i_can_find_word_boundaries_empty_line():
+    """Test word boundaries on empty line."""
+    line = ""
+    result = find_word_boundaries(line, 0)
+    assert result.start == 0
+    assert result.end == 0
+    assert result.text == ""
+
+
+def test_i_can_get_prefix():
+    """Test that get_prefix returns text from word start to cursor."""
+    line = "style"
+    prefix = get_prefix(line, 3)  # sty|le
+    assert prefix == "sty"
+
+
+def test_i_can_get_prefix_at_word_start():
+    """Test that get_prefix returns empty at word start."""
+    line = "style"
+    prefix = get_prefix(line, 0)  # |style
+    assert prefix == ""
+
+
+# =============================================================================
+# Comment Detection Tests
+# =============================================================================
+
+
+def test_i_can_detect_comment():
+    """Test that cursor after # is detected as in comment."""
+    line = "text # comment"
+    assert is_in_comment(line, 12) is True  # In "comment"
+    assert is_in_comment(line, 7) is True  # Right after #
+
+
+def test_i_cannot_detect_comment_before_hash():
+    """Test that cursor before # is not detected as in comment."""
+    line = "text # comment"
+    assert is_in_comment(line, 4) is False  # Before #
+    assert is_in_comment(line, 0) is False  # At start
+
+
+def test_i_cannot_detect_comment_hash_in_string():
+    """Test that # inside a string is not detected as comment start."""
+    line = '"#hash" text'
+    assert is_in_comment(line, 9) is False  # After the string
+
+
+def test_i_can_detect_comment_hash_after_string():
+    """Test that # after a string is detected as comment."""
+    line = '"text" # comment'
+    assert is_in_comment(line, 10) is True
+
+
+# =============================================================================
+# String Detection Tests
+# =============================================================================
+
+
+def test_i_can_detect_string_double_quote():
+    """Test detection of cursor inside double-quoted string."""
+    line = 'style("error")'
+    in_string, quote_char = is_in_string(line, 10)  # Inside "error"
+    assert in_string is True
+    assert quote_char == '"'
+
+
+def test_i_can_detect_string_single_quote():
+    """Test detection of cursor inside single-quoted string."""
+    line = "style('error')"
+    in_string, quote_char = is_in_string(line, 10)  # Inside 'error'
+    assert in_string is True
+    assert quote_char == "'"
+
+
+def test_i_cannot_detect_string_outside_quotes():
+    """Test that cursor outside quotes is not detected as in string."""
+    line = 'style("error")'
+    in_string, quote_char = is_in_string(line, 3)  # In "style"
+    assert in_string is False
+    assert quote_char is None
+
+
+def test_i_cannot_detect_string_after_closing_quote():
+    """Test that cursor after closing quote is not in string."""
+    line = '"text" other'
+    in_string, quote_char = is_in_string(line, 8)
+    assert in_string is False
+    assert quote_char is None
+
+
+# =============================================================================
+# Indentation Tests
+# =============================================================================
+
+
+def test_i_can_get_indentation_spaces():
+    """Test that spaces are counted correctly."""
+    line = "    text"
+    assert get_indentation(line) == 4
+
+
+def test_i_can_get_indentation_tabs():
+    """Test that tabs are converted to 4 spaces."""
+    line = "\ttext"
+    assert get_indentation(line) == 4
+
+
+def test_i_can_get_indentation_mixed():
+    """Test mixed spaces and tabs."""
+    line = "  \t  text"  # 2 spaces + tab (4) + 2 spaces = 8
+    assert get_indentation(line) == 8
+
+
+def test_i_can_detect_indented_line():
+    """Test that indented line is detected."""
+    assert is_indented("  text") is True
+    assert is_indented("\ttext") is True
+
+
+def test_i_cannot_detect_indented_for_non_indented():
+    """Test that non-indented line is not detected as indented."""
+    assert is_indented("text") is False
+
+
+def test_i_cannot_detect_indented_for_empty_line():
+    """Test that empty line is not detected as indented."""
+    assert is_indented("") is False
+
+
+# =============================================================================
+# Quote Stripping Tests
+# =============================================================================
+
+
+def test_i_can_strip_quotes_double():
+    """Test stripping double quotes."""
+    assert strip_quotes('"text"') == "text"
+
+
+def test_i_can_strip_quotes_single():
+    """Test stripping single quotes."""
+    assert strip_quotes("'text'") == "text"
+
+
+def test_i_cannot_strip_quotes_unquoted():
+    """Test that unquoted text is returned unchanged."""
+    assert strip_quotes("text") == "text"
+
+
+def test_i_cannot_strip_quotes_mismatched():
+    """Test that mismatched quotes are not stripped."""
+    assert strip_quotes('"text\'') == '"text\''
+    assert strip_quotes("'text\"") == "'text\""
+
+
+def test_i_cannot_strip_quotes_too_short():
+    """Test that text shorter than 2 chars is returned unchanged."""
+    assert strip_quotes('"') == '"'
+    assert strip_quotes("") == ""