I can finally chain Processor calls

tests/test_preprocessor.py

@@ -0,0 +1,491 @@
+import pytest
+
+from core.preprocessor import PlainTextPreprocessor, VariableParsingError, VariableProcessingError
+
+
+def test_i_can_parse_empty_text():
+  """Test that I can parse empty text input"""
+  processor = PlainTextPreprocessor()
+  result = processor.parse("")
+  assert result == []
+
+
+def test_i_can_parse_text_without_variables():
+  """Test that I can parse text without any variables"""
+  processor = PlainTextPreprocessor()
+  text = "This is just plain text with no variables"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "text",
+      "content": text,
+      "start": 0,
+      "end": len(text)
+  }]
+  
+  assert result == expected
+
+
+def test_i_can_parse_simple_variable():
+  """Test that I can parse text with only a simple variable"""
+  processor = PlainTextPreprocessor()
+  text = "$variable"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "variable",
+      "properties": [],
+      "start": 0,
+      "end": 9
+  }]
+  
+  assert result == expected
+
+
+def test_i_can_parse_variable_with_underscores():
+  """Test that I can parse variable with underscores in name"""
+  processor = PlainTextPreprocessor()
+  text = "$my_variable_name"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "my_variable_name",
+      "properties": [],
+      "start": 0,
+      "end": 17
+  }]
+  
+  assert result == expected
+
+
+def test_i_can_parse_variable_with_numbers():
+  """Test that I can parse variable with numbers in name"""
+  processor = PlainTextPreprocessor()
+  text = "$var123"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "var123",
+      "properties": [],
+      "start": 0,
+      "end": 7
+  }]
+  assert result == expected
+
+
+def test_i_can_parse_properties_with_underscores_and_numbers():
+  """Test that I can parse property names with underscores and numbers"""
+  processor = PlainTextPreprocessor()
+  text = "$var._prop123.sub_prop_456"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "var",
+      "properties": ["_prop123", "sub_prop_456"],
+      "start": 0,
+      "end": 26
+  }]
+  
+  assert result == expected
+
+
+def test_i_can_parse_variable_starting_with_underscore():
+  """Test that I can parse variable name starting with underscore"""
+  processor = PlainTextPreprocessor()
+  text = "$_private_var"
+  result = processor.parse(text)
+  
+  expected = [
+      {
+          "type": "variable",
+          "name": "_private_var",
+          "properties": [],
+          "start": 0,
+          "end": 13
+      }
+  ]
+  assert result == expected
+
+
+def test_i_can_parse_variable_with_single_property():
+  """Test that I can parse variable with one property"""
+  processor = PlainTextPreprocessor()
+  text = "$variable.prop"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "variable",
+      "properties": ["prop"],
+      "start": 0,
+      "end": 14
+  }]
+  assert result == expected
+
+
+def test_i_can_parse_variable_with_multiple_properties():
+  """Test that I can parse variable with multiple properties"""
+  processor = PlainTextPreprocessor()
+  text = "$variable.prop.subprop.deep"
+  result = processor.parse(text)
+  
+  expected = [{
+      "type": "variable",
+      "name": "variable",
+      "properties": ["prop", "subprop", "deep"],
+      "start": 0,
+      "end": 27
+  }]
+  
+  assert result == expected
+
+
+def test_i_can_parse_text_with_variable_in_middle():
+  """Test that I can parse text with variable in the middle"""
+  processor = PlainTextPreprocessor()
+  text = "project > $project_id and more"
+  result = processor.parse(text)
+  
+  expected = [
+      {
+          "type": "text",
+          "content": "project > ",
+          "start": 0,
+          "end": 10
+      },
+      {
+          "type": "variable",
+          "name": "project_id",
+          "properties": [],
+          "start": 10,
+          "end": 21
+      },
+      {
+          "type": "text",
+          "content": " and more",
+          "start": 21,
+          "end": 30
+      }
+  ]
+  
+  assert result == expected
+
+
+def test_i_can_parse_multiple_variables():
+  """Test that I can parse text with multiple variables"""
+  processor = PlainTextPreprocessor()
+  text = "value == $variable.prop and $other_var"
+  result = processor.parse(text)
+  
+  expected = [
+      {
+          "type": "text",
+          "content": "value == ",
+          "start": 0,
+          "end": 9
+      },
+      {
+          "type": "variable",
+          "name": "variable",
+          "properties": ["prop"],
+          "start": 9,
+          "end": 23
+      },
+      {
+          "type": "text",
+          "content": " and ",
+          "start": 23,
+          "end": 28
+      },
+      {
+          "type": "variable",
+          "name": "other_var",
+          "properties": [],
+          "start": 28,
+          "end": 38
+      }
+  ]
+  assert result == expected
+
+
+def test_i_can_preserve_all_whitespace():
+  """Test that I can preserve all whitespace including tabs and newlines"""
+  processor = PlainTextPreprocessor()
+  text = "  $var  \t\n  $other.prop  "
+  result = processor.parse(text)
+  
+  expected = [
+      {
+          "type": "text",
+          "content": "  ",
+          "start": 0,
+          "end": 2
+      },
+      {
+          "type": "variable",
+          "name": "var",
+          "properties": [],
+          "start": 2,
+          "end": 6
+      },
+      {
+          "type": "text",
+          "content": "  \t\n  ",
+          "start": 6,
+          "end": 12
+      },
+      {
+          "type": "variable",
+          "name": "other",
+          "properties": ["prop"],
+          "start": 12,
+          "end": 23
+      },
+      {
+          "type": "text",
+          "content": "  ",
+          "start": 23,
+          "end": 25
+      }
+  ]
+  assert result == expected
+
+
+def test_i_can_parse_text_with_special_characters():
+  """Test that I can parse text with special characters"""
+  processor = PlainTextPreprocessor()
+  text = "Hello $user! @#%^&*()+={}[]|\\:;\"'<>?,./~`"
+  result = processor.parse(text)
+  
+  expected = [
+      {
+          "type": "text",
+          "content": "Hello ",
+          "start": 0,
+          "end": 6
+      },
+      {
+          "type": "variable",
+          "name": "user",
+          "properties": [],
+          "start": 6,
+          "end": 11
+      },
+      {
+          "type": "text",
+          "content": "! @#%^&*()+={}[]|\\:;\"'<>?,./~`",
+          "start": 11,
+          "end": 41
+      }
+  ]
+  assert result == expected
+
+
+def test_i_can_parse_complex_expression():
+  """Test that I can parse complex but valid expression"""
+  processor = PlainTextPreprocessor()
+  text = "if ($user.profile.age > 18 && $user.status == 'active') { $action.execute(); }"
+  result = processor.parse(text)
+  
+  # Should parse successfully and find all variables
+  variables = [elem for elem in result if elem["type"] == "variable"]
+  assert len(variables) == 3
+  
+  # Check variable details
+  assert variables[0]["name"] == "user"
+  assert variables[0]["properties"] == ["profile", "age"]
+  
+  assert variables[1]["name"] == "user"
+  assert variables[1]["properties"] == ["status"]
+  
+  assert variables[2]["name"] == "action"
+  assert variables[2]["properties"] == ["execute"]
+
+
+def test_positions_are_accurate():
+  """Test that element positions are accurate"""
+  processor = PlainTextPreprocessor()
+  text = "abc$var123*def"
+  result = processor.parse(text)
+  
+  assert len(result) == 3
+  
+  # Text before
+  assert result[0]["start"] == 0
+  assert result[0]["end"] == 3
+  assert result[0]["content"] == "abc"
+  
+  # Variable
+  assert result[1]["start"] == 3
+  assert result[1]["end"] == 10
+  assert result[1]["name"] == "var123"
+  
+  # Text after
+  assert result[2]["start"] == 10
+  assert result[2]["end"] == 14
+  assert result[2]["content"] == "*def"
+
+
+# Error cases
+def test_i_cannot_parse_dollar_alone_at_end():
+  """Test that I cannot parse $ at the end of text"""
+  processor = PlainTextPreprocessor()
+  text = "Hello $"
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 7
+  assert "Invalid syntax" in str(exc_info.value)
+  # assert "Variable name missing after '$'" in str(exc_info.value)
+
+
+def test_i_cannot_parse_dollar_alone_in_middle():
+  """Test that I cannot parse $ alone in middle of text"""
+  processor = PlainTextPreprocessor()
+  text = "Hello $ world"
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 7
+  assert "Invalid syntax" in str(exc_info.value)
+
+
+def test_i_cannot_parse_dot_immediately_after_dollar():
+  """Test that I cannot parse $.property (dot immediately after $)"""
+  processor = PlainTextPreprocessor()
+  text = "$.property"
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 1
+  assert "Invalid syntax" in str(exc_info.value)
+  # assert "Variable name missing before '.'" in str(exc_info.value)
+
+
+def test_i_cannot_parse_variable_ending_with_dot():
+  """Test that I cannot parse $variable. (dot at the end)"""
+  processor = PlainTextPreprocessor()
+  text = "$variable."
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 9
+  assert "Invalid syntax in property name." in str(exc_info.value)
+
+
+@pytest.mark.parametrize("text", ["$variable. prop", "$variable .prop", "$variable . prop"])
+def test_i_cannot_parse_variable_when_space_in_variable_name(text):
+  """Test that I cannot parse $variable. (dot at the end)"""
+  processor = PlainTextPreprocessor()
+  # text = "$variable. "
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 9
+  assert "Invalid syntax in property name." in str(exc_info.value)
+
+
+def test_i_cannot_parse_variable_with_empty_property():
+  """Test that I cannot parse $variable..property (empty property between dots)"""
+  processor = PlainTextPreprocessor()
+  text = "$variable..property"
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 9
+  assert "Invalid syntax in property name." in str(exc_info.value)
+
+
+def test_i_cannot_parse_variable_ending_with_multiple_dots():
+  """Test that I cannot parse $variable... (multiple dots at end)"""
+  processor = PlainTextPreprocessor()
+  text = "$variable..."
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 9
+  assert "Invalid syntax in property name." in str(exc_info.value)
+
+
+def test_i_cannot_parse_when_consecutive_variables():
+  """Test that I can parse consecutive variables without text between"""
+  processor = PlainTextPreprocessor()
+  text = "$var1$var2"
+  
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  assert exc_info.value.position == 5
+  assert "Invalid syntax." in str(exc_info.value)
+
+
+def test_first_error_is_reported_with_multiple_errors():
+  """Test that first error is reported when multiple $ errors exist"""
+  processor = PlainTextPreprocessor()
+  text = "$ and $. and $var."
+  with pytest.raises(VariableParsingError) as exc_info:
+    processor.parse(text)
+  
+  # Should report the first error ($ alone)
+  assert exc_info.value.position == 1
+
+
+def test_i_can_preprocess_simple_variable():
+  """Test preprocessing text with a simple variable"""
+  processor = PlainTextPreprocessor()
+  namespace = {"name": "John"}
+  result = processor.preprocess("Hello $name!", namespace)
+  assert result == "Hello John!"
+
+
+def test_i_can_preprocess_with_properties():
+  """Test preprocessing text with variable properties"""
+  
+  class User:
+    def __init__(self):
+      self.profile = type('Profile', (), {'age': 25})()
+  
+  processor = PlainTextPreprocessor()
+  namespace = {"user": User()}
+  result = processor.preprocess("Age: $user.profile.age", namespace)
+  assert result == "Age: 25"
+
+
+def test_i_can_preprocess_multiple_variables():
+  """Test preprocessing text with multiple variables"""
+  processor = PlainTextPreprocessor()
+  namespace = {"first": "Hello", "second": "World"}
+  result = processor.preprocess("$first $second!", namespace)
+  assert result == "Hello World!"
+
+
+def test_i_can_preprocess_empty_text():
+  """Test preprocessing empty text"""
+  processor = PlainTextPreprocessor()
+  namespace = {}
+  result = processor.preprocess("", namespace)
+  assert result == ""
+
+
+def test_i_cannot_preprocess_undefined_variable():
+  """Test preprocessing with undefined variable raises error"""
+  processor = PlainTextPreprocessor()
+  namespace = {}
+  with pytest.raises(VariableProcessingError) as exc_info:
+    processor.preprocess("$undefined_var", namespace)
+  assert "Variable 'undefined_var' is not defined" in str(exc_info.value)
+
+
+def test_i_cannot_preprocess_invalid_property():
+  """Test preprocessing with invalid property access"""
+  processor = PlainTextPreprocessor()
+  namespace = {"obj": object()}
+  with pytest.raises(VariableProcessingError) as exc_info:
+    processor.preprocess("some text $obj.invalid_prop", namespace)
+  
+  assert "Invalid property 'invalid_prop' for variable 'obj'" in str(exc_info.value)
+  assert exc_info.value.position == 14