intermediate commit

src/parsers/SimpleConceptsParser.py

@@ -1,11 +1,62 @@
 from core.concept import DefinitionType
 from evaluators.base_evaluator import MultipleChoices
-from parsers.state_machine import ConceptToRecognize, End, ManageUnrecognized, MetadataToken, PrepareReadTokens, \
-    ReadConcept, ReadTokens, Start, StateMachine, StateMachineContext, UnrecognizedToken
+from parsers.BaseParser import BaseParser
+from parsers.parser_utils import UnexpectedEof, UnexpectedToken, get_text_from_tokens
+from parsers.state_machine import ConceptToRecognize, End, MetadataToken, PrepareReadTokens, \
+    ReadTokens, Start, State, StateMachine, StateMachineContext, StateResult, UnrecognizedToken
 from parsers.tokenizer import Token, TokenKind, Tokenizer
 
 
-class SimpleConceptsParser:
+class ReadConcept(State):
+    def run(self, state_context) -> StateResult:
+        start = state_context.parser_input.pos
+
+        for expected in state_context.concept_to_recognize.expected:
+            if not state_context.parser_input.next_token(False):
+                # eof before the concept is recognized
+                state_context.errors.append(UnexpectedEof(expected, state_context.parser_input.token))
+                state_context.concept_to_recognize = None
+                return StateResult(self.next_states[0])
+
+            token = state_context.parser_input.token
+            if token.value != expected:
+                # token mismatch
+                state_context.errors.append(UnexpectedToken(token, expected))
+                state_context.concept_to_recognize = None
+                return StateResult(self.next_states[0])
+
+        state_context.result.append(MetadataToken(state_context.concept_to_recognize.metadata,
+                                                  start,
+                                                  state_context.parser_input.pos,
+                                                  state_context.concept_to_recognize.resolution_method,
+                                                  "simple"))
+
+        state_context.concept_to_recognize = None
+        return StateResult(self.next_states[0])
+
+
+class ManageUnrecognized(State):
+    def run(self, state_context) -> StateResult:
+        if state_context.buffer:
+            buffer_as_str = get_text_from_tokens(state_context.buffer)
+            if len(state_context.result) > 0 and isinstance(old := state_context.result[-1], UnrecognizedToken):
+                # merge unrecognized if needed
+                state_context.result[-1] = UnrecognizedToken(old.buffer + buffer_as_str,
+                                                             old.start,
+                                                             state_context.parser_input.pos - 1)
+            else:
+                state_context.result.append(UnrecognizedToken(buffer_as_str,
+                                                              state_context.buffer_start_pos,
+                                                              state_context.parser_input.pos - 1))
+
+        # clear the buffer
+        state_context.buffer.clear()
+        state_context.buffer_start_pos = state_context.parser_input.pos + 1
+
+        return StateResult(self.next_states[0])
+
+
+class SimpleConceptsParser(BaseParser):
     """"
     This class is to parse concepts with no parameter
     ex : def concept I am a new concept
@@ -13,6 +64,8 @@ class SimpleConceptsParser:
     """
 
     def __init__(self):
+        super().__init__("simple")
+
         tokens_wkf = {
             Start("start", next_states=["prepare read tokens"]),
             PrepareReadTokens("prepare read tokens", next_states=["read tokens"]),
@@ -31,7 +84,6 @@ class SimpleConceptsParser:
             "#tokens_wkf": {t.name: t for t in tokens_wkf},
             "#concept_wkf": {t.name: t for t in concept_wkf},
         }
-        self.error_sink = []
 
     @staticmethod
     def get_metadata_from_first_token(context, token: Token):
@@ -56,12 +108,13 @@ class SimpleConceptsParser:
 
         return concepts_by_key + concepts_by_name
 
-    def parse(self, context, parser_input):
+    def parse(self, context, parser_input, error_sink):
         sm = StateMachine(self.workflows)
-        sm_context = StateMachineContext(context, parser_input, self.get_metadata_from_first_token)
+        sm_context = StateMachineContext(context, parser_input, self.get_metadata_from_first_token, [])
         sm.run("#tokens_wkf", "start", sm_context)
 
         selected = self.select_best_paths(sm)
+        error_sink.extend(sm_context.errors)
 
         return MultipleChoices(selected)