Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept
Fixed #33 : Do not memorize object during restore

src/parsers/BaseNodeParser.py

@@ -441,6 +441,11 @@ class GrammarErrorNode(ParsingError):
     message: str
 
 
+@dataclass()
+class NoMatchingTokenError(ParsingError):
+    pos: int
+
+
 class SyaAssociativity(Enum):
     Left = "left"
     Right = "right"
@@ -720,6 +725,35 @@ class CNC(CN):
             txt += f", {k}='{v}'"
         return txt + ")"
 
+    def to_compare(self, other, to_compare_delegate):
+        """
+        Transform other into CNC, to ease the comparison
+        :param other:
+        :param to_compare_delegate:
+        :return:
+        """
+
+        if isinstance(other, CNC):
+            return other
+
+        if isinstance(other, ConceptNode):
+            if self.exclude_body:
+                compiled = {k: v for k, v in other.concept.get_compiled().items() if k != ConceptParts.BODY}
+            else:
+                compiled = other.concept.get_compiled()
+
+            self_compile_to_use = self.compiled or compiled
+
+            compiled = to_compare_delegate(self_compile_to_use, compiled, to_compare_delegate)
+            return CNC(other.concept,
+                       other.start if self.start is not None else None,
+                       other.end if self.end is not None else None,
+                       other.source if self.source is not None else None,
+                       self.exclude_body,
+                       **compiled)
+
+        raise NotImplementedError("CNC")
+
 
 class UTN(HelperWithPos):
     """
@@ -763,6 +797,24 @@ class UTN(HelperWithPos):
             txt += f", end={self.end}"
         return txt + ")"
 
+    def to_compare(self, other, to_compare_delegate):
+        """
+        Transform other into CNC, to ease the comparison
+        :param other:
+        :param to_compare_delegate:
+        :return:
+        """
+
+        if isinstance(other, UTN):
+            return other
+
+        if isinstance(other, UnrecognizedTokensNode):
+            return UTN(other.source,
+                       other.start,
+                       other.end)
+
+        raise NotImplementedError("UTN")
+
 
 class RN(HelperWithPos):
     """
@@ -840,9 +892,19 @@ class BaseNodeParser(BaseParser):
         :return:
         """
         from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
-        concepts_by_first_keyword = SheerkaConceptManager.compute_concepts_by_first_token(context, concepts).body
-        resolved = SheerkaConceptManager.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword).body
+        service = context.sheerka.services[SheerkaConceptManager.NAME]
+        by_token, by_regex = SheerkaConceptManager.compute_concepts_by_first_item(context, concepts).body
 
+        context.sheerka.om.put(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY,
+                               False,
+                               {k.serialize(): v for k, v in by_regex.items()})
+        compiled = service.compile_concepts_by_first_regex(context, by_regex).body
+        service.compiled_concepts_by_regex.clear()
+        service.compiled_concepts_by_regex.extend(compiled)
+
+        resolved = SheerkaConceptManager.resolve_concepts_by_first_keyword(context, by_token).body
         context.sheerka.om.put(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
                                False,
                                resolved)
+
+        return self

src/parsers/BnfDefinitionParser.py

@@ -4,7 +4,7 @@ from core.sheerka.Sheerka import ExecutionContext
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 from parsers.BaseParser import BaseParser, UnexpectedTokenParsingError, UnexpectedEofParsingError
 from parsers.BnfNodeParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, \
-    ConceptExpression, StrMatch
+    ConceptExpression, StrMatch, RegExMatch, VariableExpression
 
 
 class BnfDefinitionParser(BaseParser):
@@ -231,9 +231,11 @@ class BnfDefinitionParser(BaseParser):
         if token.type == TokenKind.CONCEPT:
             self.next_token()
             concept = self.sheerka.new((token.value[0], token.value[1]))
-            expr = ConceptExpression(concept)
-            # expr = ConceptGroupExpression(concept) if self.sheerka.isaset(self.context, concept) \
-            #     else ConceptExpression(concept)
+            if not self.sheerka.is_known(concept):
+                self.add_error(concept)
+                return None
+
+            expr = ConceptExpression(concept, rule_name=concept.name)
             return self.eat_rule_name_if_needed(expr)
 
         if token.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD):
@@ -245,20 +247,19 @@ class BnfDefinitionParser(BaseParser):
             # (for example of recursive bnf definition)
             if self.context.obj and hasattr(self.context.obj, "name"):
                 if concept_name == str(self.context.obj.name):
-                    return self.eat_rule_name_if_needed(ConceptExpression(concept_name))
+                    return self.eat_rule_name_if_needed(ConceptExpression(concept_name))  # 2021-02-17 no rule name ?
 
             concept = self.context.get_concept(concept_name)
             if not self.sheerka.is_known(concept):
-                self.add_error(concept)
-                return None
+                expr = VariableExpression(concept_name)
+                return self.eat_rule_name_if_needed(expr)
             elif hasattr(concept, "__iter__"):
                 self.add_error(
                     self.sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
                                      body=("key", concept_name)))
                 return None
             else:
-                expr = ConceptExpression(concept)
-                expr.rule_name = concept.name
+                expr = ConceptExpression(concept, rule_name=concept.name)
                 return self.eat_rule_name_if_needed(expr)
 
         if token.type == TokenKind.STRING:
@@ -272,6 +273,11 @@ class BnfDefinitionParser(BaseParser):
                 ret = Sequence(*elements)
                 return self.eat_rule_name_if_needed(ret)
 
+        if token.type == TokenKind.REGEX:
+            self.next_token()
+            ret = RegExMatch(core.utils.strip_quotes(token.strip_quote))
+            return self.eat_rule_name_if_needed(ret)
+
         ret = StrMatch(core.utils.strip_quotes(token.value))
         self.next_token()
         return self.eat_rule_name_if_needed(ret)

src/parsers/PythonParser.py

@@ -30,6 +30,10 @@ class PythonErrorNode(ParsingError):
 
 @dataclass()
 class ConceptDetectedError(ParsingError):
+    """
+    When the Python parser finds an identifier, and that identifier is a concept
+    So it's not for the PythonParser to respond
+    """
     name: str
 
 

src/parsers/SyaNodeParser.py

@@ -1142,6 +1142,8 @@ class SyaNodeParser(BaseNodeParser):
         if sya_definitions:
             self.test_only_sya_definitions = sya_definitions
 
+        return self
+
     @staticmethod
     def _is_eligible(concept):
         """