ConceptLexerParser can how handle UnrecognizedTokens

evaluators/ConceptNodeEvaluator.py

@@ -1,7 +1,7 @@
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 
-from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch
+from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode
 
 
 class ConceptNodeEvaluator(OneReturnValueEvaluator):
@@ -17,15 +17,22 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
     def matches(self, context, return_value):
         if not return_value.status:
             return False
+
         if not isinstance(return_value.value, ParserResultConcept):
             return False
 
-        return (isinstance(return_value.value.value, ConceptNode) or
+        return (
+            isinstance(return_value.value.value, ConceptNode) or
+            isinstance(return_value.value.value, UnrecognizedTokensNode) or
+            (
+                hasattr(return_value.value.value, "__iter__") and
+                len(return_value.value.value) > 0 and
                 (
-                    hasattr(return_value.value.value, "__iter__") and
-                    len(return_value.value.value) > 0 and
-                    isinstance(return_value.value.value[0], ConceptNode)
-                ))
+                    isinstance(return_value.value.value[0], ConceptNode) or
+                    isinstance(return_value.value.value[0], UnrecognizedTokensNode)
+                )
+            )
+        )
 
     def eval(self, context, return_value):
         """
@@ -38,19 +45,23 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator):
             nodes = [nodes]
 
         concepts = []
+        error_found = False
         for node in nodes:
-            concept = sheerka.new(node.concept.key)
-            concept = self.update_concept(sheerka, concept, node.underlying)
-            concepts.append(concept)
+            if isinstance(node, ConceptNode):
+                concept = sheerka.new(node.concept.key)
+                concept = self.update_concept(sheerka, concept, node.underlying)
+                concepts.append(concept)
+            else:
+                error_found = True
 
         if len(concepts) == 1:
             return sheerka.ret(
                 self.name,
-                True,
+                not error_found,
                 concepts[0],
                 parents=[return_value])
 
-        raise NotImplementedError("Not yet")
+        return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value])
 
     def update_concept(self, sheerka, concept, underlying, init_empty_body=True):
         """