Added first implementation of concepts ambiguity resolution + Jenkins file test

src/evaluators/AddConceptEvaluator.py

@@ -2,7 +2,7 @@ from core.ast.nodes import python_to_concept
 from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
 from core.builtin_helpers import get_names
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
-from core.tokenizer import TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseParser import NotInitializedNode
 from parsers.BnfNodeParser import ParsingExpression, ParsingExpressionVisitor
@@ -53,7 +53,7 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         sheerka = context.sheerka
 
         # validate the node
-        props_found = set()
+        variables_found = set()
 
         concept = Concept(def_concept_node.name)
         concept.metadata.definition_type = def_concept_node.definition_type
@@ -80,16 +80,16 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
 
             # try to find what can be a property
             for p in self.get_variables(sheerka, part_ret_val, name_to_use):
-                props_found.add(p)
+                variables_found.add(p)
 
         # add variables by order of appearance when possible
         for name_part in name_to_use:
-            if name_part in props_found:
+            if name_part in variables_found:
                 concept.def_var(name_part, None)
 
         # add the remaining properties
         # They mainly come from BNF definition
-        for p in props_found:
+        for p in variables_found:
             if p not in concept.values:
                 concept.def_var(p, None)
 
@@ -130,24 +130,31 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
             names = [str(t.value) for t in ret_value.tokens if t.type in (
                 TokenKind.IDENTIFIER, TokenKind.STRING, TokenKind.KEYWORD)]
             variables = filter(lambda x: x in concept_name, names)
-            return list(variables)
+            return set(variables)
 
         #
         # Case of python code
         #
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, PythonNode):
             if len(concept_name) > 1:
-                python_node = ret_value.value.value
-                as_concept_node = python_to_concept(python_node.ast_)
-                names = get_names(sheerka, as_concept_node)
-                variables = filter(lambda x: x in concept_name, names)
-                return list(variables)
+                # tokens from ParserResult or source from python node
+                variables = set()
+                tokens = ret_value.value.tokens or list(Tokenizer(ret_value.value.value.source))
+                tokens = [t.str_value for t in tokens]
+                for identifier in [i for i in concept_name if str(i).isalnum()]:
+                    if identifier in tokens:
+                        variables.add(identifier)
+                # python_node = ret_value.value.value
+                # as_concept_node = python_to_concept(python_node.ast_)
+                # names = get_names(sheerka, as_concept_node)
+                # variables = filter(lambda x: x in concept_name, names)
+                return variables
 
         #
         # case of concept
         #
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, Concept):
-            return list(ret_value.value.value.values.keys())
+            return set(ret_value.value.value.values.keys())
 
         #
         # case of BNF
@@ -155,6 +162,6 @@ class AddConceptEvaluator(OneReturnValueEvaluator):
         if isinstance(ret_value.value, ParserResultConcept) and isinstance(ret_value.value.value, ParsingExpression):
             visitor = ConceptOrRuleNameVisitor()
             visitor.visit(ret_value.value.value)
-            return sorted(list(visitor.names))
+            return set(visitor.names)
 
         return []