Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

src/core/builtin_helpers.py

@@ -30,11 +30,11 @@ def is_same_success(context, return_values):
                     evaluated = context.sheerka.evaluate_concept(sub_context, ret_val.body)
                     if evaluated.key != ret_val.body.key:
                         raise Exception("Failed to evaluate evaluate")
-                return context.sheerka.value(evaluated)
+                return context.sheerka.objvalue(evaluated)
             else:
-                return context.sheerka.value(ret_val.body)
+                return context.sheerka.objvalue(ret_val.body)
         else:
-            return context.sheerka.value(ret_val)
+            return context.sheerka.objvalue(ret_val)
 
     try:
         reference = _get_value(return_values[0])
@@ -280,8 +280,8 @@ def get_lexer_nodes(return_values, start, tokens):
     for ret_val in return_values:
         if ret_val.who == "parsers.Python":
 
-            if ret_val.body.source.strip().isalnum() and not ret_val.body.source.strip().isnumeric():
-                # Discard SourceCodeNode which seems to be a concept
+            if ret_val.body.source.strip().isidentifier():
+                # Discard SourceCodeNode which seems to be a concept name
                 # It may be a wrong idea, so let's see
                 continue
 
@@ -309,6 +309,41 @@ def get_lexer_nodes(return_values, start, tokens):
     return lexer_nodes
 
 
+def ensure_evaluated(context, concept):
+    """
+    Evaluate a concept is not already evaluated
+    :param context:
+    :param concept:
+    :return:
+    """
+    if concept.metadata.is_evaluated:
+        return concept
+
+    with context.push(desc=f"Evaluating concept {concept}") as sub_context:
+        sub_context.local_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        evaluated = context.sheerka.evaluate_concept(sub_context, concept)
+        sub_context.add_values(return_values=evaluated)
+
+    return evaluated
+
+def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers):
+    """
+    Using parsers, try to recognize concepts from source
+    :param context:
+    :param unrecognized_tokens_node:
+    :param parsers:
+    :return:
+    """
+
+    res = parse_unrecognized(context, unrecognized_tokens_node.source, parsers)
+    res = only_parsers_results(context, res)
+
+    if not res.status:
+        return None
+
+    return get_lexer_nodes(res.body.body, unrecognized_tokens_node.start, unrecognized_tokens_node.tokens)
+
+
 def get_names(sheerka, concept_node):
     """
     Finds all the names referenced by the concept_node
@@ -352,7 +387,7 @@ def extract_predicates(sheerka, expression, variables_to_include, variables_to_e
         return NotImplementedError()
 
     concept_node = core.ast.nodes.python_to_concept(node)
-    main_op = concept_node.get_prop("body")
+    main_op = concept_node.get_value("body")
 
     return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude))
 
@@ -370,14 +405,14 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
         return _res
 
     if node.node_type == "Compare":
-        if node.get_prop("left").node_type == "Name":
+        if node.get_value("left").node_type == "Name":
             """Simple case of one comparison"""
-            comparison_name = sheerka.value(node.get_prop("left"))
+            comparison_name = sheerka.objvalue(node.get_value("left"))
             if comparison_name in variables_to_include and comparison_name not in variables_to_exclude:
                 predicates.append(node)
         else:
             """The left part is an expression"""
-            res = _extract_predicates(sheerka, node.get_prop("left"), variables_to_include, variables_to_exclude)
+            res = _extract_predicates(sheerka, node.get_value("left"), variables_to_include, variables_to_exclude)
             if len(res) > 0:
                 predicates.append(node)
     elif node.node_type == "Call":
@@ -386,9 +421,9 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
         args = list(call_node.get_args_names(sheerka))
         if _matches(args, variables_to_include, variables_to_exclude):
             predicates.append(node)
-    elif node.node_type == "UnaryOp" and node.get_prop("op").node_type == "Not":
+    elif node.node_type == "UnaryOp" and node.get_value("op").node_type == "Not":
         """Simple case of negation"""
-        res = _extract_predicates(sheerka, node.get_prop("operand"), variables_to_include, variables_to_exclude)
+        res = _extract_predicates(sheerka, node.get_value("operand"), variables_to_include, variables_to_exclude)
         if len(res) > 0:
             predicates.append(node)
     elif node.node_type == "BinOp":
@@ -398,7 +433,7 @@ def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclud
     elif node.node_type == "BoolOp":
         all_op = True
         temp_res = []
-        for op in node.get_prop("values").body:
+        for op in node.get_value("values").body:
             res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
             if len(res) == 0:
                 all_op = False