First version of explain. Creating a new parser was a wrong approach. Need to reimplement

src/core/builtin_concepts.py

@@ -58,6 +58,7 @@ class BuiltinConcepts(Enum):
     WHERE_CLAUSE_FAILED = "where clause failed"  # failed to validate where clause during evaluation
     CHICKEN_AND_EGG = "chicken and egg"  # infinite recursion when declaring concept
     ISA = "is a"  # builtin concept to express that a concept is an instance of another one
+    EXPLANATION = "explanation"
 
     NODE = "node"
     GENERIC_NODE = "generic node"
@@ -436,3 +437,17 @@ class NotForMeConcept(Concept):
 
     def __repr__(self):
         return f"NotForMeConcept(source={self.body}, reason={self.get_prop('reason')})"
+
+
+class ExplanationConcept(Concept):
+    def __init__(self, digest=None, command=None, title=None, instructions=None, execution_result=None):
+        super().__init__(BuiltinConcepts.EXPLANATION,
+                         True,
+                         False,
+                         BuiltinConcepts.EXPLANATION)
+        self.def_prop("digest", digest)  # event digest
+        self.def_prop("command", command)  # explain command parameters
+        self.def_prop("title", title)  # a title to the explanation
+        self.def_prop("instructions", instructions)  # instructions for SheerkaPrint
+        self.set_metadata_value(ConceptParts.BODY, execution_result) # list of results
+        self.metadata.is_evaluated = True

src/core/builtin_helpers.py

@@ -2,7 +2,7 @@ import ast
 import logging
 
 import core.ast.nodes
-from core.ast.nodes import CallNodeConcept, GenericNodeConcept
+from core.ast.nodes import CallNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept

src/core/concept.py

@@ -276,6 +276,10 @@ class Concept:
     def to_dict(self, props_to_use=None):
         """
         Returns a dict representing 'self'
+        to_dict() is used for serializing the definition of the concept
+        You will not that it does not dump the actual values of the properties, nor the body
+
+        If you need a dictionary version of the Concept, use to_bag()
         :return:
         """
 
@@ -368,6 +372,7 @@ class Concept:
         :return:
         """
         self.values[metadata] = value
+        return self
 
     def get_metadata_value(self, metadata: ConceptParts):
         """
@@ -407,6 +412,20 @@ class Concept:
     def get_original_definition_hash(self):
         return self.original_definition_hash
 
+    def to_bag(self):
+        """
+        Creates a dictionary with the useful properties of the concept
+        It quicker to implement than creating the actual property mechanism with @property
+        And it removes the visibility from the other attributes/methods
+        """
+        bag = {}
+        for prop in self.props:
+            bag[prop] = self.get_prop(prop)
+            bag["prop." + prop] = self.get_prop(prop)
+        for prop in ("id", "name", "key", "body"):
+            bag[prop] = getattr(self, prop)
+        return bag
+
 
 class Property:
     """

src/core/sheerka/ExecutionContext.py

@@ -3,6 +3,7 @@ import time
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from core.sheerka.Services.SheerkaExecute import NO_MATCH
 from core.sheerka_logger import get_logger
 from sdp.sheerkaDataProvider import Event
 
@@ -261,6 +262,74 @@ class ExecutionContext:
 
         return False
 
+    @staticmethod
+    def _is_return_value(obj):
+        return isinstance(obj, Concept) and obj.key == str(BuiltinConcepts.RETURN_VALUE)
+
+    def _at_least_one_success(self, return_values):
+        status = False
+        for ret_val in return_values:
+            if not self._is_return_value(ret_val):
+                return None
+            status |= ret_val.status
+        return status
+
+    def _all_success(self, return_values):
+        status = True
+        for ret_val in return_values:
+            if not self._is_return_value(ret_val):
+                return None
+            status &= ret_val.status
+        return status
+
+    def get_status(self):
+        # In the function, I cannot use sheerka.isinstance() as self.sheerka may not be initialized
+        # This is the case when ExecutionContext is deserialized
+
+        if "return_values" not in self.values:
+            return None
+
+        if hasattr(self.values["return_values"], "__iter__"):
+            values = self.values["return_values"]
+            if len(values) == 0:
+                return None
+
+            if isinstance(values, str):
+                return "No Match" if values == NO_MATCH else values
+
+            if isinstance(values[0], dict):
+                for result in values:
+                    if "return_value" not in result:
+                        return None
+                    if self._is_return_value(result["return_value"]):
+                        return result["return_value"].status
+                return "No Match"
+            else:
+                return self._at_least_one_success(self.values["return_values"])
+
+        else:
+            ret_val = self.values["return_values"]
+            if not isinstance(ret_val, Concept) or not ret_val.key == str(BuiltinConcepts.RETURN_VALUE):
+                return None
+            return ret_val.status
+
+    def to_bag(self):
+        """
+        Creates a dictionary with the useful properties of the concept
+        It quicker to implement than creating the actual property mechanism with @property
+        And it removes the visibility from the other attributes/methods
+        """
+        bag = {}
+        for k, v in self._bag.items():
+            bag[k] = v
+            bag["bag." + k] = v
+        for prop in ("id", "who", "desc", "obj", "inputs", "values", "concepts"):
+            bag[prop] = getattr(self, prop)
+        bag["status"] = self.get_status()
+        bag["elapsed"] = self.elapsed
+        bag["digest"] = self.event.get_digest() if self.event else None
+        return bag
+
     @staticmethod
     def return_value_to_str(r):
         value = str(r.value)

src/core/sheerka/Services/SheerkaDump.py

@@ -86,7 +86,7 @@ class SheerkaDump:
 
             while True:
                 try:
-                    if h.event.user != self.sheerka.name:
+                    if h.result:
                         self.sheerka.log.info(h)
                         count += 1
                     h = next(history)

src/core/sheerka/Services/SheerkaExecute.py

@@ -1,6 +1,7 @@
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 import core.utils
 
+NO_MATCH = "** No Match **"
 
 class SheerkaExecute:
     """
@@ -159,7 +160,7 @@ class SheerkaExecute:
                                             evaluated_items.append(result)
                                         debug_result.append({"input": item, "return_value": result})
                                     else:
-                                        debug_result.append({"input": item, "return_value": "** No Match **"})
+                                        debug_result.append({"input": item, "return_value": NO_MATCH})
                                 sub_context.add_values(return_values=debug_result)
 
                             # process evaluators that work on all return values
@@ -175,7 +176,7 @@ class SheerkaExecute:
                                         to_delete.extend(result.parents)
                                     sub_context.add_values(return_values=results)
                                 else:
-                                    sub_context.add_values(return_values="** No Match **")
+                                    sub_context.add_values(return_values=NO_MATCH)
 
                     return_values = evaluated_items
                     return_values.extend([item for item in original_items if item not in to_delete])

src/core/sheerka/Services/SheerkaHistoryManager.py

@@ -2,7 +2,7 @@ from collections import namedtuple
 
 from sdp.sheerkaDataProvider import Event
 
-hist = namedtuple("History", "text status")  # tests purposes only
+hist = namedtuple("HistoryTest", "text status")  # tests purposes only
 
 
 class History:
@@ -38,34 +38,23 @@ class History:
         if self._status:
             return self._status
 
-        if not self.result or "return_values" not in self.result.values:
-            return
-
-        if hasattr(self.result.values["return_values"], "__iter__"):
-            if len(self.result.values["return_values"]) != 1:
-                self._status = False
-                return self._status
-            else:
-                self._status = self.result.values["return_values"][0].status
-                return self._status
-        else:
-            self._status = self.result.values["return_values"].status
-            return self._status
+        self._status = self.result.get_status() if self.result else None
+        return self._status
 
 
 class SheerkaHistoryManager:
     def __init__(self, sheerka):
         self.sheerka = sheerka
 
-    def history(self, depth_or_digest, start):
+    def history(self, depth, start):
         """
         Load history
-        :param depth_or_digest: number of items or digest
+        :param depth: number of items
         :param start:
         :return:
         """
 
-        events = list(self.sheerka.sdp.load_events(depth_or_digest, start))
+        events = list(self.sheerka.sdp.load_events(depth, start))
         for event in events:
             try:
                 result = self.sheerka.sdp.load_result(event.get_digest())

src/core/sheerka/Services/SheerkaVariableManager.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from typing import List
+
+from sdp.sheerkaSerializer import Serializer
+
+
+@dataclass
+class Variable:
+    """
+    Variable to store
+    """
+    event_id: str  # event where the variable is modified
+    who: str  # who is the modifier
+    key: str  # key of the variable
+    value: object  # value
+    parents: List[str]  # previous references of the variable (Note that there should be only one parent)
+
+    def get_key(self):
+        return f"{self.who}.{self.key}"
+
+
+class SheerkaVariableManager:
+    VARIABLES_ENTRY = "All_Variables"  # to store all the concepts
+
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+
+    def record(self, context, who, key, value):
+        """Persist a variable"""
+        # first check if there is a previous version of the variable
+        try:
+            old = self.sheerka.sdp.get(self.VARIABLES_ENTRY, who + "." + key)
+            if old.value == value:
+                return
+
+            parent = getattr(old, Serializer.ORIGIN)
+        except IndexError:
+            parent = None
+
+        variable = Variable(context.event.get_digest(), who, key, value, [parent] if parent else None)
+        self.sheerka.sdp.set(context.event.get_digest(), self.VARIABLES_ENTRY, variable, use_ref=True)
+
+    def load(self, who, key):
+        variable = self.sheerka.sdp.get_safe(self.VARIABLES_ENTRY, who + "." + key)
+        if variable is None:
+            return None
+
+        return variable.value
+
+    def delete(self, context, who, key):
+        self.sheerka.sdp.remove(
+            context.event.get_digest(),
+            self.VARIABLES_ENTRY,
+            lambda _key, _var: _key == who + "." + key)

src/core/sheerka/Sheerka.py

@@ -1,3 +1,7 @@
+import logging
+
+import core.builtin_helpers
+import core.utils
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
     UnknownConcept
 from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
@@ -9,13 +13,10 @@ from core.sheerka.Services.SheerkaExecute import SheerkaExecute
 from core.sheerka.Services.SheerkaHistoryManager import SheerkaHistoryManager
 from core.sheerka.Services.SheerkaModifyConcept import SheerkaModifyConcept
 from core.sheerka.Services.SheerkaSetsManager import SheerkaSetsManager
-from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
-import core.utils
-import core.builtin_helpers
-
+from core.sheerka.Services.SheerkaVariableManager import SheerkaVariableManager
 from core.sheerka_logger import console_handler
-
-import logging
+from printer.SheerkaPrinter import SheerkaPrinter
+from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
 
 CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
 BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
@@ -93,6 +94,8 @@ class Sheerka(Concept):
         self.sets_handler = SheerkaSetsManager(self)
         self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
         self.history_handler = SheerkaHistoryManager(self)
+        self.printer_handler = SheerkaPrinter(self)
+        self.variable_handler = SheerkaVariableManager(self)
 
         self.during_restore = False
         self._builtins_classes_cache = None
@@ -127,7 +130,7 @@ class Sheerka(Concept):
 
                 exec_context.add_values(return_values=res)
                 if not self.skip_builtins_in_db:
-                    self.sdp.save_result(exec_context)
+                    self.sdp.save_result(exec_context, is_admin=True)
                 self.init_log.debug(f"Sheerka successfully initialized")
 
         except IOError as e:
@@ -299,9 +302,26 @@ class Sheerka(Concept):
         #     if len(ret) == 1 and ret[0].status and self.isinstance(ret[0].value, BuiltinConcepts.NEW_CONCEPT):
         #         with open(CONCEPTS_FILE, "a") as f:
         #             f.write(text + "\n")
-
         return ret
 
+    def print(self, result, instructions=None):
+        """
+        Print the result to output
+        :param result:
+        :param instructions:
+        :return:
+        """
+        self.printer_handler.print(result, instructions)
+
+    def record(self, context, who, key, value):
+        return self.variable_handler.record(context, who, key, value)
+
+    def load(self, who, key):
+        return self.variable_handler.load(who, key)
+
+    def delete(self, context, who, key):
+        return self.variable_handler.delete(context, who, key)
+
     def execute(self, execution_context, return_values, execution_steps):
         """
         Executes process for all initial contexts
@@ -639,12 +659,27 @@ class Sheerka(Concept):
 
         return self.value(body_to_use)
 
+    def value_by_concept(self, obj, concept):
+        if obj is None:
+            return None
+
+        if not isinstance(obj, Concept):
+            return None
+
+        if isinstance(concept, tuple) and obj.key in [str(key) for key in concept]:
+            return obj
+
+        if obj.key == str(concept):
+            return obj
+
+        return self.value_by_concept(obj.body, concept)
+
     def get_error(self, obj):
         if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
             return obj
 
         if isinstance(obj, list):
-            return  obj
+            return obj
 
         if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
             if obj.status:

src/core/tokenizer.py

@@ -46,6 +46,8 @@ class TokenKind(Enum):
     TILDE = "tilde"  # ~
     UNDERSCORE = "underscore"  # _
     DEGREE = "degree"  # °
+    WORD = "word"
+    EQUALSEQUALS = "=="
 
 
 @dataclass()
@@ -99,12 +101,13 @@ class Tokenizer:
 
     KEYWORDS = set(x.value for x in Keywords)
 
-    def __init__(self, text):
+    def __init__(self, text, parse_word=False):
         self.text = text
         self.text_len = len(text)
         self.column = 1
         self.line = 1
         self.i = 0
+        self.parse_word = parse_word
 
     def __iter__(self):
 
@@ -175,9 +178,14 @@ class Tokenizer:
                 self.i += 1
                 self.column += 1
             elif c == "=":
-                yield Token(TokenKind.EQUALS, "=", self.i, self.line, self.column)
-                self.i += 1
-                self.column += 1
+                if self.i + 1 < self.text_len and self.text[self.i + 1] == "=":
+                    yield Token(TokenKind.EQUALSEQUALS, "==", self.i, self.line, self.column)
+                    self.i += 2
+                    self.column += 2
+                else:
+                    yield Token(TokenKind.EQUALS, "=", self.i, self.line, self.column)
+                    self.i += 1
+                    self.column += 1
             elif c == " " or c == "\t":
                 whitespace = self.eat_whitespace(self.i)
                 yield Token(TokenKind.WHITESPACE, whitespace, self.i, self.line, self.column)
@@ -270,6 +278,11 @@ class Tokenizer:
                 yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column)
                 self.i += length + 2
                 self.column += length + 2
+            elif self.parse_word and (c.isalpha() or c.isdigit()):
+                word = self.eat_word(self.i)
+                yield Token(TokenKind.WORD, word, self.i, self.line, self.column)
+                self.i += len(word)
+                self.column += len(word)
             elif c.isalpha() or c == "_":
                 identifier = self.eat_identifier(self.i)
                 token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
@@ -419,3 +432,16 @@ class Tokenizer:
                              1 if lines_count > 0 else start_column + len(result))
 
         return result, lines_count
+
+    def eat_word(self, start):
+        result = self.text[start]
+        i = start + 1
+        while i < self.text_len:
+            c = self.text[i]
+            if c.isalpha() or c.isdigit():
+                result += c
+                i += 1
+            else:
+                break
+
+        return result

src/core/utils.py

@@ -176,7 +176,7 @@ def product(a, b):
     res = []
     for item_b in b:
         for item_a in a:
-            #items = item_a + [item_b]
+            # items = item_a + [item_b]
             items = item_a[:]
             if hasattr(item_b, "__iter__"):
                 items.extend(item_b)
@@ -235,16 +235,6 @@ def escape_char(text, to_escape):
     return res
 
 
-def pp(items):
-    if not hasattr(items, "__iter__"):
-        return str(items)
-
-    if len(items) == 0:
-        return str(items)
-
-    return "    \n" + "    \n".join(str(item) for item in items)
-
-
 def decode_enum(enum_repr: str):
     """
     Tries to transform ClassName.Name into an enum