diff --git a/.gitignore b/.gitignore index eb57703..3da35fb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ venv .idea __pycache__ build -prof \ No newline at end of file +prof +tests/_concepts.txt \ No newline at end of file diff --git a/_concepts.txt b/_concepts.txt index c456dd6..9612afd 100644 --- a/_concepts.txt +++ b/_concepts.txt @@ -39,5 +39,4 @@ seventeen isa number eighteen isa number nineteen isa number twenty isa number -def concept twenties from bnf twenty number where number < 10 as twenty + number - +def concept twenties from bnf twenty number where number < 10 as twenty + number \ No newline at end of file diff --git a/src/core/builtin_concepts.py b/src/core/builtin_concepts.py index 9fd3872..2981f07 100644 --- a/src/core/builtin_concepts.py +++ b/src/core/builtin_concepts.py @@ -50,6 +50,7 @@ class BuiltinConcepts(Enum): CONCEPT_EVAL_REQUESTED = "concept eval requested" REDUCE_REQUESTED = "reduce requested" # remove meaningless error when possible NOT_A_SET = "not a set" # the concept has no entry in sets + WHERE_CLAUSE_FAILED = "where clause failed" # failed to validate where clause during evaluation NODE = "node" GENERIC_NODE = "generic node" @@ -91,6 +92,7 @@ BuiltinErrors = [str(e) for e in { BuiltinConcepts.CONCEPT_EVAL_ERROR, BuiltinConcepts.CONCEPT_ALREADY_IN_SET, BuiltinConcepts.NOT_A_SET, + BuiltinConcepts.WHERE_CLAUSE_FAILED }] """ @@ -385,3 +387,20 @@ class ConceptAlreadyInSet(Concept): @property def concept_set(self): return self.props["concept_set"].value + + +class WhereClauseFailed(Concept): + def __init__(self, concept=None): + super().__init__(BuiltinConcepts.WHERE_CLAUSE_FAILED, + True, + False, + BuiltinConcepts.WHERE_CLAUSE_FAILED) + self.set_metadata_value(ConceptParts.BODY, concept) + self.metadata.is_evaluated = True + + def __repr__(self): + return f"WhereClauseFailed(concept={self.concept})" + + @property + def concept(self): + return self.body diff --git a/src/core/concept.py b/src/core/concept.py index ceb3321..b67b507 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -231,7 +231,8 @@ class Concept: if token.value in variables: key += VARIABLE_PREFIX + str(variables.index(token.value)) else: - key += token.value[1:-1] if token.type == TokenKind.STRING else token.value + value = token.value[1:-1] if token.type == TokenKind.STRING else token.value + key += value first = False self.metadata.key = key diff --git a/src/core/sheerka/Services/SheerkaCreateNewConcept.py b/src/core/sheerka/Services/SheerkaCreateNewConcept.py index 65bd19e..ba0b69b 100644 --- a/src/core/sheerka/Services/SheerkaCreateNewConcept.py +++ b/src/core/sheerka/Services/SheerkaCreateNewConcept.py @@ -45,7 +45,7 @@ class SheerkaCreateNewConcept: # add the BNF if known if concept.bnf: - concepts_definitions = self.sheerka.get_concept_definition() + concepts_definitions = self.sheerka.get_concepts_definitions(context) concepts_definitions[concept] = concept.bnf # check if it's a valid BNF or whether it breaks the known rules @@ -61,22 +61,26 @@ class SheerkaCreateNewConcept: # save the new concept in sdp try: # TODO : needs to make these calls atomic (or at least one single call) + # save the new concept self.sheerka.sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_ENTRY, concept, use_ref=True) + # save it by id self.sheerka.sdp.add( context.event.get_digest(), self.sheerka.CONCEPTS_BY_ID_ENTRY, {concept.id: concept.get_digest()}, is_ref=True) + # update the definition table if concepts_definitions is not None: self.sheerka.sdp.set( context.event.get_digest(), self.sheerka.CONCEPTS_DEFINITIONS_ENTRY, - concepts_definitions, + concept_lexer_parser.encode_grammar(init_ret_value.body), use_ref=True) + self.sheerka.concepts_definitions_cache = None # invalidate cache except SheerkaDataProviderDuplicateKeyError as error: context.log_error(logger, "Failed to create a new concept.", who=self.logger_name) return self.sheerka.ret( @@ -94,6 +98,3 @@ class SheerkaCreateNewConcept: # process the return in needed ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept)) return ret - - - diff --git a/src/core/sheerka/Services/SheerkaDump.py b/src/core/sheerka/Services/SheerkaDump.py index dd5ce5f..f04f499 100644 --- a/src/core/sheerka/Services/SheerkaDump.py +++ b/src/core/sheerka/Services/SheerkaDump.py @@ -1,5 +1,14 @@ from core.builtin_concepts import BuiltinConcepts from core.concept import Concept +from sdp.sheerkaDataProvider import SheerkaDataProvider +import pprint +import os + + +def get_pp(): + rows, columns = os.popen('stty size', 'r').read().split() + pp = pprint.PrettyPrinter(width=columns, compact=True) + return pp class SheerkaDump: @@ -71,3 +80,8 @@ class SheerkaDump: break page_count += 1 + + def dump_state(self): + snapshot = self.sheerka.sdp.get_snapshot(SheerkaDataProvider.HeadFile) + state = self.sheerka.sdp.load_state(snapshot) + self.sheerka.log.info(get_pp().pformat(state.data)) diff --git a/src/core/sheerka/Services/SheerkaEvaluateConcept.py b/src/core/sheerka/Services/SheerkaEvaluateConcept.py index 7229867..8a33da5 100644 --- a/src/core/sheerka/Services/SheerkaEvaluateConcept.py +++ b/src/core/sheerka/Services/SheerkaEvaluateConcept.py @@ -147,9 +147,6 @@ class SheerkaEvaluateConcept: if concept.metadata.is_evaluated: return concept - # WHERE condition should already be validated by the parser. - # It's a mandatory condition for the concept before it can be recognized - # # TODO : Validate the PRE condition # @@ -157,8 +154,8 @@ class SheerkaEvaluateConcept: self.initialize_concept_asts(context, concept, logger) # to make sure of the order, it don't use ConceptParts.get_parts() - # props must be evaluated first - all_metadata_to_eval = ["props", "where", "pre", "post", "body"] + # props must be evaluated first, body must be evaluated before where + all_metadata_to_eval = ["pre", "post", "props", "body", "where"] for metadata_to_eval in all_metadata_to_eval: if metadata_to_eval == "props": @@ -186,6 +183,12 @@ class SheerkaEvaluateConcept: else: concept.values[part_key] = resolved + # validate where clause + if concept.metadata.where is not None: + where_value = concept.values[ConceptParts.WHERE] + if not (where_value is None or self.sheerka.value(where_value) is True): + return self.sheerka.new(BuiltinConcepts.WHERE_CLAUSE_FAILED, body=concept) + # # TODO : Validate the POST condition # diff --git a/src/core/sheerka/Services/SheerkaHistoryManager.py b/src/core/sheerka/Services/SheerkaHistoryManager.py index 2004701..5c7f268 100644 --- a/src/core/sheerka/Services/SheerkaHistoryManager.py +++ b/src/core/sheerka/Services/SheerkaHistoryManager.py @@ -19,7 +19,7 @@ class History: return msg def __repr__(self): - return f"event={self.event!r}, status={self.status}, result={self.result}" + return f"History(event={self.event!r}, status={self.status}, result={self.result})" def __eq__(self, other): if id(self) == id(other): diff --git a/src/core/sheerka/Services/SheerkaSetsManager.py b/src/core/sheerka/Services/SheerkaSetsManager.py index 092c37c..180ce61 100644 --- a/src/core/sheerka/Services/SheerkaSetsManager.py +++ b/src/core/sheerka/Services/SheerkaSetsManager.py @@ -38,6 +38,21 @@ class SheerkaSetsManager: context.log_error(logger, "Failed to add to set.", who=self.logger_name) return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0]) + def add_concepts_to_set(self, context, concepts, concept_set, logger=None): + """Adding multiple concepts at the same time""" + logger = logger or self.sheerka.log + + context.log(logger, f"Adding concepts {concepts} to set {concept_set}", who=self.logger_name) + previous = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept_set.id) + + new_ids = [c.id for c in concepts] if previous is None else previous + [c.id for c in concepts] + try: + self.sheerka.sdp.set(context.event.get_digest(), GROUP_PREFIX + concept_set.id, new_ids) + return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS)) + except Exception as error: + context.log_error(logger, "Failed to add to set.", who=self.logger_name) + return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0]) + def get_set_elements(self, concept): """ Concept is supposed to be a set diff --git a/src/core/sheerka/Sheerka.py b/src/core/sheerka/Sheerka.py index 6e61369..b70778b 100644 --- a/src/core/sheerka/Sheerka.py +++ b/src/core/sheerka/Sheerka.py @@ -22,6 +22,7 @@ import logging # BuiltinConcepts.AFTER_EVALUATION] CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser" +BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser" CONCEPTS_FILE = "_concepts.txt" @@ -52,7 +53,7 @@ class Sheerka(Concept): # cache for concept definitions, # Primarily used for unit test that does not have access to sdp - self.concepts_definition_cache = {} + self.concepts_definitions_cache = {} # # cache for concepts grammars @@ -187,7 +188,8 @@ class Sheerka(Concept): def initialize_concepts_definitions(self, execution_context): self.init_log.debug("Initializing concepts definitions") - definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) + # definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) + definitions = self.get_concepts_definitions(execution_context) if definitions is None: self.init_log.debug("No BNF defined") @@ -389,14 +391,26 @@ class Sheerka(Concept): return result or self._get_unknown(('id', concept_id)) - def get_concept_definition(self): - if self.concepts_definition_cache: - return self.concepts_definition_cache + def get_concepts_definitions(self, context): + if self.concepts_definitions_cache: + return self.concepts_definitions_cache - self.concepts_definition_cache = self.sdp.get_safe( + encoded = self.sdp.get_safe( self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False) or {} - return self.concepts_definition_cache + + self.concepts_definitions_cache = {} + bnf_parser = self.parsers[BNF_PARSER_CLASS]() + for k, v in encoded.items(): + key, id_ = core.utils.unstr_concept(k) + concept = self.new((key, id_)) + rule_result = bnf_parser.parse(context, v) + if rule_result.status: + self.concepts_definitions_cache[concept] = rule_result.value.value + else: + self.log.error(f"Failed to load bnf rule for concept {key}") + + return self.concepts_definitions_cache def new(self, concept_key, **kwargs): """ @@ -411,7 +425,7 @@ class Sheerka(Concept): else: concept_id = None - template = self.get(concept_key, concept_id) + template = self.get_by_id(concept_id) if not concept_key else self.get(concept_key, concept_id) # manage concept not found if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \ @@ -579,7 +593,10 @@ class Sheerka(Concept): self.during_restore = True with open(CONCEPTS_FILE, "r") as f: for line in f.readlines(): - self.log.info(line.strip()) + line = line.strip() + if line == "" or line.startswith("#"): + continue + self.log.info(line) self.evaluate_user_input(line) self.during_restore = False except IOError: diff --git a/src/core/tokenizer.py b/src/core/tokenizer.py index e609b25..4a8a112 100644 --- a/src/core/tokenizer.py +++ b/src/core/tokenizer.py @@ -266,10 +266,10 @@ class Tokenizer: self.column = 1 self.line += 1 elif c == "c" and self.i + 1 < self.text_len and self.text[self.i + 1] == ":": - concept_name = self.eat_concept_name(self.i + 2, self.line, self.column) - yield Token(TokenKind.CONCEPT, concept_name, self.i, self.line, self.column) - self.i += len(concept_name) + 3 - self.column += len(concept_name) + 3 + name, id, length = self.eat_concept(self.i + 2, self.line, self.column + 2) + yield Token(TokenKind.CONCEPT, (name, id), self.i, self.line, self.column) + self.i += length + 2 + self.column += length + 2 elif c.isalpha() or c == "_": identifier = self.eat_identifier(self.i) token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER @@ -297,31 +297,41 @@ class Tokenizer: yield Token(TokenKind.EOF, "", self.i, self.line, self.column) - def eat_concept_name(self, start, line, column): - result = "" + def eat_concept(self, start, line, column): + key, id, buffer = None, None, "" i = start - end_colon_found = False + processing_key = True while i < self.text_len: - c = self.text[i] + c = self.text[i] if c == "\n": - raise LexerError(f"New line is forbidden in concept name", result, i, line, column + 2 + len(result)) + raise LexerError(f"New line in concept name", self.text[start:i], i, line, column + i - start) if c == ":": - end_colon_found = True + if processing_key: + key = buffer if buffer else None + else: + id = buffer if buffer else None + i += 1 # eat the colon break - result += c + if c == "|": + key = buffer if buffer else None + buffer = "" + processing_key = False + i += 1 + continue + + buffer += c i += 1 + else: + raise LexerError(f"Missing ending colon", self.text[start:i], i, line, column + i - start) - if not end_colon_found: - raise LexerError(f"Missing ending colon", result, i, line, column + 2 + len(result)) + if (key, id) == (None, None): + raise LexerError(f"Concept identifiers not found", "", start, line, column) - if result == "": - raise LexerError(f"Concept name not found", result, start, line, column + 2 + len(result)) - - return result + return key, id, i - start def eat_whitespace(self, start): result = self.text[start] diff --git a/src/core/utils.py b/src/core/utils.py index 6a6cf3c..6554247 100644 --- a/src/core/utils.py +++ b/src/core/utils.py @@ -1,6 +1,7 @@ import importlib import inspect import pkgutil +import re from core.tokenizer import TokenKind @@ -239,43 +240,6 @@ def pp(items): return " \n" + " \n".join(str(item) for item in items) -def decode_concept(concept_repr): - """ - if concept_repr is like :c:key:id: - return the key and the id - :param concept_repr: - :return: - """ - if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith(":c:")): - return None, None - - i = 3 - length = len(concept_repr) - key = "" - while i < length: - if concept_repr[i] == ":": - break - key += concept_repr[i] - i += 1 - else: - return None, None - - i += 1 - if i >= length: - return key, None - - id = "" - while i < length: - if concept_repr[i] == ":": - break - id += concept_repr[i] - i += 1 - else: - return None, None - - return key, id - - def decode_enum(enum_repr: str): """ Tries to transform ClassName.Name into an enum @@ -300,3 +264,110 @@ def decode_enum(enum_repr: str): except TypeError: return None + + +def str_concept(t): + """ + The key,id identifiers of a concept are stored in a tuple + we want to return the key and the id, separated by a pipe + None value must be replaced by an empty string + + >>> assert str_concept(("key", "id")) == "c:key|id:" + >>> assert str_concept((None, "id")) == "c:|id:" + >>> assert str_concept(("key", None)) == "c:key:" + >>> assert str_concept((None, None)) == "" + :param t: + :return: + """ + if isinstance(t, tuple): + key, id_ = t[0], t[1] + else: + key, id_ = t.key, t.id + + if key is None and id_ is None: + return "" + + result = 'c:' if key is None else "c:" + key + if id_: + result += "|" + id_ + return result + ":" + + +def unstr_concept(concept_repr): + """ + if concept_repr is like :c:key:id: + return the key and the id + :param concept_repr: + :return: + """ + if not (concept_repr and isinstance(concept_repr, str) and concept_repr.startswith("c:")): + return None, None + + i = 2 + length = len(concept_repr) + key = "" + while i < length: + c = concept_repr[i] + if c in (":", "|"): + break + key += c + i += 1 + else: + return None, None + + if c == ":": + return key if key != "" else None, None + + i += 1 + id = "" + while i < length: + c = concept_repr[i] + if c == ":": + break + id += c + i += 1 + else: + return None, None + + return key if key != "" else None, id if id != "" else None + + +def encode_concept(t, use_concept=False): + """ + Given a tuple of concept id, concept id + Create a valid Python identifier that can be parsed back + + >>> assert encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__" + >>> assert encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__" + >>> assert encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__" + >>> assert encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__" + + :param t: + :param use_concept: + :return: + """ + + key, id_ = (t[0], t[1]) if isinstance(t, tuple) else (t.key, t.id) + prefix = "__C__USE_CONCEPT" if use_concept else "__C" + sanitized_key = "".join(c if c.isalnum() else "0" for c in key) if key else "00None00" + return prefix + f"__KEY_{sanitized_key}__ID_{id_ or '00None00'}__C__" + + +decode_regex = re.compile(r"__KEY_(\w+)__ID_(\w+)__C__") + + +def decode_concept(text): + """ + Decode what was encoded by encode_concept_key_id + :param text: + :return: + """ + use_concept = text.startswith("__C__USE_CONCEPT") + m = decode_regex.search(text) + lookup = {"00None00": None} + if m: + key = lookup.get(m.group(1), m.group(1)) + id_ = lookup.get(m.group(2), m.group(2)) + return key, id_, use_concept + + return None, None, None diff --git a/src/evaluators/PythonEvaluator.py b/src/evaluators/PythonEvaluator.py index da17fc9..64166f2 100644 --- a/src/evaluators/PythonEvaluator.py +++ b/src/evaluators/PythonEvaluator.py @@ -8,6 +8,7 @@ from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode import ast import core.ast.nodes +import core.utils class PythonEvaluator(OneReturnValueEvaluator): @@ -40,9 +41,11 @@ class PythonEvaluator(OneReturnValueEvaluator): not_for_me = context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=node) return sheerka.ret(self.name, False, not_for_me, parents=[return_value]) + # get locals my_locals = self.get_locals(context, node) context.log(self.verbose_log, f"locals={my_locals}", self.name) + # eval if isinstance(node.ast_, ast.Expression): context.log(self.verbose_log, "Evaluating using 'eval'.", self.name) compiled = compile(node.ast_, "", "eval") @@ -53,6 +56,7 @@ class PythonEvaluator(OneReturnValueEvaluator): context.log(self.verbose_log, f"{evaluated=}", self.name) return sheerka.ret(self.name, True, evaluated, parents=[return_value]) + except Exception as error: context.log_error(self.verbose_log, error, self.name) error = sheerka.new(BuiltinConcepts.ERROR, body=error) @@ -65,16 +69,19 @@ class PythonEvaluator(OneReturnValueEvaluator): "concepts": context.sheerka.dump_handler.dump_concepts, "definitions": context.sheerka.dump_handler.dump_definitions, "history": context.sheerka.dump_handler.dump_history, + "state": context.sheerka.dump_handler.dump_state, } if context.obj: context.log(self.verbose_log, - f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name) + f"Concept '{context.obj}' is in context. Adding it and its properties to locals.", self.name) for prop_name, prop_value in context.obj.props.items(): - if not isinstance(prop_value.value, Concept): - my_locals[prop_name] = prop_value.value - else: + if isinstance(prop_value.value, Concept): my_locals[prop_name] = context.sheerka.value(prop_value.value) + else: + my_locals[prop_name] = prop_value.value + + my_locals["self"] = context.obj.body node_concept = core.ast.nodes.python_to_concept(node.ast_) unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka) @@ -89,16 +96,17 @@ class PythonEvaluator(OneReturnValueEvaluator): return_concept = False else: - concept_key, concept_id, return_concept = self.resolve_name(context, name) + c_key, c_id, return_concept = self.resolve_name(name) - if concept_key in my_locals: + if c_key in my_locals: context.log(self.verbose_log, f"Using value from property.", self.name) continue - context.log(self.verbose_log, f"Instantiating new concept.", self.name) - concept = context.sheerka.new((concept_key, concept_id)) + context.log(self.verbose_log, f"Instantiating new concept with {c_key=}, {c_id=}.", self.name) + new = context.sheerka.new + concept = new((None, c_id)) if c_id else new(c_key) if context.sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT): - context.log(self.verbose_log, f"'{concept_key}' is not a concept. Skipping.", self.name) + context.log(self.verbose_log, f"({c_key=}, {c_id=}) is not a concept. Skipping.", self.name) continue context.log(self.verbose_log, f"Evaluating '{concept}'", self.name) @@ -110,62 +118,68 @@ class PythonEvaluator(OneReturnValueEvaluator): if evaluated.key == concept.key: my_locals[name] = evaluated if return_concept else context.sheerka.value(evaluated) - if self.locals: + if self.locals: # when exta values are given. Add them my_locals.update(self.locals) return my_locals - def resolve_name(self, context, to_resolve): + @staticmethod + def resolve_name(to_resolve): """ Try to match __C__concept_key__C__ or __C__concept_key__concept_id__C__ - :param context: :param to_resolve: :return: """ - if not to_resolve.startswith("__C__"): - return to_resolve, None, False - - context.log(self.verbose_log, f"Resolving name '{to_resolve}'.", self.name) - - if len(to_resolve) >= 18 and to_resolve[:18] == "__C__USE_CONCEPT__": - use_concept = True - index = 18 + key, id_, use_concept = core.utils.decode_concept(to_resolve) + if key or id_: + return key, id_, use_concept else: - use_concept = False - index = 5 - - try: - next_index = to_resolve.index("__", index) - if next_index == index: - context.log(self.verbose_log, f"Error: no key between '__'.", self.name) - return None - concept_key = to_resolve[index: next_index] - except ValueError: - context.log(self.verbose_log, f"Error: Missing trailing '__'.", self.name) - return None - - if next_index == len(to_resolve) - 5: - context.log(self.verbose_log, f"Recognized concept '{concept_key}'", self.name) - return concept_key, None, use_concept - - index = next_index + 2 - try: - next_index = to_resolve.index("__", index) - if next_index == index: - context.log(self.verbose_log, f"Error: no id between '__'.", self.name) - return None - - concept_id = to_resolve[index: next_index] - except ValueError: - context.log(self.verbose_log, f"Recognized concept '{concept_key}'.", self.name) - return concept_key, None, use_concept - - context.log(self.verbose_log, f"Recognized concept '{concept_key}' (id='{concept_id}').", self.name) - return concept_key, concept_id, use_concept + return to_resolve, None, False + # + # if not to_resolve.startswith("__C__"): + # return to_resolve, None, False + # + # context.log(self.verbose_log, f"Resolving name '{to_resolve}'.", self.name) + # + # if len(to_resolve) >= 18 and to_resolve[:18] == "__C__USE_CONCEPT__": + # use_concept = True + # index = 18 + # else: + # use_concept = False + # index = 5 + # + # try: + # next_index = to_resolve.index("__", index) + # if next_index == index: + # context.log(self.verbose_log, f"Error: no key between '__'.", self.name) + # return None + # concept_key = to_resolve[index: next_index] + # except ValueError: + # context.log(self.verbose_log, f"Error: Missing trailing '__'.", self.name) + # return None + # + # if next_index == len(to_resolve) - 5: + # context.log(self.verbose_log, f"Recognized concept '{concept_key}'", self.name) + # return concept_key, None, use_concept + # + # index = next_index + 2 + # try: + # next_index = to_resolve.index("__", index) + # if next_index == index: + # context.log(self.verbose_log, f"Error: no id between '__'.", self.name) + # return None + # + # concept_id = to_resolve[index: next_index] + # except ValueError: + # context.log(self.verbose_log, f"Recognized concept '{concept_key}'.", self.name) + # return concept_key, None, use_concept + # + # context.log(self.verbose_log, f"Recognized concept '{concept_key}' (id='{concept_id}').", self.name) + # return concept_key, concept_id, use_concept @staticmethod def expr_to_expression(expr): diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py index 4786a04..76df3d1 100644 --- a/src/parsers/BaseParser.py +++ b/src/parsers/BaseParser.py @@ -2,8 +2,9 @@ from dataclasses import dataclass from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.tokenizer import TokenKind, Keywords +from core.tokenizer import TokenKind, Keywords, Token from core.sheerka_logger import get_logger +import core.utils import logging @@ -35,8 +36,34 @@ class ErrorNode(Node): @dataclass() class UnexpectedTokenErrorNode(ErrorNode): message: str + token: Token expected_tokens: list + def __eq__(self, other): + if id(other) == id(self): + return True + + if not isinstance(other, UnexpectedTokenErrorNode): + return False + + if self.message != other.message: + return False + + if self.token.type != other.token.type or self.token.value != other.token.value: + return False + + if len(self.expected_tokens) != len(other.expected_tokens): + return False + + for i, t in enumerate(self.expected_tokens): + if t != other.expected_tokens[i]: + return False + + return True + + def __hash__(self): + return hash((self.message, self.token, self.expected_tokens)) + class BaseParser: PREFIX = "parsers." @@ -108,7 +135,7 @@ class BaseParser: switcher = { TokenKind.KEYWORD: lambda t: Keywords(t.value).value, - TokenKind.CONCEPT: lambda t: "c:" + t.value + ":", + TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), } if custom_switcher: diff --git a/src/parsers/BnfParser.py b/src/parsers/BnfParser.py index be53c74..c8a8628 100644 --- a/src/parsers/BnfParser.py +++ b/src/parsers/BnfParser.py @@ -5,7 +5,8 @@ from core.builtin_concepts import BuiltinConcepts from core.sheerka.Sheerka import ExecutionContext from core.tokenizer import Tokenizer, Token, TokenKind, LexerError from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode -from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, StrMatch +from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, \ + StrMatch, ConceptGroupExpression @dataclass() @@ -119,11 +120,11 @@ class BnfParser(BaseParser): tree = None try: self.reset_parser(context, text) - tree = self.parser_outer_rule_name() + tree = self.parse_choice() token = self.get_token() if token and token.type != TokenKind.EOF: - self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", [])) + self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [])) except LexerError as e: self.add_error(e, False) @@ -136,10 +137,11 @@ class BnfParser(BaseParser): return ret - def parser_outer_rule_name(self): - return self.parser_rule_name(self.parse_choice) - def parse_choice(self): + """ + a | b | c + :return: + """ sequence = self.parse_sequence() self.eat_white_space() @@ -159,9 +161,13 @@ class BnfParser(BaseParser): sequence = self.parse_sequence() elements.append(sequence) - return OrderedChoice(*elements) + return self.eat_rule_name_if_needed(OrderedChoice(*elements)) def parse_sequence(self): + """ + a b c + :return: + """ expr_and_modifier = self.parse_modifier() token = self.get_token() if token is None or \ @@ -185,30 +191,31 @@ class BnfParser(BaseParser): sequence = self.parse_modifier() elements.append(sequence) - return Sequence(*elements) + return self.eat_rule_name_if_needed(Sequence(*elements)) def parse_modifier(self): - expression = self.parser_inner_rule_name() + """ + a? | a* | a+ + :return: + """ + expression = self.parse_expression() token = self.get_token() if token.type == TokenKind.QMARK: self.next_token() - return Optional(expression) + return self.eat_rule_name_if_needed(Optional(expression)) if token.type == TokenKind.STAR: self.next_token() - return ZeroOrMore(expression) + return self.eat_rule_name_if_needed(ZeroOrMore(expression)) if token.type == TokenKind.PLUS: self.next_token() - return OneOrMore(expression) + return self.eat_rule_name_if_needed(OneOrMore(expression)) return expression - def parser_inner_rule_name(self): - return self.parser_rule_name(self.parse_expression) - def parse_expression(self): token = self.get_token() if token.type == TokenKind.EOF: @@ -216,15 +223,21 @@ class BnfParser(BaseParser): if token.type == TokenKind.LPAR: self.nb_open_par += 1 self.next_token() - expression = self.parse_choice() + expr = self.parse_choice() token = self.get_token() if token.type == TokenKind.RPAR: self.nb_open_par -= 1 self.next_token() - return expression + return self.eat_rule_name_if_needed(expr) else: - self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", [TokenKind.RPAR])) - return expression + self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.RPAR])) + return expr + + if token.type == TokenKind.CONCEPT: + self.next_token() + concept = self.sheerka.new((token.value[0], token.value[1])) + expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept) + return self.eat_rule_name_if_needed(expr) if token.type == TokenKind.IDENTIFIER: self.next_token() @@ -247,14 +260,15 @@ class BnfParser(BaseParser): body=("key", concept_name))) return None else: - return concept + expr = ConceptGroupExpression(concept) if self.sheerka.isaset(concept) else ConceptExpression(concept) + expr.rule_name = concept.name + return expr ret = StrMatch(core.utils.strip_quotes(token.value)) self.next_token() - return ret + return self.eat_rule_name_if_needed(ret) - def parser_rule_name(self, next_to_parse): - expression = next_to_parse() + def eat_rule_name_if_needed(self, expression): token = self.get_token() if token is None or token.type != TokenKind.EQUALS: return expression @@ -263,7 +277,8 @@ class BnfParser(BaseParser): token = self.get_token() if token is None or token.type != TokenKind.IDENTIFIER: - return self.add_error(UnexpectedTokenErrorNode(f"Unexpected token '{token}'", [TokenKind.IDENTIFIER])) + return self.add_error( + UnexpectedTokenErrorNode(f"Unexpected token '{token}'", token, [TokenKind.IDENTIFIER])) expression.rule_name = token.value self.next_token() diff --git a/src/parsers/ConceptLexerParser.py b/src/parsers/ConceptLexerParser.py index db453af..284d3fd 100644 --- a/src/parsers/ConceptLexerParser.py +++ b/src/parsers/ConceptLexerParser.py @@ -243,6 +243,9 @@ class ParsingExpression: def parse(self, parser): return self._parse(parser) + def add_rule_name_if_needed(self, text): + return text + "=" + self.rule_name if self.rule_name else text + class ConceptExpression(ParsingExpression): """ @@ -257,7 +260,7 @@ class ConceptExpression(ParsingExpression): self.concept = concept def __repr__(self): - return f"{self.concept}" + return self.add_rule_name_if_needed(f"{self.concept}") def __eq__(self, other): if not super().__eq__(other): @@ -352,7 +355,7 @@ class Sequence(ParsingExpression): def __repr__(self): to_str = ", ".join(repr(n) for n in self.elements) - return f"({to_str})" + return self.add_rule_name_if_needed(f"({to_str})") class OrderedChoice(ParsingExpression): @@ -375,7 +378,7 @@ class OrderedChoice(ParsingExpression): def __repr__(self): to_str = "| ".join(repr(n) for n in self.elements) - return f"({to_str})" + return self.add_rule_name_if_needed(f"({to_str})") class Optional(ParsingExpression): @@ -413,7 +416,7 @@ class Optional(ParsingExpression): return f"{self.elements[0]}?" else: to_str = ", ".join(repr(n) for n in self.elements) - return f"({to_str})?" + return self.add_rule_name_if_needed(f"({to_str})?") class Repetition(ParsingExpression): @@ -467,7 +470,7 @@ class ZeroOrMore(Repetition): def __repr__(self): to_str = ", ".join(repr(n) for n in self.elements) - return f"({to_str})*" + return self.add_rule_name_if_needed(f"({to_str})*") class OneOrMore(Repetition): @@ -507,7 +510,7 @@ class OneOrMore(Repetition): def __repr__(self): to_str = ", ".join(repr(n) for n in self.elements) - return f"({to_str})+" + return self.add_rule_name_if_needed(f"({to_str})+") class UnorderedGroup(Repetition): @@ -541,13 +544,13 @@ class StrMatch(Match): Matches a literal """ - def __init__(self, to_match, rule_name="", root=False, ignore_case=True): - super(Match, self).__init__(rule_name=rule_name, root=root) + def __init__(self, to_match, rule_name="", ignore_case=True): + super(Match, self).__init__(rule_name=rule_name) self.to_match = to_match self.ignore_case = ignore_case def __repr__(self): - return f"'{self.to_match}'" + return self.add_rule_name_if_needed(f"'{self.to_match}'") def __eq__(self, other): if not super().__eq__(other): @@ -699,10 +702,14 @@ class ConceptLexerParser(BaseParser): else: ret = ConceptExpression(expression, rule_name=expression.name) concepts_to_resolve.add(expression) - elif isinstance(expression, ConceptExpression): + elif isinstance(expression, ConceptExpression): # it includes ConceptGroupExpression if expression.rule_name is None or expression.rule_name == "": expression.rule_name = expression.concept.name if isinstance(expression.concept, Concept) \ else expression.concept + if isinstance(expression.concept, str): + concept = self.get_concept(expression.concept) + if self.sheerka.is_known(concept): + expression.concept = concept concepts_to_resolve.add(expression.concept) ret = expression elif isinstance(expression, str): @@ -955,6 +962,47 @@ class ConceptLexerParser(BaseParser): return concept + def encode_grammar(self, grammar): + """ + Transform the grammar into something that can easily can be serialized + :param grammar: + :return: + """ + + def _encode(expression): + if isinstance(expression, StrMatch): + res = f"'{expression.to_match}'" + + elif isinstance(expression, ConceptExpression): + res = core.utils.str_concept(expression.concept) + + elif isinstance(expression, Sequence): + res = "(" + " ".join(_encode(c) for c in expression.nodes) + ")" + + elif isinstance(expression, OrderedChoice): + res = "(" + "|".join(_encode(c) for c in expression.nodes) + ")" + + elif isinstance(expression, Optional): + res = _encode(expression.nodes[0]) + "?" + + elif isinstance(expression, ZeroOrMore): + res = _encode(expression.nodes[0]) + "*" + + elif isinstance(expression, OneOrMore): + res = _encode(expression.nodes[0]) + "+" + + if expression.rule_name: + res += "=" + expression.rule_name + + return res + + result = {} + for k, v in grammar.items(): + key = core.utils.str_concept(k) + value = _encode(v) + result[key] = value + return result + @staticmethod def get_bests(results): """ diff --git a/src/parsers/PythonParser.py b/src/parsers/PythonParser.py index e667f55..713af12 100644 --- a/src/parsers/PythonParser.py +++ b/src/parsers/PythonParser.py @@ -1,9 +1,10 @@ from core.builtin_concepts import BuiltinConcepts from core.tokenizer import Tokenizer, LexerError, TokenKind from parsers.BaseParser import BaseParser, Node, ErrorNode -from dataclasses import dataclass, field +from dataclasses import dataclass import ast import logging +import core.utils from parsers.ConceptLexerParser import ConceptNode @@ -71,7 +72,7 @@ class PythonParser(BaseParser): tree = None python_switcher = { - TokenKind.CONCEPT: lambda t: f"__C__USE_CONCEPT__{t.value}__C__" + TokenKind.CONCEPT: lambda t: core.utils.encode_concept(t.value, True) } try: @@ -136,6 +137,7 @@ class PythonGetNamesVisitor(ast.NodeVisitor): def visit_Name(self, node): self.names.add(node.id) + class LexerNodeParserHelperForPython: """Helper class to parse mix of concepts and Python""" diff --git a/src/sdp/sheerkaSerializer.py b/src/sdp/sheerkaSerializer.py index b2ce8e9..2ceddfc 100644 --- a/src/sdp/sheerkaSerializer.py +++ b/src/sdp/sheerkaSerializer.py @@ -223,7 +223,7 @@ class PickleSerializer(BaseSerializer): class StateSerializer(PickleSerializer): - def __init__(self, ): + def __init__(self): PickleSerializer.__init__( self, lambda obj: core.utils.get_full_qualified_name(obj) == "sdp.sheerkaDataProvider.State", @@ -239,13 +239,23 @@ class ConceptSerializer(JsonSerializer): return isinstance(obj, Concept) -class DictionarySerializer(PickleSerializer): - def __init__(self, ): - PickleSerializer.__init__( - self, - lambda obj: isinstance(obj, dict), - "D", - 1) +class DictionarySerializer(BaseSerializer): + def __init__(self): + super().__init__("D", 1) + + def matches(self, obj): + return isinstance(obj, dict) + + def dump(self, stream, obj, context): + stream.write(json.dumps(obj, default=json_default_converter).encode("utf-8")) + stream.seek(0) + return stream + + def load(self, stream, context): + json_stream = stream.read().decode("utf-8") + obj = json.loads(json_stream) + + return obj class ExecutionContextSerializer(BaseSerializer): diff --git a/src/sheerkapickle/SheerkaPickler.py b/src/sheerkapickle/SheerkaPickler.py index 15bcdea..75e7248 100644 --- a/src/sheerkapickle/SheerkaPickler.py +++ b/src/sheerkapickle/SheerkaPickler.py @@ -70,7 +70,7 @@ class SheerkaPickler: elif utils.is_enum(k): k_str = core.utils.get_full_qualified_name(k) + "." + k.name elif isinstance(k, Concept): - k_str = f":c:{k.key}:{k.id}:" + k_str = core.utils.str_concept(k) else: k_str = k diff --git a/src/sheerkapickle/SheerkaUnpickler.py b/src/sheerkapickle/SheerkaUnpickler.py index ab7f9e4..1bdb06a 100644 --- a/src/sheerkapickle/SheerkaUnpickler.py +++ b/src/sheerkapickle/SheerkaUnpickler.py @@ -90,7 +90,7 @@ class SheerkaUnpickler: if key == "null": return None - concept_key, concept_id = core.utils.decode_concept(key) + concept_key, concept_id = core.utils.unstr_concept(key) if concept_key is not None: return self.sheerka.new((concept_key, concept_id)) if concept_id else self.sheerka.new(concept_key) diff --git a/tests/_concepts.txt b/tests/_concepts.txt deleted file mode 100644 index d0aa449..0000000 --- a/tests/_concepts.txt +++ /dev/null @@ -1,15 +0,0 @@ -def concept one as 1 -def concept two as 2 -def concept three as 3 -def concept four as 4 -def concept five as 5 -def concept one as 1 -def concept two as 2 -def concept three as 3 -def concept four as 4 -def concept five as 5 -def concept one as 1 -def concept two as 2 -def concept three as 3 -def concept four as 4 -def concept five as 5 diff --git a/tests/core/_concepts.txt b/tests/core/_concepts.txt index f1b1305..072f8fe 100644 --- a/tests/core/_concepts.txt +++ b/tests/core/_concepts.txt @@ -8,3 +8,13 @@ def concept two as 2 def concept three as 3 def concept four as 4 def concept five as 5 +def concept one as 1 +def concept two as 2 +def concept three as 3 +def concept four as 4 +def concept five as 5 +def concept one as 1 +def concept two as 2 +def concept three as 3 +def concept four as 4 +def concept five as 5 diff --git a/tests/core/test_SheerkaEvaluateConcept.py b/tests/core/test_SheerkaEvaluateConcept.py index 5120f1a..b31185e 100644 --- a/tests/core/test_SheerkaEvaluateConcept.py +++ b/tests/core/test_SheerkaEvaluateConcept.py @@ -18,7 +18,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): ("True", True), ("1 > 2", False), ]) - def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected): + def test_i_can_evaluate_a_concept_with_simple_body(self, body, expected): sheerka = self.get_sheerka() concept = Concept("foo", body=body) @@ -43,9 +43,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): ("True", True), ("1 > 2", False), ]) - def test_i_can_evaluate_the_other_metadata(self,expr, expected): + def test_i_can_evaluate_the_other_metadata(self, expr, expected): """ - I only test WHERE, it's the same for the others + I only test PRE, it's the same for the others :param expr: :param expected: :return: @@ -53,15 +53,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): sheerka = self.get_sheerka() - concept = Concept("foo", where=expr) + concept = Concept("foo", pre=expr) evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) assert evaluated.key == concept.key assert evaluated.metadata.body is None - assert evaluated.metadata.pre is None + assert evaluated.metadata.pre == expr assert evaluated.metadata.post is None - assert evaluated.metadata.where == expr - assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected + assert evaluated.metadata.where is None + assert evaluated.get_metadata_value(ConceptParts.PRE) == expected assert evaluated.props == {} assert evaluated.metadata.is_evaluated assert len(evaluated.values) == 0 if expr is None else 1 @@ -330,3 +330,42 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka): assert evaluated.key == concept.init_key().key + @pytest.mark.parametrize("where_clause, expected", [ + ("True", True), + ("False", False), + ("self < 10", False), + ("self < 11", True), + ("a < 20", False), + ("a > 19", True), + ("a + self > 20", True), + ]) + def test_i_can_evaluate_simple_where(self, where_clause, expected): + sheerka = self.get_sheerka() + + concept = Concept("foo", body="10", where=where_clause).def_prop("a", "20") + sheerka.add_in_cache(concept) + + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + + if expected: + assert evaluated.key == concept.key + else: + assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED) + assert evaluated.body == concept + + def test_i_can_evaluate_where_when_using_other_concept(self): + sheerka = self.get_sheerka() + + foo_true = Concept("foo_true", body="True").init_key() + foo_false = Concept("foo_false", body="False").init_key() + sheerka.add_in_cache(foo_false) + sheerka.add_in_cache(foo_true) + + concept = Concept("foo", where="foo_true").init_key() + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert evaluated.key == concept.key + + concept = Concept("foo", where="foo_false") + evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept) + assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED) + assert evaluated.body == concept diff --git a/tests/core/test_SheerkaSetsManager.py b/tests/core/test_SheerkaSetsManager.py index 158d98e..61e9663 100644 --- a/tests/core/test_SheerkaSetsManager.py +++ b/tests/core/test_SheerkaSetsManager.py @@ -1,8 +1,3 @@ -import os -import shutil -from os import path - -import pytest from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts from core.concept import Concept @@ -11,42 +6,36 @@ from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka class TestSheerkaSetsManager(TestUsingFileBasedSheerka): - - - def test_i_can_add_concept_to_set(self): - sheerka = self.get_sheerka(False, False) - - foo = Concept("foo") - sheerka.set_id_if_needed(foo, False) - - all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) + def init(self, use_dict, *concepts): + sheerka = self.get_sheerka(use_dict, True) + for c in concepts: + sheerka.set_id_if_needed(c, False) + sheerka.add_in_cache(c) context = self.get_context(sheerka) + return sheerka, context + + def test_i_can_add_concept_to_set(self): + foo = Concept("foo") + all_foos = Concept("all_foos") + sheerka, context = self.init(False, foo, all_foos) + res = sheerka.add_concept_to_set(context, foo, all_foos) assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) - all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) + all_entries = self.get_sheerka(False, True).sdp.get("All_" + all_foos.id, None, False) assert len(all_entries) == 1 assert foo.id in all_entries def test_i_can_add_several_concepts_to_set(self): - sheerka = self.get_sheerka(False, False) - foo1 = Concept("foo1") - sheerka.set_id_if_needed(foo1, False) - - foo2 = Concept("foo1") - sheerka.set_id_if_needed(foo2, False) - + foo2 = Concept("foo2") all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) + sheerka, context = self.init(False, foo1, foo2, all_foos) - context = self.get_context(sheerka) - sheerka.add_concept_to_set(context, foo1, all_foos) - res = sheerka.add_concept_to_set(context, foo2, all_foos) + res = sheerka.sets_handler.add_concepts_to_set(context, (foo1, foo2), all_foos) assert res.status assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) @@ -56,16 +45,30 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka): assert foo1.id in all_entries assert foo2.id in all_entries + # I can add another elements + foo3 = Concept("foo3") + foo4 = Concept("foo4") + for c in [foo3, foo4]: + sheerka.set_id_if_needed(c, False) + sheerka.add_in_cache(c) + + res = sheerka.sets_handler.add_concepts_to_set(context, (foo3, foo4), all_foos) + + assert res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS) + + all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False) + assert len(all_entries) == 4 + assert foo1.id in all_entries + assert foo2.id in all_entries + assert foo3.id in all_entries + assert foo4.id in all_entries + def test_i_cannot_add_the_same_concept_twice_in_a_set(self): - sheerka = self.get_sheerka() - foo = Concept("foo") - sheerka.set_id_if_needed(foo, False) - all_foos = Concept("all_foos") - sheerka.set_id_if_needed(all_foos, False) + sheerka, context = self.init(True, foo, all_foos) - context = self.get_context(sheerka) sheerka.add_concept_to_set(context, foo, all_foos) res = sheerka.add_concept_to_set(context, foo, all_foos) @@ -77,18 +80,12 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka): assert foo.id in all_entries def test_i_get_elements_from_a_set(self): - sheerka = self.get_sheerka() - one = Concept("one") two = Concept("two") three = Concept("three") number = Concept("number") + sheerka, context = self.init(True, one, two, three, number) - for c in [one, two, three, number]: - sheerka.set_id_if_needed(c, False) - sheerka.add_in_cache(c) - - context = self.get_context(sheerka) for c in [one, two, three]: sheerka.add_concept_to_set(context, c, number) @@ -97,10 +94,8 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka): assert set(elements) == {one, two, three} def test_i_cannot_get_elements_if_not_a_set(self): - sheerka = self.get_sheerka() one = Concept("one") - sheerka.set_id_if_needed(one, False) - sheerka.add_in_cache(one) + sheerka, context = self.init(True, one) error = sheerka.get_set_elements(one) @@ -108,17 +103,17 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka): assert error.body == one def test_isa_and_isa_group(self): - sheerka = self.get_sheerka() + group = Concept("group") + foo = Concept("foo") + sheerka, context = self.init(True, group, foo) - group = Concept("group").init_key() - group.metadata.id = "1001" assert not sheerka.isaset(group) - - foo = Concept("foo").init_key() - foo.metadata.id = "1002" assert not sheerka.isa(foo, group) context = self.get_context(sheerka) sheerka.add_concept_to_set(context, foo, group) assert sheerka.isaset(group) assert sheerka.isa(foo, group) + + def test_i_can_a_multiples_concepts(self): + pass diff --git a/tests/core/test_tokenizer.py b/tests/core/test_tokenizer.py index 81c2323..6e376a4 100644 --- a/tests/core/test_tokenizer.py +++ b/tests/core/test_tokenizer.py @@ -40,7 +40,7 @@ def test_i_can_tokenize(): assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20) assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21) assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22) - assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23) + assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 23) assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30) assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31) assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32) @@ -79,8 +79,8 @@ def test_i_can_tokenize_identifiers(text, expected): ('"string', "Missing Trailing quote", '"string', 7, 1, 8), ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14), ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8), - ("c::", "Concept name not found", "", 2, 1, 3), - ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6), + ("c::", "Concept identifiers not found", "", 2, 1, 3), + ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6), ("c:foo", "Missing ending colon", "foo", 5, 1, 6) ]) def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column): @@ -139,3 +139,17 @@ def test_i_can_recognize_keywords(text, expected): tokens = list(Tokenizer(text)) assert tokens[0].type == TokenKind.KEYWORD assert tokens[0].value == expected + + +@pytest.mark.parametrize("text, expected", [ + ("c:key:", ("key", None)), + ("c:key|id:", ("key", "id")), + ("c:key|:", ("key", None)), + ("c:|id:", (None, "id")), + ("c:125:", ("125", None)), +]) +def test_i_can_parse_concept_token(text, expected): + tokens = list(Tokenizer(text)) + + assert tokens[0].type == TokenKind.CONCEPT + assert tokens[0].value == expected diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index c303000..5b73082 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -1,10 +1,25 @@ import core.utils import pytest -from core.concept import ConceptParts +from core.concept import ConceptParts, Concept from core.tokenizer import Token, TokenKind +def get_tokens(lst): + res = [] + for e in lst: + if e == " ": + res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0)) + elif e == "\n": + res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0)) + elif e == "": + res.append(Token(TokenKind.EOF, "\n", 0, 0, 0)) + else: + res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0)) + + return res + + @pytest.mark.parametrize("lst, as_string", [ (None, "",), ([], ""), @@ -136,18 +151,33 @@ def test_i_can_escape(): (10, None, None), ("", None, None), ("xxx", None, None), - (":c:", None, None), - (":c:key", None, None), - (":c:key:", "key", None), - (":c:key:id", None, None), - (":c:key:id:", "key", "id"), + ("c:", None, None), + ("c:key", None, None), + ("c:key:", "key", None), + ("c:key|id", None, None), + ("c:key|id:", "key", "id"), + ("c:|id:", None, "id"), + ("c:key|:", "key", None), ]) -def test_i_can_decode_concept_repr(text, expected_key, expected_id): - k, i = core.utils.decode_concept(text) +def test_i_can_unstr_concept(text, expected_key, expected_id): + k, i = core.utils.unstr_concept(text) assert k == expected_key assert i == expected_id +def test_i_can_str_concept(): + assert core.utils.str_concept(("key", "id")) == "c:key|id:" + assert core.utils.str_concept((None, "id")) == "c:|id:" + assert core.utils.str_concept(("key", None)) == "c:key:" + assert core.utils.str_concept((None, None)) == "" + + concept = Concept("foo").init_key() + assert core.utils.str_concept(concept) == "c:foo:" + + concept.metadata.id = "1001" + assert core.utils.str_concept(concept) == "c:foo|1001:" + + @pytest.mark.parametrize("text, expected", [ (None, None), (10, None), @@ -162,16 +192,22 @@ def test_i_can_decode_enum(text, expected): assert actual == expected -def get_tokens(lst): - res = [] - for e in lst: - if e == " ": - res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0)) - elif e == "\n": - res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0)) - elif e == "": - res.append(Token(TokenKind.EOF, "\n", 0, 0, 0)) - else: - res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0)) +def test_encode_concept_key_id(): + assert core.utils.encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__" + assert core.utils.encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__" + assert core.utils.encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__" + assert core.utils.encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__" + assert core.utils.encode_concept(("k + y", "id")) == "__C__KEY_k000y__ID_id__C__" - return res + concept = Concept("foo").init_key() + assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__" + + concept.metadata.id = "1001" + assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__" + + +def test_decode_concept_key_id(): + assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id", False) + assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id", False) + assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None, False) + assert core.utils.decode_concept("__C__USE_CONCEPT__KEY_key__ID_id__C__") == ("key", "id", True) diff --git a/tests/evaluators/test_ConceptEvaluator.py b/tests/evaluators/test_ConceptEvaluator.py index b0bd27f..4589ce7 100644 --- a/tests/evaluators/test_ConceptEvaluator.py +++ b/tests/evaluators/test_ConceptEvaluator.py @@ -21,7 +21,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): def test_i_can_evaluate_concept(self): context = self.get_context() concept = Concept(name="foo", - where="1", + where="True", pre="2", post="3").def_prop("a", "4").def_prop("b", "5") @@ -32,7 +32,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): assert result.who == evaluator.name assert result.status assert result.value.name == "foo" - assert result.value.get_metadata_value(ConceptParts.WHERE) == 1 + assert result.value.get_metadata_value(ConceptParts.WHERE) == True assert result.value.get_metadata_value(ConceptParts.PRE) == 2 assert result.value.get_metadata_value(ConceptParts.POST) == 3 assert result.value.get_prop("a") == 4 @@ -44,7 +44,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() concept = Concept(name="foo", body="'I have a value'", - where="1", + where="True", pre="2", post="3").set_prop("a", "4").set_prop("b", "5") @@ -61,7 +61,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka): context = self.get_context() concept = Concept(name="foo", body="'I have a value'", - where="1", + where="True", pre="2", post="3").set_prop("a", "4").set_prop("b", "5") diff --git a/tests/evaluators/test_PythonEvaluator.py b/tests/evaluators/test_PythonEvaluator.py index b1bc306..0bec46b 100644 --- a/tests/evaluators/test_PythonEvaluator.py +++ b/tests/evaluators/test_PythonEvaluator.py @@ -116,23 +116,23 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka): assert not evaluated.status assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'" - @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [ - ("__C__key__C__", "key", None, False), - ("__C__key__id__C__", "key", "id", False), - ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), - ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), - ]) - def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept): - context = self.get_context() - assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept) - - @pytest.mark.parametrize("text", [ - "__C__", - "__C__key", - "__C__key____", - "__C____", - "__C__USE_CONCEPT__", - ]) - def test_i_cannot_resolve_name(self, text): - context = self.get_context() - assert PythonEvaluator().resolve_name(context, text) is None + # @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [ + # ("__C__key__C__", "key", None, False), + # ("__C__key__id__C__", "key", "id", False), + # ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), + # ("__C__USE_CONCEPT__key__id__C__", "key", "id", True), + # ]) + # def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept): + # context = self.get_context() + # assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept) + # + # @pytest.mark.parametrize("text", [ + # "__C__", + # "__C__key", + # "__C__key____", + # "__C____", + # "__C__USE_CONCEPT__", + # ]) + # def test_i_cannot_resolve_name(self, text): + # context = self.get_context() + # assert PythonEvaluator().resolve_name(context, text) is None diff --git a/tests/non_reg/test_sheerka_non_reg.py b/tests/non_reg/test_sheerka_non_reg.py index c8e3471..7cde5e3 100644 --- a/tests/non_reg/test_sheerka_non_reg.py +++ b/tests/non_reg/test_sheerka_non_reg.py @@ -273,8 +273,9 @@ as: saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY) expected_bnf = Sequence( - a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus")))) - assert saved_definitions[saved_concept] == expected_bnf + ConceptExpression(a, rule_name="a"), + Optional(Sequence(StrMatch("plus"), ConceptExpression(saved_concept, rule_name="plus")))) + assert saved_definitions["c:plus|1001:"] == "(c:a:=a ('plus' c:plus|1001:=plus)?)" new_concept = res[0].value.body assert new_concept.metadata.name == "plus" @@ -456,6 +457,26 @@ as: assert res[0].status assert res[0].body == 23 + def test_i_can_mix_bnf_and_isa_when_concept_other_case(self): + sheerka = self.get_sheerka() + + init = [ + "def concept one as 1", + "def concept twenty as 20", + "def concept number", + "one isa number", + "twenty isa number", + "def concept twenties from bnf twenty number as twenty + number" + ] + + for exp in init: + sheerka.evaluate_user_input(exp) + + res = sheerka.evaluate_user_input("twenty one") + assert len(res) == 1 + assert res[0].status + assert res[0].body == simplec("twenties", 21) + def test_i_can_mix_concept_of_concept(self): sheerka = self.get_sheerka() @@ -623,3 +644,30 @@ as: assert len(res) == 1 assert res[0].status assert res[0].body == 21 + + def test_i_can_use_where_in_bnf(self): + sheerka = self.get_sheerka() + + init = [ + "def concept one as 1", + "def concept two as 2", + "def concept three as 3", + "def concept twenty as 20", + "def concept number", + "one isa number", + "two isa number", + "three isa number", + "def concept twenties from bnf twenty number where number <= 2 as twenty + number" + ] + + for exp in init: + sheerka.evaluate_user_input(exp) + + res = sheerka.evaluate_user_input("eval twenty one") + assert len(res) == 1 and res[0].status and res[0].body == 21 + + res = sheerka.evaluate_user_input("eval twenty two") + assert len(res) == 1 and res[0].status and res[0].body == 22 + + res = sheerka.evaluate_user_input("eval twenty three") + assert len(res) > 1 diff --git a/tests/parsers/test_BaseParser.py b/tests/parsers/test_BaseParser.py index 31cf9bf..bb9b969 100644 --- a/tests/parsers/test_BaseParser.py +++ b/tests/parsers/test_BaseParser.py @@ -18,7 +18,7 @@ def test_i_can_get_text_from_tokens(text, expected_text): @pytest.mark.parametrize("text, custom, expected_text", [ - ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)") + ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)") ]) def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text): tokens = list(Tokenizer(text)) diff --git a/tests/parsers/test_BnfParser.py b/tests/parsers/test_BnfParser.py index 90163da..f6a9aa7 100644 --- a/tests/parsers/test_BnfParser.py +++ b/tests/parsers/test_BnfParser.py @@ -2,7 +2,7 @@ import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept -from core.tokenizer import Tokenizer, TokenKind, LexerError +from core.tokenizer import Tokenizer, TokenKind, LexerError, Token from parsers.BaseParser import UnexpectedTokenErrorNode from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \ @@ -16,6 +16,14 @@ class ClassWithName: self.name = name +def c(name): + concept = Concept(name).init_key() + return ConceptExpression(concept, rule_name=name) + + +eof_token = Token(TokenKind.EOF, "", 0, 0, 0) + + class TestBnfParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("expression, expected", [ @@ -33,9 +41,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): ("1 2 | 3 4+", OrderedChoice( Sequence(StrMatch("1"), StrMatch("2")), Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))), - ( - "1 (2 | 3) 4+", - Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))), + ("1 (2 | 3) 4+", Sequence( + StrMatch("1"), + OrderedChoice(StrMatch("2"), StrMatch("3")), + OneOrMore(StrMatch("4")))), ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))), ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))), ("1 *", Sequence(StrMatch("1"), StrMatch("*"))), @@ -61,6 +70,13 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")), ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")), ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))), + ("(1=a 2=b)=c", Sequence(StrMatch("1", rule_name="a"), StrMatch("2", rule_name="b"), rule_name="c")), + ("(1*=a)", ZeroOrMore(StrMatch("1"), rule_name="a")), + ("'a'* 'b'+", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))), + ("('a'* 'b'+)", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))), + ("('a'*=x 'b'+=y)=z", Sequence( + ZeroOrMore(StrMatch("a"), rule_name="x"), + OneOrMore(StrMatch("b"), rule_name="y"), rule_name="z")), ]) def test_i_can_parse_regex(self, expression, expected): parser = BnfParser() @@ -72,12 +88,12 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): assert res.value.source == expression @pytest.mark.parametrize("expression, expected", [ - ("foo", Concept("foo").init_key()), - ("foo*", ZeroOrMore(Concept("foo").init_key())), - ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))), - ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))), - ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), - ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())), + ("foo", c("foo")), + ("foo*", ZeroOrMore(c("foo"))), + ("foo 'and' bar+", Sequence(c("foo"), StrMatch("and"), OneOrMore(c("bar")))), + ("foo | bar?", OrderedChoice(c("foo"), Optional(c("bar")))), + ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), c("var"))), + ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), c("var"))), ]) def test_i_can_parse_regex_with_concept(self, expression, expected): foo = Concept("foo") @@ -113,8 +129,8 @@ class TestBnfParser(TestUsingMemoryBasedSheerka): @pytest.mark.parametrize("expression, error", [ ("1 ", UnexpectedEndOfFileError()), ("1|", UnexpectedEndOfFileError()), - ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.RPAR])), - ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token()'", [TokenKind.IDENTIFIER])), + ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token()'", eof_token, [TokenKind.RPAR])), + ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token()'", eof_token, [TokenKind.IDENTIFIER])), ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6)) ]) def test_i_can_detect_errors(self, expression, error): diff --git a/tests/parsers/test_ConceptLexerParser.py b/tests/parsers/test_ConceptLexerParser.py index 9723d12..a393cf7 100644 --- a/tests/parsers/test_ConceptLexerParser.py +++ b/tests/parsers/test_ConceptLexerParser.py @@ -1,10 +1,13 @@ +from ast import Str + import pytest from core.builtin_concepts import BuiltinConcepts from core.concept import Concept, ConceptParts, DoNotResolve from core.tokenizer import Tokenizer, TokenKind, Token +from parsers.BnfParser import BnfParser from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \ ParsingExpressionVisitor, TerminalNode, NonTerminalNode, ZeroOrMore, OneOrMore, \ - UnrecognizedTokensNode, cnode, short_cnode + UnrecognizedTokensNode, cnode, short_cnode, ConceptExpression, ConceptGroupExpression from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -75,6 +78,7 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka): context = self.get_context() for c in concepts: context.sheerka.add_in_cache(c) + context.sheerka.set_id_if_needed(c, False) parser = ConceptLexerParser() parser.initialize(context, grammar) @@ -586,6 +590,32 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka): assert res.status assert res.value.body == [cnode("foo", 0, 2, "twenty one")] + def test_i_can_initialize_when_cyclic_reference(self): + foo = Concept(name="foo") + grammar = {foo: Optional("one", ConceptExpression("foo"))} + context, parser = self.init([foo], grammar) + + assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo")) + + def test_i_cannot_initialize_when_cyclic_reference_when_concept_is_under_construction_and_not_known(self): + foo = Concept(name="foo").init_key() + grammar = {foo: Optional("one", ConceptExpression("foo"))} + + context = self.get_context() + parser = ConceptLexerParser() + parser.initialize(context, grammar) + assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression("foo", rule_name="foo")) + + def test_i_can_initialize_when_cyclic_reference_when_concept_is_under_construction_and_known(self): + foo = Concept(name="foo").init_key() + grammar = {foo: Optional("one", ConceptExpression("foo"))} + + context = self.get_context() + context.concepts["foo"] = foo + parser = ConceptLexerParser() + parser.initialize(context, grammar) + assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo")) + def test_i_can_parse_concept_reference_that_is_group(self): """ if one is number, then number is a 'group' @@ -1092,6 +1122,109 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka): assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok") assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok") + @pytest.mark.parametrize("rule, expected", [ + (StrMatch("string"), "'string'"), + (StrMatch("string", rule_name="rule_name"), "'string'=rule_name"), + (Sequence(StrMatch("foo"), StrMatch("bar")), "('foo' 'bar')"), + (Sequence(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo' 'bar')=rule_name"), + (OrderedChoice(StrMatch("foo"), StrMatch("bar")), "('foo'|'bar')"), + (OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo'|'bar')=rule_name"), + (Optional(StrMatch("foo")), "'foo'?"), + (Optional(StrMatch("foo"), rule_name="rule_name"), "'foo'?=rule_name"), + (ZeroOrMore(StrMatch("foo")), "'foo'*"), + (ZeroOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'*=rule_name"), + (OneOrMore(StrMatch("foo")), "'foo'+"), + (OneOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'+=rule_name"), + (Sequence( + Optional(StrMatch("foo"), rule_name="a"), + ZeroOrMore(StrMatch("bar"), rule_name="b"), + OneOrMore(StrMatch("baz"), rule_name="c"), + rule_name="d"), "('foo'?=a 'bar'*=b 'baz'+=c)=d"), + (OrderedChoice( + Optional(StrMatch("foo"), rule_name="a"), + ZeroOrMore(StrMatch("bar"), rule_name="b"), + OneOrMore(StrMatch("baz"), rule_name="c"), + rule_name="d"), "('foo'?=a|'bar'*=b|'baz'+=c)=d"), + (Sequence( + OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="a"), + OrderedChoice(StrMatch("x"), StrMatch("y"), rule_name="b"), + rule_name="c"), "(('foo'|'bar')=a ('x'|'y')=b)=c") + ]) + def test_i_can_encode_grammar(self, rule, expected): + foo = Concept(name="foo") + grammar = {foo: rule} + context, parser = self.init([foo], grammar) + + encoded = parser.encode_grammar(parser.concepts_grammars) + assert encoded["c:foo|1001:"] == expected + + bnf_parser = BnfParser() + parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"]) + assert parse_res.status + assert parse_res.value.value == rule + + def test_i_can_encode_grammar_when_concept_simple(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + grammar = {foo: ConceptExpression(bar)} + context, parser = self.init([foo, bar], grammar) + + encoded = parser.encode_grammar(parser.concepts_grammars) + assert encoded["c:foo|1001:"] == "c:bar|1002:=bar" + + bnf_parser = BnfParser() + parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"]) + assert parse_res.status + assert parse_res.value.value == grammar[foo] + + def test_i_can_encode_grammar_when_concepts(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + baz = Concept(name="baz") + grammar = {foo: Sequence( + StrMatch("a"), + OrderedChoice(ConceptExpression(bar), + OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")} + context, parser = self.init([foo, bar, baz], grammar) + + encoded = parser.encode_grammar(parser.concepts_grammars) + assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s" + + bnf_parser = BnfParser() + parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"]) + assert parse_res.status + assert parse_res.value.value == grammar[foo] + + def test_i_can_encode_grammar_when_set_concepts(self): + foo = Concept(name="foo") + bar = Concept(name="bar") + baz = Concept(name="baz") + grammar = {foo: Sequence( + StrMatch("a"), + OrderedChoice(bar, + OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")} + context = self.get_context() + for c in [foo, bar, baz]: + context.sheerka.add_in_cache(c) + context.sheerka.set_id_if_needed(c, False) + context.sheerka.add_concept_to_set(context, baz, bar) + + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + encoded = parser.encode_grammar(parser.concepts_grammars) + assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s" + + bnf_parser = BnfParser() + parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"]) + assert parse_res.status + + expected = Sequence( + StrMatch("a"), + OrderedChoice(ConceptGroupExpression(bar, rule_name="bar"), + OneOrMore(ConceptExpression(baz, rule_name="baz")), rule_name="oc"), rule_name="s") + assert parse_res.value.value == expected + # # def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(self): # context = self.get_context() diff --git a/tests/parsers/test_DefaultParser.py b/tests/parsers/test_DefaultParser.py index c23f942..bb9b99c 100644 --- a/tests/parsers/test_DefaultParser.py +++ b/tests/parsers/test_DefaultParser.py @@ -3,7 +3,7 @@ import ast from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept from core.concept import Concept -from parsers.ConceptLexerParser import OrderedChoice, StrMatch +from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression from parsers.PythonParser import PythonParser, PythonNode from core.tokenizer import Keywords, Tokenizer, LexerError from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode @@ -246,7 +246,7 @@ def concept add one to a as parser = DefaultParser() res = parser.parse(context, text) node = res.value.value - definition = OrderedChoice(a_concept, StrMatch("a_string")) + definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string")) parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition) expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result) @@ -321,7 +321,7 @@ def concept add one to a as ("def concept 'name", "Missing Trailing quote", "'name"), ("def concept name as 'body", "Missing Trailing quote", "'body"), ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"), - ("def concept c::", "Concept name not found", ""), + ("def concept c::", "Concept identifiers not found", ""), ]) def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text): parser = DefaultParser() diff --git a/tests/parsers/test_PythonParser.py b/tests/parsers/test_PythonParser.py index 226c1c5..945c820 100644 --- a/tests/parsers/test_PythonParser.py +++ b/tests/parsers/test_PythonParser.py @@ -3,6 +3,7 @@ import pytest from core.builtin_concepts import ParserResultConcept from core.tokenizer import Tokenizer, LexerError from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode +import core.utils from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka @@ -52,8 +53,8 @@ class TestPythonParser(TestUsingMemoryBasedSheerka): assert isinstance(res.value.value[0].exception, SyntaxError) @pytest.mark.parametrize("text, error_msg, error_text", [ - ("c::", "Concept name not found", ""), - ("c:: + 1", "Concept name not found", ""), + ("c::", "Concept identifiers not found", ""), + ("c:: + 1", "Concept identifiers not found", ""), ]) def test_i_can_detect_lexer_errors(self, text, error_msg, error_text): parser = PythonParser() @@ -66,12 +67,12 @@ class TestPythonParser(TestUsingMemoryBasedSheerka): assert res.body.body[0].text == error_text def test_i_can_parse_a_concept(self): - text = "c:concept_name: + 1" + text = "c:name|key: + 1" parser = PythonParser() res = parser.parse(self.get_context(), text) assert res assert res.value.value == PythonNode( - "c:concept_name: + 1", - ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval")) + "c:name|key: + 1", + ast.parse(core.utils.encode_concept(("name", "key"), True) + "+1", mode="eval")) diff --git a/tests/sheerkapickle/test_SheerkaPickler.py b/tests/sheerkapickle/test_SheerkaPickler.py index 09e9e09..c7dff5a 100644 --- a/tests/sheerkapickle/test_SheerkaPickler.py +++ b/tests/sheerkapickle/test_SheerkaPickler.py @@ -127,7 +127,7 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka): sheerka.add_in_cache(concept) obj = {concept: "a"} flatten = SheerkaPickler(sheerka).flatten(obj) - assert flatten == {':c:foo:1001:': 'a'} + assert flatten == {'c:foo|1001:': 'a'} decoded = SheerkaUnpickler(sheerka).restore(flatten) assert decoded == obj