From ffd98d74077742675a0532ae4b4ef42cb225bfe6 Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Fri, 3 Jan 2020 19:19:57 +0100 Subject: [PATCH] Fixed BNF concept evaluations --- core/builtin_concepts.py | 47 +++---- core/sheerka.py | 144 ++++++++++++------- core/utils.py | 9 ++ evaluators/AddConceptInSetEvaluator.py | 6 +- evaluators/ConceptComposerEvaluator.py | 110 +++++++++++++++ evaluators/ConceptEvaluator.py | 7 +- evaluators/ConceptNodeEvaluator.py | 96 +++++++++++-- evaluators/OneErrorEvaluator.py | 2 +- evaluators/PythonEvaluator.py | 7 +- parsers/BaseParser.py | 7 +- parsers/ConceptLexerParser.py | 2 +- parsers/PythonParser.py | 10 +- tests/test_BaseParser.py | 10 +- tests/test_ConceptComposerEvaluator.py | 128 +++++++++++++++++ tests/test_ConceptNodeEvaluator.py | 187 +++++++++++++------------ tests/test_PythonEvaluator.py | 2 +- tests/test_sheerka.py | 80 +++++++---- tests/test_sheerka_call_evaluators.py | 4 +- tests/test_sheerka_non_reg.py | 56 +++++++- tests/test_utils.py | 5 + 20 files changed, 682 insertions(+), 237 deletions(-) create mode 100644 evaluators/ConceptComposerEvaluator.py create mode 100644 tests/test_ConceptComposerEvaluator.py diff --git a/core/builtin_concepts.py b/core/builtin_concepts.py index f144889..fdaf827 100644 --- a/core/builtin_concepts.py +++ b/core/builtin_concepts.py @@ -60,6 +60,22 @@ class BuiltinConcepts(Enum): return "__" + self.name +BuiltinUnique = [ + BuiltinConcepts.BEFORE_PARSING, + BuiltinConcepts.PARSING, + BuiltinConcepts.AFTER_PARSING, + BuiltinConcepts.BEFORE_EVALUATION, + BuiltinConcepts.EVALUATION, + BuiltinConcepts.AFTER_EVALUATION, + BuiltinConcepts.BEFORE_RENDERING, + BuiltinConcepts.RENDERING, + BuiltinConcepts.AFTER_RENDERING, + BuiltinConcepts.SUCCESS, + BuiltinConcepts.NOP, + BuiltinConcepts.CONCEPT_EVAL_REQUESTED, + BuiltinConcepts.REDUCE_REQUESTED, +] + BuiltinErrors = [str(e) for e in { BuiltinConcepts.ERROR, BuiltinConcepts.UNKNOWN_CONCEPT, @@ -75,7 +91,7 @@ BuiltinErrors = [str(e) for e in { """ Some concepts have a specific implementation -It's mainly to a have proper __repr__ implementation, or because they are singleton (is_unique=True) +It's mainly to ease the usage """ @@ -96,11 +112,6 @@ class UserInputConcept(Concept): return f"({self.id}){self.name}: '{self.body}'" -class SuccessConcept(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.SUCCESS, True, True, BuiltinConcepts.SUCCESS) - - class ErrorConcept(Concept): def __init__(self, error=None): super().__init__(BuiltinConcepts.ERROR, True, False, BuiltinConcepts.ERROR, error) @@ -256,30 +267,6 @@ class InvalidReturnValueConcept(Concept): self.set_prop("evaluator", evaluator) -class BeforeParsingConcept(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.BEFORE_PARSING, True, True, BuiltinConcepts.BEFORE_PARSING) - - -class EvaluationConcept(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.EVALUATION, True, True, BuiltinConcepts.EVALUATION) - - -class AfterEvaluationConcept(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.AFTER_EVALUATION, True, True, BuiltinConcepts.AFTER_EVALUATION) - - -class ConceptEvalRequested(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.CONCEPT_EVAL_REQUESTED, True, True, BuiltinConcepts.CONCEPT_EVAL_REQUESTED) - - -class ReduceRequested(Concept): - def __init__(self): - super().__init__(BuiltinConcepts.REDUCE_REQUESTED, True, True, BuiltinConcepts.REDUCE_REQUESTED) - class ConceptEvalError(Concept): def __init__(self, error=None, concept=None, property_name=None): super().__init__(BuiltinConcepts.CONCEPT_EVAL_ERROR, diff --git a/core/sheerka.py b/core/sheerka.py index d1597e8..9650246 100644 --- a/core/sheerka.py +++ b/core/sheerka.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors +from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW from parsers.BaseParser import BaseParser from sdp.sheerkaDataProvider import SheerkaDataProvider, Event, SheerkaDataProviderDuplicateKeyError @@ -114,6 +114,10 @@ class Sheerka(Concept): else builtins_classes[str(key)]() if str(key) in builtins_classes \ else Concept(key, True, False, key) + if key in BuiltinUnique: + concept.metadata.is_unique = True + concept.metadata.is_evaluated = True + if not concept.metadata.is_unique and str(key) in builtins_classes: self.builtin_cache[key] = builtins_classes[str(key)] @@ -214,7 +218,7 @@ class Sheerka(Concept): result.append(return_value) continue - to_parse = self.value(return_value) + to_parse = return_value.body.body # get the underlying text if self.log.isEnabledFor(logging.DEBUG): debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \ @@ -509,14 +513,18 @@ class Sheerka(Concept): concept.cached_asts[part_key] = self.execute(sub_context, to_parse, steps, logger) for prop in concept.props: - if concept.props[prop].value: - to_parse = self.ret( - context.who, - True, - self.new(BuiltinConcepts.USER_INPUT, body=concept.props[prop].value)) - sub_context = context.push(desc=f"Initializing AST for property {prop}") - sub_context.log_new(logger) - concept.cached_asts[prop] = self.execute(context, to_parse, steps) + value = concept.props[prop].value + if value: + if isinstance(value, Concept): + concept.cached_asts[prop] = value + else: + to_parse = self.ret( + context.who, + True, + self.new(BuiltinConcepts.USER_INPUT, body=value)) + sub_context = context.push(desc=f"Initializing AST for property {prop}") + sub_context.log_new(logger) + concept.cached_asts[prop] = self.execute(context, to_parse, steps) # Updates the cache of concepts when possible if concept.key in self.concepts_cache: @@ -545,7 +553,6 @@ class Sheerka(Concept): def _resolve(return_value, desc, obj): context.log(logger, desc, self.evaluate_concept.__name__) sub_context = context.push(desc=desc, obj=obj) - sub_context.add_preprocess(self.get_evaluator_name("Concept"), return_body=True) sub_context.log_new(logger) r = self.execute(sub_context, return_value, CONCEPT_EVALUATION_STEPS, logger) return core.builtin_helpers.expect_one(context, r) @@ -568,14 +575,23 @@ class Sheerka(Concept): for metadata_to_eval in all_metadata_to_eval: if metadata_to_eval == "props": for prop_name in (p for p in concept.props if p in concept.cached_asts): - res = _resolve(concept.cached_asts[prop_name], f"Evaluating property '{prop_name}'", None) - if res.status: - concept.set_prop(prop_name, res.value) + prop_ast = concept.cached_asts[prop_name] + if isinstance(concept.cached_asts[prop_name], Concept): + context.log( + logger, f"Evaluation prop={prop_name}, value={prop_ast}", self.evaluate_concept.__name__) + sub_context = context.push(f"Evaluation property '{prop_name}', value='{prop_ast}'") + sub_context.log_new(logger) + evaluated = self.evaluate_concept(sub_context, prop_ast) + concept.set_prop(prop_name, evaluated) else: - return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, - body=res.value, - concept=concept, - property_name=prop_name) + res = _resolve(prop_ast, f"Evaluating property '{prop_name}'", None) + if res.status: + concept.set_prop(prop_name, res.value) + else: + return self.new(BuiltinConcepts.CONCEPT_EVAL_ERROR, + body=res.value, + concept=concept, + property_name=prop_name) else: part_key = ConceptParts(metadata_to_eval) @@ -654,41 +670,41 @@ class Sheerka(Concept): """ template = self.get(concept_key) - def new_from_template(t, k, **kwargs_): - # manage singleton - if t.metadata.is_unique: - return t - - # otherwise, create another instance - concept = self.builtin_cache[k]() if k in self.builtin_cache else Concept() - concept.update_from(t) - - # update the properties - for k, v in kwargs_.items(): - if k in concept.props: - concept.set_prop(k, v) - elif k in PROPERTIES_FOR_NEW: - setattr(concept.metadata, k, v) - elif hasattr(concept, k): - setattr(concept, k, v) - else: - return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept) - - # TODO : add the concept to the list of known concepts (self.instances) - return concept - # manage concept not found if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \ concept_key != BuiltinConcepts.UNKNOWN_CONCEPT: return template if not isinstance(template, list): - return new_from_template(template, concept_key, **kwargs) + return self.new_from_template(template, concept_key, **kwargs) # if template is a list, it means that there a multiple concepts under the same key - concepts = [new_from_template(t, concept_key, **kwargs) for t in template] + concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template] return self.new(BuiltinConcepts.ENUMERATION, body=concepts) + def new_from_template(self, template, key, **kwargs): + # manage singleton + if template.metadata.is_unique: + return template + + # otherwise, create another instance + concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept() + concept.update_from(template) + + # update the properties + for key, v in kwargs.items(): + if key in concept.props: + concept.set_prop(key, v) + elif key in PROPERTIES_FOR_NEW: + setattr(concept.metadata, key, v) + elif hasattr(concept, key): + setattr(concept, key, v) + else: + return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=key, concept=concept) + + # TODO : add the concept to the list of known concepts (self.instances) + return concept + def ret(self, who: str, status: bool, value, message=None, parents=None): """ Creates and returns a ReturnValue concept @@ -841,17 +857,25 @@ class Sheerka(Concept): def dump_desc(self, concept_name): if isinstance(concept_name, Concept): - c = concept_name + concepts = concept_name else: - c = self.get(concept_name) - if self.isinstance(c, BuiltinConcepts.UNKNOWN_CONCEPT): + concepts = self.get(concept_name) + if self.isinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT): self.log.error("Concept unknown") return False - self.log.info(f"name : {c.name}") - self.log.info(f"bnf : {c.metadata.definition}") - self.log.info(f"key : {c.key}") - self.log.info(f"body : {c.body}") + if not hasattr(concepts, "__iter__"): + concepts = [concepts] + + first = True + for c in concepts: + if not first: + self.log.info("") + self.log.info(f"name : {c.name}") + self.log.info(f"bnf : {c.metadata.definition}") + self.log.info(f"key : {c.key}") + self.log.info(f"body : {c.body}") + first = False @staticmethod def get_builtins_classes_as_dict(): @@ -920,6 +944,26 @@ class ExecutionContext: self.preprocess.add(preprocess) return self + def new_concept(self, key, **kwargs): + # search in obj + if self.obj: + if self.obj.key == key: + return self.sheerka.new_from_template(self.obj, key, **kwargs) + for prop in self.obj.props: + if prop == key: + value = self.obj.props[prop].value + if isinstance(value, Concept): + return self.sheerka.new_from_template(value, key, **kwargs) + else: + return value + + if self.concepts: + for k, c in self.concepts.items(): + if k == key: + return self.sheerka.new_from_template(c, key, **kwargs) + + return self.sheerka.new(key, **kwargs) + @property def id(self): return self._id diff --git a/core/utils.py b/core/utils.py index 4216574..0f8987d 100644 --- a/core/utils.py +++ b/core/utils.py @@ -220,6 +220,15 @@ def strip_tokens(tokens, strip_eof=False): return tokens[start: end + 1] +def escape_char(text, to_escape): + res = "" + + for c in text: + res += ("\\" + c) if c in to_escape else c + + return res + + def pp(items): if not hasattr(items, "__iter__"): return str(items) diff --git a/evaluators/AddConceptInSetEvaluator.py b/evaluators/AddConceptInSetEvaluator.py index 87f0d88..330f88f 100644 --- a/evaluators/AddConceptInSetEvaluator.py +++ b/evaluators/AddConceptInSetEvaluator.py @@ -65,6 +65,10 @@ class AddConceptInSetEvaluator(OneReturnValueEvaluator): else: context.log(self.log, f"Concept added.", self.name) - return res + return sheerka.ret( + self.name, + res.status, + res.body, + parents=[return_value]) diff --git a/evaluators/ConceptComposerEvaluator.py b/evaluators/ConceptComposerEvaluator.py new file mode 100644 index 0000000..6e5d010 --- /dev/null +++ b/evaluators/ConceptComposerEvaluator.py @@ -0,0 +1,110 @@ +from core.builtin_concepts import BuiltinConcepts, ParserResultConcept +from core.concept import Concept +from core.tokenizer import TokenKind +from evaluators.BaseEvaluator import AllReturnValuesEvaluator, BaseEvaluator +from parsers.BaseParser import BaseParser +from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, ConceptLexerParser +import core.utils + + +class ConceptComposerEvaluator(AllReturnValuesEvaluator): + """ + Try to reassemble parts of concepts from different evaluators + """ + + NAME = "ConceptComposer" + + def __init__(self): + super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 40) + + def matches(self, context, return_values): + concept_lexer_parser_name = ConceptLexerParser().name + + for return_value in return_values: + if return_value.who.startswith(BaseParser.PREFIX) and return_value.status: + return False + + if return_value.who.startswith(BaseEvaluator.PREFIX): + return False + + if return_value.who != concept_lexer_parser_name: + continue + + if not isinstance(return_value.value, ParserResultConcept): + return False + + if not ( + isinstance(return_value.value.value, ConceptNode) or + isinstance(return_value.value.value, UnrecognizedTokensNode) or + ( + hasattr(return_value.value.value, "__iter__") and + len(return_value.value.value) > 0 and + ( + isinstance(return_value.value.value[0], ConceptNode) or + isinstance(return_value.value.value[0], UnrecognizedTokensNode) + ))): + return False + + self.eaten = return_value + return True + + return False + + def eval(self, context, return_value): + sheerka = context.sheerka + nodes = self.eaten.value.value + temp_res = [] + has_error = False + concepts_only = True + + for node in nodes: + if isinstance(node, UnrecognizedTokensNode): + tokens = core.utils.strip_tokens(node.tokens, True) + for token in tokens: + if token.type == TokenKind.IDENTIFIER: + concept = context.new_concept(token.value) + if sheerka.isinstance(concept, BuiltinConcepts.UNKNOWN_CONCEPT): + has_error = True + else: + sub_context = context.push(self.name, desc=f"Evaluating '{concept}'") + sub_context.log_new(self.verbose_log) + concept = sheerka.evaluate_concept(sub_context, concept, self.verbose_log) + temp_res.append(concept) + else: + temp_res.append(core.utils.strip_quotes(token.value)) + concepts_only &= token.type == TokenKind.WHITESPACE or token.type == TokenKind.NEWLINE + else: + sub_context = context.push(self.name, desc=f"Evaluating '{node.concept}'") + sub_context.log_new(self.verbose_log) + concept = sheerka.evaluate_concept(sub_context, node.concept, self.verbose_log) + temp_res.append(concept) + + if has_error: + return sheerka.ret( + self.name, + False, + temp_res, + parents=[self.eaten]) + + if concepts_only: + res = [] + for r in temp_res: + if isinstance(r, Concept): + res.append(r) + else: + res = "" + for r in temp_res: + if isinstance(r, Concept): + res += sheerka.value(r) + else: + res += r + + return sheerka.ret( + self.name, + True, + res, + parents=[self.eaten]) + + + + diff --git a/evaluators/ConceptEvaluator.py b/evaluators/ConceptEvaluator.py index 9f4355b..a9ec47e 100644 --- a/evaluators/ConceptEvaluator.py +++ b/evaluators/ConceptEvaluator.py @@ -11,11 +11,6 @@ class ConceptEvaluator(OneReturnValueEvaluator): Then checks the POST conditions """ NAME = "Concept" - evaluation_steps = [ - BuiltinConcepts.BEFORE_EVALUATION, - BuiltinConcepts.EVALUATION, - BuiltinConcepts.AFTER_EVALUATION - ] def __init__(self, return_body=False): super().__init__(self.NAME, [BuiltinConcepts.EVALUATION], 50) @@ -34,7 +29,7 @@ class ConceptEvaluator(OneReturnValueEvaluator): # If the concept that is requested is in the context(at least its name), drop the call. # Why ? # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'") - # The body should be 'property_a', and not a concept called a in our universe + # The body should be 'property_a', and not a concept called 'a' if context.obj and concept.name in context.obj.props: value = context.obj.props[concept.name].value context.log(self.verbose_log, f"{concept.name} is a property. Returning value '{value}'.", self.name) diff --git a/evaluators/ConceptNodeEvaluator.py b/evaluators/ConceptNodeEvaluator.py index b610f77..4ef034a 100644 --- a/evaluators/ConceptNodeEvaluator.py +++ b/evaluators/ConceptNodeEvaluator.py @@ -1,7 +1,7 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts from evaluators.BaseEvaluator import OneReturnValueEvaluator - -from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode +import core.utils +from parsers.ConceptLexerParser import ConceptNode, NonTerminalNode, ConceptMatch, UnrecognizedTokensNode, TerminalNode class ConceptNodeEvaluator(OneReturnValueEvaluator): @@ -46,10 +46,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator): concepts = [] error_found = False + source = "" for node in nodes: if isinstance(node, ConceptNode): + source += node.source if source == "" else (" " + node.source) concept = sheerka.new(node.concept.key) - concept = self.update_concept(sheerka, concept, node.underlying) + concept = self.finalize_concept(sheerka, concept, node.underlying) concepts.append(concept) else: error_found = True @@ -58,12 +60,17 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator): return sheerka.ret( self.name, not error_found, - concepts[0], + context.sheerka.new( + BuiltinConcepts.PARSER_RESULT, + parser=self, + source=source, + body=concepts[0], + try_parsed=None), parents=[return_value]) return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME), parents=[return_value]) - def update_concept(self, sheerka, concept, underlying, init_empty_body=True): + def finalize_concept(self, sheerka, concept, underlying, init_empty_body=True): """ Updates the properties of the concept Goes in recursion if the property is a concept @@ -74,9 +81,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator): Adds a new entry, makes a list if the property already exists """ + if prop_name not in c.props or c.props[prop_name].value is None: + # new entry c.set_prop(prop_name, value) else: + # make a list if there was a value previous_value = c.props[prop_name].value if isinstance(previous_value, list): previous_value.append(value) @@ -87,11 +97,12 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator): parsing_expression = underlying.parsing_expression if parsing_expression.rule_name: - _add_prop(concept, parsing_expression.rule_name, underlying.source) + _add_prop(concept, parsing_expression.rule_name, self.get_underlying_as_string(underlying)) # the update of the body must come BEFORE the recursion + # otherwise it will be updated by a children and it won't be possible to modify the value if init_empty_body and concept.body is None: - concept.metadata.body = underlying.source + concept.metadata.body = self.get_underlying_as_string(underlying) # self.escape_if_needed(underlying.source) if isinstance(underlying, NonTerminalNode): for child in underlying.children: @@ -101,8 +112,75 @@ class ConceptNodeEvaluator(OneReturnValueEvaluator): if sheerka.isinstance(new_concept, BuiltinConcepts.UNKNOWN_CONCEPT): continue else: - self.update_concept(sheerka, new_concept, child.children[0], init_empty_body) + self.finalize_concept(sheerka, new_concept, child.children[0], init_empty_body) else: - self.update_concept(sheerka, concept, child, init_empty_body) + self.finalize_concept(sheerka, concept, child, init_empty_body) return concept + + @staticmethod + def escape_if_needed(value): + if not isinstance(value, str): + return value + + return "'" + core.utils.escape_char(value, "'") + "'" + + def get_underlying_as_string(self, underlying): + """ + Return the sequence of the recognized character + When a concept is recognized, return the string version of the concept eg c:concept name: + :param underlying: + :return: + """ + + # Example + # grammar = { + # foo: Sequence("one", "two", rule_name="var"), + # bar: Sequence(foo, "three", rule_name="var")} + # + # we want bar.body and bar.prop["var"] + # to be "foo 'three'" (no quotes surrounding foo, as it is a concept, not a string) + + if isinstance(underlying, TerminalNode): + return self.escape_if_needed(underlying.source) + + res = "" + first = True + in_quote = "" + for node in underlying.children: + if isinstance(node.parsing_expression, ConceptMatch): + if in_quote != "": + res += in_quote + "'" + if not first: + res += " " + res += node.parsing_expression.concept.key + in_quote = "" + else: + if in_quote == "": + in_quote = ("'" if first else " '") + core.utils.escape_char(node.source, "'") + else: + in_quote += ("" if first else " ") + core.utils.escape_char(node.source, "'") + + first = False + + if in_quote: + res += in_quote + "'" + return res + +# - - - E X P L A N A T I O N S - - - +# why do we need to update the body ? +# cf test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more() +# def concept number from bnf one | two | three +# def concept add from bnf number plus number +# +# the expression 'one plus two plus three' will match concept add +# add.props["number"] is a list of concepts 'number' +# But which one is 'one', which one is 'two' which one is 'three' ? +# +# That's the reason why we update the body +# add.props["number"] is a list of concepts 'number' but they won't have the same body +# +# !!! C A U T I O N !!! +# In the current implementation, the body is the sequence of char found +# If a concept is recognized, we don't put this information in the body +# Use get_body_as_string() instead of escape_if_needed() if we need this information diff --git a/evaluators/OneErrorEvaluator.py b/evaluators/OneErrorEvaluator.py index 6547915..5c7354a 100644 --- a/evaluators/OneErrorEvaluator.py +++ b/evaluators/OneErrorEvaluator.py @@ -12,7 +12,7 @@ class OneErrorEvaluator(AllReturnValuesEvaluator): NAME = "OneError" def __init__(self): - super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 40) + super().__init__(self.NAME, [BuiltinConcepts.AFTER_EVALUATION], 30) self.return_value_in_error = None def matches(self, context, return_values): diff --git a/evaluators/PythonEvaluator.py b/evaluators/PythonEvaluator.py index db539cb..7db4be1 100644 --- a/evaluators/PythonEvaluator.py +++ b/evaluators/PythonEvaluator.py @@ -3,7 +3,7 @@ from enum import Enum from core.ast.visitors import UnreferencedNamesVisitor from core.builtin_concepts import BuiltinConcepts, ParserResultConcept -from core.concept import ConceptParts +from core.concept import ConceptParts, Concept from evaluators.BaseEvaluator import OneReturnValueEvaluator from parsers.PythonParser import PythonNode import ast @@ -65,7 +65,10 @@ class PythonEvaluator(OneReturnValueEvaluator): f"Concept '{context.obj}' is in context. Adding its properties to locals if any.", self.name) for prop_name, prop_value in context.obj.props.items(): - my_locals[prop_name] = prop_value.value + if not isinstance(prop_value.value, Concept): + my_locals[prop_name] = prop_value.value + else: + my_locals[prop_name] = context.sheerka.value(prop_value.value) node_concept = core.ast.nodes.python_to_concept(ast_) unreferenced_names_visitor = UnreferencedNamesVisitor(context.sheerka) diff --git a/parsers/BaseParser.py b/parsers/BaseParser.py index cb68391..8c6e4fd 100644 --- a/parsers/BaseParser.py +++ b/parsers/BaseParser.py @@ -78,7 +78,7 @@ class BaseParser: context.log(self.log, f" Recognized '{value}'", self.name) @staticmethod - def get_text_from_tokens(tokens): + def get_text_from_tokens(tokens, custom_switcher=None): if tokens is None: return "" res = "" @@ -88,9 +88,12 @@ class BaseParser: switcher = { TokenKind.KEYWORD: lambda t: Keywords(t.value).value, - TokenKind.CONCEPT: lambda t: f"__C__{t.value}__C__" + TokenKind.CONCEPT: lambda t: "c:" + t.value + ":", } + if custom_switcher: + switcher.update(custom_switcher) + for token in tokens: value = switcher.get(token.type, lambda t: t.value)(token) res += value diff --git a/parsers/ConceptLexerParser.py b/parsers/ConceptLexerParser.py index 862da6a..5a7eeea 100644 --- a/parsers/ConceptLexerParser.py +++ b/parsers/ConceptLexerParser.py @@ -795,7 +795,7 @@ class ConceptLexerParser(BaseParser): if not self.next_token(): break - # Fix the source if we were working on unrecognized tokens + # Fix the source for unrecognized tokens if unrecognized_tokens: unrecognized_tokens.fix_source() diff --git a/parsers/PythonParser.py b/parsers/PythonParser.py index a377785..f3bc2d7 100644 --- a/parsers/PythonParser.py +++ b/parsers/PythonParser.py @@ -1,5 +1,5 @@ from core.builtin_concepts import BuiltinConcepts -from core.tokenizer import Tokenizer, LexerError +from core.tokenizer import Tokenizer, LexerError, TokenKind from parsers.BaseParser import BaseParser, Node, ErrorNode from dataclasses import dataclass import ast @@ -66,13 +66,17 @@ class PythonParser(BaseParser): sheerka = context.sheerka tree = None + python_switcher = { + TokenKind.CONCEPT: lambda t: f"__C__{t.value}__C__" + } + try: if isinstance(text, str) and "c:" in text: - source = self.get_text_from_tokens(list(Tokenizer(text))) + source = self.get_text_from_tokens(list(Tokenizer(text)), python_switcher) elif isinstance(text, str): source = text else: - source = self.get_text_from_tokens(text) + source = self.get_text_from_tokens(text, python_switcher) source = source.strip() text = text if isinstance(text, str) else source diff --git a/tests/test_BaseParser.py b/tests/test_BaseParser.py index f6c8b7d..3b78576 100644 --- a/tests/test_BaseParser.py +++ b/tests/test_BaseParser.py @@ -9,9 +9,17 @@ from parsers.BaseParser import BaseParser ("'hello' 'world'", "'hello' 'world'"), ("def concept a from", "def concept a from"), ("()[]{}1=1.5+-/*><&é", "()[]{}1=1.5+-/*><&é"), - ("execute(c:concept_name:)", "execute(__C__concept_name__C__)") + ("execute(c:concept_name:)", "execute(c:concept_name:)") ]) def test_i_can_get_text_from_tokens(text, expected_text): tokens = list(Tokenizer(text)) assert BaseParser.get_text_from_tokens(tokens) == expected_text + + +@pytest.mark.parametrize("text, custom, expected_text", [ + ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)") +]) +def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text): + tokens = list(Tokenizer(text)) + assert BaseParser.get_text_from_tokens(tokens, custom) == expected_text diff --git a/tests/test_ConceptComposerEvaluator.py b/tests/test_ConceptComposerEvaluator.py new file mode 100644 index 0000000..0dc825f --- /dev/null +++ b/tests/test_ConceptComposerEvaluator.py @@ -0,0 +1,128 @@ +import pytest + +from core.builtin_concepts import ReturnValueConcept, ParserResultConcept +from core.concept import Concept +from core.sheerka import Sheerka, ExecutionContext +from evaluators.BaseEvaluator import BaseEvaluator +from evaluators.ConceptComposerEvaluator import ConceptComposerEvaluator +from parsers.BaseParser import BaseParser +from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence +from sdp.sheerkaDataProvider import Event + +concept_lexer_name = ConceptLexerParser().name + + +def get_context(): + sheerka = Sheerka(skip_builtins_in_db=True) + sheerka.initialize("mem://") + return ExecutionContext("test", Event(), sheerka) + + +def get_return_values(context, grammar, expression): + parser = ConceptLexerParser() + parser.initialize(context, grammar) + + ret_val = parser.parse(context, expression) + assert not ret_val.status + return [ret_val] + + +def init(concepts, grammar, expression): + context = get_context() + for c in concepts: + context.sheerka.add_in_cache(c) + return_values = get_return_values(context, grammar, expression) + + return context, return_values + + +@pytest.mark.parametrize("return_values, expected", [ + ([ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"), + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ReturnValueConcept("not a parser", True, "some value"), + ], True), + ([ + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ], True), + ([ + ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not in error"), + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ], False), + ([ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"), + ReturnValueConcept(concept_lexer_name, True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ], False), + ([ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"), + ReturnValueConcept(concept_lexer_name, False, "some value"), + ], False), + ([ + ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"), + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=["not a concept"])), + ], False), + ([ + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", False, "evaluator in error"), + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ReturnValueConcept("not a parser", True, "some value"), + ], False), + ([ + ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "evaluator"), + ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), + ReturnValueConcept("not a parser", True, "some value"), + ], False), +]) +def test_i_can_match(return_values, expected): + context = get_context() + assert ConceptComposerEvaluator().matches(context, return_values) == expected + + +def test_i_can_eval_simple_concepts(): + foo = Concept("foo", body="'foo'") + bar = Concept("bar", body="'bar'") + grammar = {} + context, return_values = init([foo, bar], grammar, "bar foo") + + composer = ConceptComposerEvaluator() + assert composer.matches(context, return_values) + + ret_val = composer.eval(context, return_values) + assert ret_val.status + assert ret_val.who == composer.name + assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()] + assert ret_val.value[0].metadata.is_evaluated + assert ret_val.value[1].metadata.is_evaluated + assert ret_val.parents == [return_values[0]] + + +def test_i_can_eval_simple_concepts_when_some_are_bnf(): + foo = Concept("foo", body="'foo'") + bar = Concept("bar", body="'bar'") + grammar = {foo: "foo"} + context, return_values = init([foo, bar], grammar, "bar foo") + + composer = ConceptComposerEvaluator() + assert composer.matches(context, return_values) + + ret_val = composer.eval(context, return_values) + assert ret_val.status + assert ret_val.who == composer.name + assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()] + assert ret_val.value[0].metadata.is_evaluated + assert ret_val.value[1].metadata.is_evaluated + assert ret_val.parents == [return_values[0]] + + +def test_i_can_eval_simple_concept_and_text(): + foo = Concept("foo", body="'foo'") + grammar = {} + context, return_values = init([foo], grammar, "'bar' foo") + + composer = ConceptComposerEvaluator() + assert composer.matches(context, return_values) + + ret_val = composer.eval(context, return_values) + assert ret_val.status + assert ret_val.who == composer.name + assert ret_val.value == "bar foo" + assert ret_val.parents == [return_values[0]] diff --git a/tests/test_ConceptNodeEvaluator.py b/tests/test_ConceptNodeEvaluator.py index 1c36bc8..0244654 100644 --- a/tests/test_ConceptNodeEvaluator.py +++ b/tests/test_ConceptNodeEvaluator.py @@ -5,7 +5,7 @@ from core.concept import Concept from core.sheerka import Sheerka, ExecutionContext from evaluators.ConceptNodeEvaluator import ConceptNodeEvaluator from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence, TerminalNode, \ - StrMatch, Optional, OrderedChoice, ZeroOrMore, UnrecognizedTokensNode + StrMatch, Optional, OrderedChoice, ZeroOrMore, UnrecognizedTokensNode, ConceptMatch from sdp.sheerkaDataProvider import Event @@ -15,23 +15,26 @@ def get_context(): return ExecutionContext("test", Event(), sheerka) -def get_return_value(nodes, source): - return ReturnValueConcept( - "some_name", - True, - ParserResultConcept(parser=ConceptLexerParser(), - source=source, - value=nodes, - try_parsed=nodes)) - - -def get_concept_node(context, grammar, expression): +def get_return_value(context, grammar, expression): parser = ConceptLexerParser() parser.initialize(context, grammar) - res = parser.parse(context, expression) - assert res.status - return res.value.value[0] + ret_val = parser.parse(context, expression) + assert ret_val.status + return ret_val + + +def init(concept, grammar, text): + context = get_context() + if isinstance(concept, list): + for c in concept: + context.sheerka.add_in_cache(c) + else: + context.sheerka.add_in_cache(concept) + ret_val = get_return_value(context, grammar, text) + node = ret_val.value.value[0] + + return context, node @pytest.mark.parametrize("ret_val, expected", [ @@ -53,174 +56,172 @@ def test_i_can_match(ret_val, expected): assert ConceptNodeEvaluator().matches(context, ret_val) == expected -def test_concept_is_returned_when_list_of_one_concept_node(): +def test_parser_result_of_concept_is_returned_when_list_of_one_concept_node(): foo = Concept("foo") context = get_context() context.sheerka.add_in_cache(foo) evaluator = ConceptNodeEvaluator() - node = ConceptNode(foo, 0, 0, underlying=TerminalNode(StrMatch("foo"), 0, 0, "foo")) + ret_val = get_return_value(context, {foo: StrMatch("foo")}, "foo") - ret_val = get_return_value([node], "h") result = evaluator.eval(context, ret_val) assert result.who == evaluator.name assert result.status - assert result.value == Concept("foo", body="foo").init_key() + assert result.value == ParserResultConcept( + evaluator, + "foo", + Concept("foo", body="'foo'").init_key(), + None) assert result.parents == [ret_val] def test_concept_property_is_correctly_updated_for_str_match(): - context = get_context() - foo = Concept("foo") - concept_node = get_concept_node(context, {foo: StrMatch("foo", rule_name="variable")}, "foo") - updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying) + grammar = {foo: StrMatch("foo", rule_name="variable")} + context, node = init(foo, grammar, "foo") + + updated = ConceptNodeEvaluator().finalize_concept(context.sheerka, node.concept, node.underlying) assert "variable" in updated.props - assert updated.props["variable"].value == "foo" + assert updated.props["variable"].value == "'foo'" + assert updated.body == "'foo'" def test_concept_property_is_correctly_updated_for_sequence(): - context = get_context() - foo = Concept("foo") grammar = {foo: Sequence("one", "two", rule_name="variable")} - concept_node = get_concept_node(context, grammar, "one two") - updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying) + context, node = init(foo, grammar, "one two") + + updated = ConceptNodeEvaluator().finalize_concept( + context.sheerka, + context.sheerka.new(node.concept.key), + node.underlying) assert "variable" in updated.props - assert updated.props["variable"].value == "one two" + assert updated.props["variable"].value == "'one two'" + assert updated.body == "'one two'" def test_concept_property_is_updated_for_str_in_sequence(): - context = get_context() - foo = Concept("foo") grammar = {foo: Sequence(StrMatch("one", rule_name="s1"), StrMatch("two", rule_name="s2"), rule_name="variable")} - concept_node = get_concept_node(context, grammar, "one two") + context, node = init(foo, grammar, "one two") - updated = ConceptNodeEvaluator().update_concept(context.sheerka, concept_node.concept, concept_node.underlying) + updated = ConceptNodeEvaluator().finalize_concept( + context.sheerka, + context.sheerka.new(node.concept.key), + node.underlying) - assert updated.props["variable"].value == "one two" - assert updated.props["s1"].value == "one" - assert updated.props["s2"].value == "two" + assert updated.props["variable"].value == "'one two'" + assert updated.props["s1"].value == "'one'" + assert updated.props["s2"].value == "'two'" + assert updated.body == "'one two'" def test_concept_property_is_correctly_updated_for_optional(): - context = get_context() - foo = Concept("foo") grammar = {foo: Sequence("one", Optional("two", rule_name="o"), rule_name="variable")} - concept_node = get_concept_node(context, grammar, "one two") + context, node = init(foo, grammar, "one two") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying) + context.sheerka.new(node.concept.key), + node.underlying) assert "variable" in updated.props - assert updated.props["variable"].value == "one two" - assert updated.props["o"].value == "two" + assert updated.props["variable"].value == "'one two'" + assert updated.props["o"].value == "'two'" + assert updated.body == "'one two'" def test_concept_property_is_correctly_updated_for_zero_or_more(): - context = get_context() - foo = Concept("foo") grammar = {foo: ZeroOrMore("one", rule_name="variable")} - concept_node = get_concept_node(context, grammar, "one one one") + context, node = init(foo, grammar, "one one one") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying) + context.sheerka.new(node.concept.key), + node.underlying) assert "variable" in updated.props - assert updated.props["variable"].value == "one one one" + assert updated.props["variable"].value == "'one one one'" + assert updated.body == "'one one one'" def test_concept_property_is_correctly_updated_when_list_of_properties(): - context = get_context() - foo = Concept("foo") grammar = {foo: Sequence(StrMatch("one", rule_name="s"), StrMatch("two", rule_name="s"), rule_name="variable")} - concept_node = get_concept_node(context, grammar, "one two") + context, node = init(foo, grammar, "one two") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying) + context.sheerka.new(node.concept.key), + node.underlying) - assert updated.props["variable"].value == "one two" - assert updated.props["s"].value == ["one", "two"] + assert updated.props["variable"].value == "'one two'" + assert updated.props["s"].value == ["'one'", "'two'"] + assert updated.body == "'one two'" def test_concept_property_is_correctly_updated_when_another_concept(): - context = get_context() - foo = Concept("foo") bar = Concept("bar") - context.sheerka.add_in_cache(foo) - context.sheerka.add_in_cache(bar) grammar = { foo: Sequence("one", "two", rule_name="var"), - bar: Sequence(foo, "three", rule_name="var")} - concept_node = get_concept_node(context, grammar, "one two three") + bar: Sequence(foo, "three", "four", rule_name="var")} + context, node = init([foo, bar], grammar, "one two three four") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying) + context.sheerka.new(node.concept.key), + node.underlying) - assert updated.props["var"].value == "one two three" - assert updated.props["foo"].value == Concept("foo", body="one two").set_prop("var", "one two").init_key() + assert updated.body == "foo 'three four'" + assert updated.props["var"].value == "foo 'three four'" + assert updated.props["foo"].value == Concept("foo", body="'one two'").set_prop("var", "'one two'").init_key() def test_concept_property_is_correctly_updated_when_concept_recursion_using_optional(): - context = get_context() - number = Concept("number") add = Concept("add") - context.sheerka.add_in_cache(number) - context.sheerka.add_in_cache(add) grammar = { number: OrderedChoice("one", "two"), add: Sequence(number, Optional(Sequence(OrderedChoice("plus", "minus", rule_name="op"), add))) } - concept_node = get_concept_node(context, grammar, "one plus two") + context, node = init([number, add], grammar, "one plus two") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying) + context.sheerka.new(node.concept.key), + node.underlying) - assert updated.props["number"].value == Concept("number", body="one").init_key() - assert updated.props["op"].value == "plus" - expected_add = Concept("add", body="two").set_prop("number", Concept("number", body="two").init_key()).init_key() + assert updated.props["number"].value == Concept("number", body="'one'").init_key() + assert updated.props["op"].value == "'plus'" + expected_add = Concept("add", body="number"). \ + set_prop("number", Concept("number", body="'two'").init_key()). \ + init_key() assert updated.props["add"].value == expected_add def test_concept_property_is_correctly_updated_when_concept_recursion_using_zero_or_more(): - context = get_context() - number = Concept("number") add = Concept("add") - context.sheerka.add_in_cache(number) - context.sheerka.add_in_cache(add) grammar = { number: OrderedChoice("one", "two", 'three'), add: Sequence(number, ZeroOrMore(Sequence(OrderedChoice("plus", "minus", rule_name="op"), number))) } - concept_node = get_concept_node(context, grammar, "one plus two minus three") + context, node = init([number, add], grammar, "one plus two minus three") - updated = ConceptNodeEvaluator().update_concept( + updated = ConceptNodeEvaluator().finalize_concept( context.sheerka, - context.sheerka.new(concept_node.concept.key), - concept_node.underlying, + context.sheerka.new(node.concept.key), + node.underlying, init_empty_body=True) - assert updated.props["number"].value == [Concept("number", body="one").init_key(), - Concept("number", body="two").init_key(), - Concept("number", body="three").init_key()] - assert updated.props["op"].value == ["plus", "minus"] + assert updated.props["number"].value == [Concept("number", body="'one'").init_key(), + Concept("number", body="'two'").init_key(), + Concept("number", body="'three'").init_key()] + assert updated.props["op"].value == ["'plus'", "'minus'"] + diff --git a/tests/test_PythonEvaluator.py b/tests/test_PythonEvaluator.py index 13d920c..fe10982 100644 --- a/tests/test_PythonEvaluator.py +++ b/tests/test_PythonEvaluator.py @@ -118,7 +118,7 @@ def test_i_can_eval_concept_token(): assert evaluated.status assert evaluated.value == "foo" - # sanity, to make sure that otherwise foo is resolved to '2' + # sanity, does not work otherwise parsed = PythonParser().parse(context, "get_context_name(foo)") python_evaluator = PythonEvaluator() python_evaluator.locals["get_context_name"] = get_context_name diff --git a/tests/test_sheerka.py b/tests/test_sheerka.py index 953304c..7830e49 100644 --- a/tests/test_sheerka.py +++ b/tests/test_sheerka.py @@ -332,6 +332,7 @@ def test_i_cannot_instantiate_when_properties_are_not_recognized(): (Concept("name", body=["foo"]), True, "foo"), (Concept("name", body=Concept("foo")), False, Concept("foo")), (Concept("name", body=Concept("foo", body="value")), False, "value"), + (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"), (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"), ]) def test_i_can_get_value(concept, reduce_simple_list, expected): @@ -404,7 +405,7 @@ def test_i_can_evaluate_the_other_metadata(expr, expected): @pytest.mark.parametrize("expr, expected", [ - (None, None), + # (None, None), ("", ""), ("1", 1), ("1+1", 2), @@ -450,6 +451,7 @@ def test_i_can_evaluate_when_another_concept_is_referenced(): assert sheerka.isinstance(evaluated.body, concept_a) assert id(evaluated.body) != id(concept_a) assert evaluated.metadata.is_evaluated + assert evaluated.body.metadata.is_evaluated def test_i_can_evaluate_when_the_referenced_concept_has_a_body(): @@ -461,34 +463,50 @@ def test_i_can_evaluate_when_the_referenced_concept_has_a_body(): evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key - assert evaluated.body == 1 - assert not concept_a.metadata.is_evaluated # + assert evaluated.body == Concept("a", body=1).init_key() + assert not concept_a.metadata.is_evaluated + assert evaluated.metadata.is_evaluated def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'a'").init_key()) - sheerka.add_in_cache(Concept(name="b", body="a").init_key()) - sheerka.add_in_cache(Concept(name="c", body="b").init_key()) - concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key()) + sheerka.add_in_cache(Concept(name="a", body="'a'")) + sheerka.add_in_cache(Concept(name="b", body="a")) + sheerka.add_in_cache(Concept(name="c", body="b")) + concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d) assert evaluated.key == concept_d.key - assert evaluated.body == 'a' + assert evaluated.body == Concept( + name="c", + body=Concept( + name="b", + body=Concept( + name="a", + body="a").init_key()).init_key()).init_key() + assert sheerka.value(evaluated) == 'a' + assert evaluated.metadata.is_evaluated def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(): sheerka = get_sheerka() - concept_a = sheerka.add_in_cache(Concept(name="a").init_key()) - sheerka.add_in_cache(Concept(name="b", body="a").init_key()) - sheerka.add_in_cache(Concept(name="c", body="b").init_key()) - concept_d = sheerka.add_in_cache(Concept(name="d", body="c").init_key()) + sheerka.add_in_cache(Concept(name="a")) + sheerka.add_in_cache(Concept(name="b", body="a")) + sheerka.add_in_cache(Concept(name="c", body="b")) + concept_d = sheerka.add_in_cache(Concept(name="d", body="c")) evaluated = sheerka.evaluate_concept(get_context(sheerka), concept_d) assert evaluated.key == concept_d.key - assert sheerka.isinstance(evaluated.body, concept_a) + assert evaluated.body == Concept( + name="c", + body=Concept( + name="b", + body=Concept( + name="a").init_key()).init_key()).init_key() + assert sheerka.value(evaluated) == Concept(name="a").init_key() + assert evaluated.metadata.is_evaluated def test_i_can_evaluate_concept_when_properties_reference_others_concepts(): @@ -509,9 +527,9 @@ def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(): :return: """ sheerka = get_sheerka() - concept_a = sheerka.add_in_cache(Concept(name="a").init_key()) + concept_a = sheerka.add_in_cache(Concept(name="a")) - concept = Concept("foo", body="concept_a").set_prop("concept_a", "a").init_key() + concept = Concept("foo", body="concept_a").set_prop("concept_a", "a") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key @@ -520,10 +538,10 @@ def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(): def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="1").init_key()) - sheerka.add_in_cache(Concept(name="b", body="2").init_key()) + sheerka.add_in_cache(Concept(name="a", body="1")) + sheerka.add_in_cache(Concept(name="b", body="2")) - concept = Concept("foo", body="propA + propB").set_prop("propA", "a").set_prop("propB", "b").init_key() + concept = Concept("foo", body="propA + propB").set_prop("propA", "a").set_prop("propB", "b") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key @@ -542,33 +560,33 @@ def test_i_can_reference_sheerka(): def test_properties_values_takes_precedence_over_the_outside_world(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key()) - sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key()) + sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) + sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) - concept = Concept("foo", body="a").init_key() + concept = Concept("foo", body="a") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key - assert evaluated.body == 'concept_a' # this test was already done + assert evaluated.body == Concept(name="a", body="concept_a").init_key() # this test was already done # so check this one. - concept = Concept("foo", body="a").set_prop("a", "'property_a'").init_key() + concept = Concept("foo", body="a").set_prop("a", "'property_a'") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key assert evaluated.body == 'property_a' # or this one. - concept = Concept("foo", body="a").set_prop("a", "b").init_key() + concept = Concept("foo", body="a").set_prop("a", "b") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key - assert evaluated.body == 'concept_b' + assert evaluated.body == Concept(name="b", body="concept_b").init_key() def test_properties_values_takes_precedence(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="a", body="'concept_a'").init_key()) - sheerka.add_in_cache(Concept(name="b", body="'concept_b'").init_key()) + sheerka.add_in_cache(Concept(name="a", body="'concept_a'")) + sheerka.add_in_cache(Concept(name="b", body="'concept_b'")) - concept = Concept("foo", body="a + b").set_prop("a", "'prop_a'").init_key() + concept = Concept("foo", body="a + b").set_prop("a", "'prop_a'") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key assert evaluated.body == 'prop_aconcept_b' @@ -576,9 +594,9 @@ def test_properties_values_takes_precedence(): def test_i_can_reference_sub_property_of_a_property(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'").init_key()) + sheerka.add_in_cache(Concept(name="concept_a").set_prop("subProp", "'sub_a'")) - concept = Concept("foo", body="a.props['subProp'].value").set_prop("a", "concept_a").init_key() + concept = Concept("foo", body="a.props['subProp'].value").set_prop("a", "concept_a") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert evaluated.key == concept.key assert evaluated.body == 'sub_a' @@ -587,7 +605,7 @@ def test_i_can_reference_sub_property_of_a_property(): def test_i_cannot_evaluate_concept_if_property_is_in_error(): sheerka = get_sheerka() - concept = Concept(name="concept_a").set_prop("subProp", "undef_concept").init_key() + concept = Concept(name="concept_a").set_prop("subProp", "undef_concept") evaluated = sheerka.evaluate_concept(get_context(sheerka), concept) assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR) diff --git a/tests/test_sheerka_call_evaluators.py b/tests/test_sheerka_call_evaluators.py index c42c391..65ab457 100644 --- a/tests/test_sheerka_call_evaluators.py +++ b/tests/test_sheerka_call_evaluators.py @@ -1,5 +1,5 @@ # Make sure that the evaluators works as expected -from core.builtin_concepts import BuiltinConcepts, SuccessConcept +from core.builtin_concepts import BuiltinConcepts from core.concept import Concept from core.sheerka import Sheerka, ExecutionContext from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator @@ -157,7 +157,7 @@ class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority): def eval(self, context, return_values): super().eval(context, return_values) - ret = get_ret_val(context.sheerka, SuccessConcept()) + ret = get_ret_val(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS)) ret.parents = return_values return ret diff --git a/tests/test_sheerka_non_reg.py b/tests/test_sheerka_non_reg.py index 3c38843..2b51b7e 100644 --- a/tests/test_sheerka_non_reg.py +++ b/tests/test_sheerka_non_reg.py @@ -240,13 +240,17 @@ def test_i_can_eval_concept_with_variable(): def test_i_can_eval_concept_with_variable_and_python_as_body(): sheerka = get_sheerka() - sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a")) + hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").set_prop("a")) sheerka.add_in_cache(Concept(name="foo", body="'foo'")) res = sheerka.evaluate_user_input("hello foo") assert len(res) == 1 assert res[0].status - assert res[0].value, "hello foo" + assert sheerka.isinstance(res[0].value, hello_a) + assert res[0].value.body == "hello foo" + assert res[0].value.metadata.is_evaluated + assert res[0].value.props["a"].value == Concept(name="foo", body="foo").init_key() + assert res[0].value.props["a"].value.metadata.is_evaluated def test_i_can_eval_duplicate_concepts_with_same_value(): @@ -259,7 +263,7 @@ def test_i_can_eval_duplicate_concepts_with_same_value(): res = sheerka.evaluate_user_input("hello foo") assert len(res) == 1 assert res[0].status - assert res[0].value, "hello foo" + assert res[0].value.body == "hello foo" assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME) @@ -362,7 +366,11 @@ def test_i_can_eval_bnf_definitions_with_variables(): return_value = res[0].value assert sheerka.isinstance(return_value, concept_b) - assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one")) + assert return_value.body == "one three" + assert return_value.metadata.is_evaluated + + assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key()) + assert return_value.props["a"].value.metadata.is_evaluated def test_i_can_eval_bnf_definitions_from_separate_instances(): @@ -390,6 +398,8 @@ def test_i_can_eval_bnf_definitions_from_separate_instances(): assert len(res) == 1 assert res[0].status assert sheerka.isinstance(res[0].value, concept_b) + assert res[0].value.body == "one two three" + assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key()) def test_i_can_say_that_a_concept_isa_another_concept(): @@ -398,6 +408,7 @@ def test_i_can_say_that_a_concept_isa_another_concept(): sheerka.evaluate_user_input("def concept bar") res = sheerka.evaluate_user_input("foo isa bar") + assert len(res) == 1 assert res[0].status assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS) @@ -436,3 +447,40 @@ def test_i_can_manage_tokenizer_error(text): assert len(res) > 1 for r in [r for r in res if r.who.startswith("parsers.")]: assert not r.status + + +def test_i_can_recognize_concept_from_string(): + sheerka = get_sheerka() + sheerka.add_in_cache(Concept("one", body="1")) + + res = sheerka.evaluate_user_input("'one'") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == "one" + + res = sheerka.evaluate_user_input("eval 'one'") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == "one" + + +@pytest.mark.parametrize("expression", [ + "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit", + "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit", + "def concept twenties from bnf twenty (one | two)=unit as 20 + unit", + "def concept twenties from bnf twenty (one | two)=unit as twenty + unit", +]) +def test_i_can_evaluate_bnf_concepts(expression): + sheerka = get_sheerka() + + sheerka.evaluate_user_input("def concept one as 1") + sheerka.evaluate_user_input("def concept two as 2") + sheerka.evaluate_user_input("def concept twenty as 20") + sheerka.evaluate_user_input(expression) + res = sheerka.evaluate_user_input("eval twenty one") + + assert len(res) == 1 + assert res[0].status + assert res[0].body == 21 diff --git a/tests/test_utils.py b/tests/test_utils.py index 5d00b30..5742c43 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -125,6 +125,11 @@ def test_i_can_strip_eof(): assert actual == expected +def test_i_can_escape(): + actual = core.utils.escape_char("hello 'world' my friend", "'") + assert actual == "hello \\'world\\' my friend" + + def get_tokens(lst): res = [] for e in lst: