diff --git a/src/core/concept.py b/src/core/concept.py index 11e08e0..eefb85d 100644 --- a/src/core/concept.py +++ b/src/core/concept.py @@ -89,7 +89,7 @@ class Concept: self._all_attrs = None def __repr__(self): - text = f"(Concept {self._metadata.name}#{self._metadata.id}" + text = f"Concept({self._metadata.name}#{self._metadata.id}" if self._metadata.pre: text += f", #pre={self._metadata.pre}" diff --git a/src/core/error.py b/src/core/error.py index c9bcd13..7d6eb3d 100644 --- a/src/core/error.py +++ b/src/core/error.py @@ -18,6 +18,20 @@ class MethodAccessError(SheerkaException): return f"Cannot access method '{self.method_name}'" +class NotEnoughParameters(SheerkaException): + """ + Exception when not enough parameters are found during Sya parsing + """ + + def __init__(self, concept_to_recognize, expected_nb_parameters, nb_parameters_found): + self.concept = concept_to_recognize + self.expected = expected_nb_parameters + self.found = nb_parameters_found + + def get_error_msg(self) -> str: + return f"Failed to parse {self.concept}. Expecting {self.expected} parameters, but only found {self.found}." + + @dataclass class ErrorObj: def get_error_msg(self) -> str: diff --git a/src/evaluators/base_evaluator.py b/src/evaluators/base_evaluator.py index 63e9571..17609e9 100644 --- a/src/evaluators/base_evaluator.py +++ b/src/evaluators/base_evaluator.py @@ -102,6 +102,13 @@ class MultipleChoices: return True + def __iadd__(self, other): + if not isinstance(other, MultipleChoices): + raise TypeError(f"unsupported operand type(s) for +=: 'MultipleChoices' and '{type(other)}'") + + self.items += other.items + return self + def __hash__(self): return hash(tuple(self.items)) diff --git a/src/parsers/BaseParser.py b/src/parsers/BaseParser.py new file mode 100644 index 0000000..aa94933 --- /dev/null +++ b/src/parsers/BaseParser.py @@ -0,0 +1,21 @@ +from core.ExecutionContext import ExecutionContext +from parsers.ParserInput import ParserInput + + +class BaseParser: + """ + Base class for parser than can be used in concept recognition + """ + + def __init__(self, name): + self.name = name # name of the parser + + def parse(self, context: ExecutionContext, parser_input: ParserInput, error_sink: list): + """ + Default signature for parsing + :param context: + :param parser_input: + :param error_sink: + :return: + """ + pass diff --git a/src/parsers/ParserInput.py b/src/parsers/ParserInput.py index 611d029..a9f1aff 100644 --- a/src/parsers/ParserInput.py +++ b/src/parsers/ParserInput.py @@ -100,5 +100,20 @@ class ParserInput: return res + @staticmethod + def from_tokens(tokens, text=None): + """ + returns a parser input, given already computed tokens + :param tokens: + :param text: + :return: + """ + res = ParserInput(None) + res.all_tokens = tokens + res.original_text = text or get_text_from_tokens(tokens) + res.pos = -1 + res.end = len(res.all_tokens) + return res + def __repr__(self): return f"ParserInput('{self.original_text}', len={len(self.all_tokens)})" diff --git a/src/parsers/SimpleConceptsParser.py b/src/parsers/SimpleConceptsParser.py index e123803..5756970 100644 --- a/src/parsers/SimpleConceptsParser.py +++ b/src/parsers/SimpleConceptsParser.py @@ -1,11 +1,62 @@ from core.concept import DefinitionType from evaluators.base_evaluator import MultipleChoices -from parsers.state_machine import ConceptToRecognize, End, ManageUnrecognized, MetadataToken, PrepareReadTokens, \ - ReadConcept, ReadTokens, Start, StateMachine, StateMachineContext, UnrecognizedToken +from parsers.BaseParser import BaseParser +from parsers.parser_utils import UnexpectedEof, UnexpectedToken, get_text_from_tokens +from parsers.state_machine import ConceptToRecognize, End, MetadataToken, PrepareReadTokens, \ + ReadTokens, Start, State, StateMachine, StateMachineContext, StateResult, UnrecognizedToken from parsers.tokenizer import Token, TokenKind, Tokenizer -class SimpleConceptsParser: +class ReadConcept(State): + def run(self, state_context) -> StateResult: + start = state_context.parser_input.pos + + for expected in state_context.concept_to_recognize.expected: + if not state_context.parser_input.next_token(False): + # eof before the concept is recognized + state_context.errors.append(UnexpectedEof(expected, state_context.parser_input.token)) + state_context.concept_to_recognize = None + return StateResult(self.next_states[0]) + + token = state_context.parser_input.token + if token.value != expected: + # token mismatch + state_context.errors.append(UnexpectedToken(token, expected)) + state_context.concept_to_recognize = None + return StateResult(self.next_states[0]) + + state_context.result.append(MetadataToken(state_context.concept_to_recognize.metadata, + start, + state_context.parser_input.pos, + state_context.concept_to_recognize.resolution_method, + "simple")) + + state_context.concept_to_recognize = None + return StateResult(self.next_states[0]) + + +class ManageUnrecognized(State): + def run(self, state_context) -> StateResult: + if state_context.buffer: + buffer_as_str = get_text_from_tokens(state_context.buffer) + if len(state_context.result) > 0 and isinstance(old := state_context.result[-1], UnrecognizedToken): + # merge unrecognized if needed + state_context.result[-1] = UnrecognizedToken(old.buffer + buffer_as_str, + old.start, + state_context.parser_input.pos - 1) + else: + state_context.result.append(UnrecognizedToken(buffer_as_str, + state_context.buffer_start_pos, + state_context.parser_input.pos - 1)) + + # clear the buffer + state_context.buffer.clear() + state_context.buffer_start_pos = state_context.parser_input.pos + 1 + + return StateResult(self.next_states[0]) + + +class SimpleConceptsParser(BaseParser): """" This class is to parse concepts with no parameter ex : def concept I am a new concept @@ -13,6 +64,8 @@ class SimpleConceptsParser: """ def __init__(self): + super().__init__("simple") + tokens_wkf = { Start("start", next_states=["prepare read tokens"]), PrepareReadTokens("prepare read tokens", next_states=["read tokens"]), @@ -31,7 +84,6 @@ class SimpleConceptsParser: "#tokens_wkf": {t.name: t for t in tokens_wkf}, "#concept_wkf": {t.name: t for t in concept_wkf}, } - self.error_sink = [] @staticmethod def get_metadata_from_first_token(context, token: Token): @@ -56,12 +108,13 @@ class SimpleConceptsParser: return concepts_by_key + concepts_by_name - def parse(self, context, parser_input): + def parse(self, context, parser_input, error_sink): sm = StateMachine(self.workflows) - sm_context = StateMachineContext(context, parser_input, self.get_metadata_from_first_token) + sm_context = StateMachineContext(context, parser_input, self.get_metadata_from_first_token, []) sm.run("#tokens_wkf", "start", sm_context) selected = self.select_best_paths(sm) + error_sink.extend(sm_context.errors) return MultipleChoices(selected) diff --git a/src/parsers/SyaConceptsParser.py b/src/parsers/SyaConceptsParser.py index 8877cdd..07f0b3a 100644 --- a/src/parsers/SyaConceptsParser.py +++ b/src/parsers/SyaConceptsParser.py @@ -1,11 +1,163 @@ from core.concept import DefinitionType -from parsers.state_machine import ConceptToRecognize, End, ManageUnrecognized, PrepareReadTokens, ReadConcept, \ - ReadTokens, Start, \ - StateMachine, StateMachineContext +from core.error import NotEnoughParameters +from evaluators.base_evaluator import MultipleChoices +from parsers.BaseParser import BaseParser +from parsers.ParserInput import ParserInput +from parsers.SimpleConceptsParser import SimpleConceptsParser +from parsers.parser_utils import UnexpectedEof, UnexpectedToken, get_text_from_tokens +from parsers.state_machine import ConceptToRecognize, ConceptToken, End, PrepareReadTokens, ReadTokens, Start, State, \ + StateMachine, \ + StateMachineContext, StateResult, UnrecognizedToken from parsers.tokenizer import Token, TokenKind, Tokenizer -class SyaConceptsParser: +class InitConceptParsing(State): + """ + A new concept is detected + Add some validations and prepare the list of expected tokens to read + """ + + def must_pop(self, current_concept, previous_concept): + return False + + def apply_shunting_yard_algorithm(self, state_context): + """ + Apply the sya + for all concepts in the stack + Check the precedence to define the concept must be popped (to result) or not + :param state_context: + :type state_context: + :return: + :rtype: + """ + if len(state_context.stack) > 0: + while self.must_pop(state_context.concept_to_recognize.metadata, state_context.stack[-1].metadata): + state_context.parameters.append(state_context.stack.pop()) + + state_context.stack.append(state_context.concept_to_recognize) + + def run(self, state_context) -> StateResult: + expected = state_context.concept_to_recognize.expected + + # check that there is enough parameters + if len(state_context.parameters) < expected[0][1]: + raise NotEnoughParameters(state_context.concept_to_recognize, + expected[0][1], + len(state_context.parameters)) + + # remove white space before the first token if any + if expected[0][0][0].type == TokenKind.WHITESPACE: + expected[0][0].pop(0) + + # pop the first token (as it is already recognized) + expected[0][0].pop(0) + + # apply shunting yard algorithm + self.apply_shunting_yard_algorithm(state_context) + + return StateResult(self.next_states[0]) + + +class ReadConcept(State): + """ + This state reads the tokens of the concepts that are known (that are not parameters) + For example, given the concept 'let me create the concept x' + We will parse 'let' 'me' 'create' 'the' 'concept' + But we will not parse 'x' because it's a parameter + """ + + def run(self, state_context) -> StateResult: + + expected = state_context.concept_to_recognize.expected + + # eat the tokens + for expected_token in expected[0][0]: + if not state_context.parser_input.next_token(skip_whitespace=False): + # Failed to recognize concept because of eof + state_context.errors.append(UnexpectedEof(expected_token, None)) + return StateResult("error eof") + + token = state_context.parser_input.token + if expected_token.type != token.type or expected_token.value != token.value: + # Failed to recognize concept because of token mismatch + state_context.errors.append(UnexpectedToken(token, expected_token)) + return StateResult("token mismatch") + + expected.pop(0) + if not expected: + state_context.concept_to_recognize = None + return StateResult("finalize concept") + else: + return StateResult("read parameters") + + +class ReadParameters(State): + def run(self, state_context) -> StateResult: + assert not state_context.buffer + + if not state_context.parser_input.next_token(False): + return StateResult("finalize concept") + + state_context.buffer.append(state_context.parser_input.token) + + +class ManageUnrecognized(State): + def run(self, state_context) -> StateResult: + if state_context.buffer: + buffer_as_str = get_text_from_tokens(state_context.buffer) + res = MultipleChoices([]) + pi = ParserInput.from_tokens(state_context.buffer, text=buffer_as_str) + error_sink = [] + + # Try to parse the buffer + for parser in state_context.other_parsers: + res += parser.parse(state_context.context, pi, error_sink) + + if error_sink: + raise NotImplemented("Cannot manage errors") + + if len(res.items) == 0: + state_context.parameters.append(UnrecognizedToken(buffer_as_str, + state_context.buffer_start_pos, + state_context.parser_input.pos - 1)) + + elif len(res.items) == 1: + state_context.parameters.append(ConceptToken(res.items[0], + state_context.buffer_start_pos, + state_context.parser_input.pos - 1)) + + else: + raise NotImplemented("Cannot manage multiple results") + + # clear the buffer + state_context.buffer.clear() + state_context.buffer_start_pos = state_context.parser_input.pos + 1 + + return StateResult(self.next_states[0]) + + +class TokenMismatch(State): + """ + When we realize that we are not parsing the correct concept + """ + pass + + +class ErrorEof(State): + """ + When EOF (end of file) detected before successfully parsing the concept + """ + pass + + +class FinalizeConceptParsing(State): + """ + The concept is fully parsed. Let's wrap up + """ + pass + + +class SyaConceptsParser(BaseParser): """" This class is to parse concepts with parameter ex : def concept a plus b as a + b @@ -13,6 +165,8 @@ class SyaConceptsParser: """ def __init__(self): + super().__init__("sya") + tokens_wkf = { Start("start", next_states=["prepare read tokens"]), PrepareReadTokens("prepare read tokens", next_states=["read tokens"]), @@ -23,8 +177,16 @@ class SyaConceptsParser: } concept_wkf = { - Start("start", next_states=["read concept"]), - ReadConcept("read concept", next_states=["#tokens_wkf"]), + Start("start", next_states=["init concept parsing"]), + InitConceptParsing("init concept parsing", ["manage parameters"]), + ManageUnrecognized("manage parameters", next_states=["read concept"]), + ReadConcept("read concept", next_states=["finalize concept", "eof", "wrong concept", "read parameters"]), + ReadParameters("read parameters", next_states=["manage parameters", "eof"]), + ManageUnrecognized("eof", next_states=["end"]), + FinalizeConceptParsing("finalize concept", next_states=["#tokens_wkf"]), + ErrorEof("eof", ["end"]), + TokenMismatch("token mismatch", ["end"]), + End("end", next_states=None) } self.workflows = { @@ -104,8 +266,12 @@ class SyaConceptsParser: for m in context.sheerka.get_metadatas_from_first_token("key", token.value) if m.definition_type == DefinitionType.DEFAULT and len(m.parameters) > 0] - def parse(self, context, parser_input): + def parse(self, context, parser_input, error_sink): sm = StateMachine(self.workflows) - sm_context = StateMachineContext(context, parser_input, self.get_metadata_from_first_token) + sm_context = StateMachineContext(context, + parser_input, + self.get_metadata_from_first_token, + [SimpleConceptsParser()]) sm.run("#tokens_wkf", "start", sm_context) - pass + + error_sink.extend(sm_context.errors) diff --git a/src/parsers/state_machine.py b/src/parsers/state_machine.py index b92cd95..466ca8d 100644 --- a/src/parsers/state_machine.py +++ b/src/parsers/state_machine.py @@ -3,23 +3,23 @@ from typing import Any, Literal from common.utils import str_concept from core.ExecutionContext import ExecutionContext -from core.concept import ConceptMetadata +from core.concept import Concept, ConceptMetadata from parsers.ParserInput import ParserInput -from parsers.parser_utils import UnexpectedEof, UnexpectedToken, get_text_from_tokens from parsers.tokenizer import Token @dataclass class MetadataToken: """ - Class that represents a text that is recognized as a concept + When a concept definition is recognized We keep track of the start and the end position + MetadataToken is a shortcut for ConceptMetadataToken """ - metadata: ConceptMetadata - start: int - end: int - resolution_method: Literal["name", "key", "id"] - parser: str + metadata: ConceptMetadata # concept that is recognized + start: int # start position in the texts + end: int # end position + resolution_method: Literal["name", "key", "id"] # did we use the name, the id or the key to recognize the concept + parser: str # which parser recognized the concept (SimpleConcepts, Sya, ...) def __repr__(self): return f"(MetadataToken metadata={str_concept(self.metadata, drop_name=True)}, " + \ @@ -41,7 +41,7 @@ class MetadataToken: @dataclass class UnrecognizedToken: """ - Class that represents a text that is not recognized + Class that represents a text that is not recognized (yet) We keep track of the start and the end position """ buffer: str @@ -49,6 +49,17 @@ class UnrecognizedToken: end: int +@dataclass +class ConceptToken: + """ + When an already defined concept is found during the parsing + We keep track of the start and the end position + """ + concept: Concept + start: int # start position in the texts + end: int # end position + + @dataclass class StateResult: next_state: str | None @@ -59,30 +70,57 @@ class StateResult: class ConceptToRecognize: """ Holds information about the concept to recognize + During the parsing, we have a hint on a concept, But we need to finish the parsing to make sure that we are right """ metadata: ConceptMetadata - expected_tokens: Any + expected: list[tuple] resolution_method: Literal["name", "key", "id"] # which attribute was used to resolve the concept + def __repr__(self): + return f"ConceptToRecognize(#{self.metadata.id}, expected={self.expected})" + @dataclass class StateMachineContext: + """ + Internal state of a state machine + """ + # initialization context: ExecutionContext parser_input: ParserInput - get_metadata_from_first_token: Any + get_metadata_from_first_token: Any # This is a callback that gives the possible concepts, for a token + other_parsers: list # parsers to call when managing unrecognized tokens + + # attributes used when parsing token + # tokens currently being read buffer: list[Token] = field(default_factory=list) buffer_start_pos: int = -1 + + # attributes used when parsing concept + # parameters already recognized + Concept under recognition concept_to_recognize: ConceptToRecognize | None = None - result: list = field(default_factory=list) - errors: list = field(default_factory=list) + stack: list = field(default_factory=list) + parameters: list = field(default_factory=list) # it is called 'output' in shunting yard explanations + + # runtime info + result: list = field(default_factory=list) # list of tokens found + errors: list = field(default_factory=list) # error sink def get_clones(self, concepts_to_recognize): + """ + Helper function that clone the context when multiple concepts are found + :param concepts_to_recognize: + :return: + """ return [StateMachineContext(self.context, self.parser_input.clone(), self.get_metadata_from_first_token, + self.other_parsers, self.buffer.copy(), self.buffer_start_pos, concept, + self.stack.copy(), + self.parameters.copy(), self.result.copy(), self.errors.copy()) for concept in concepts_to_recognize] @@ -152,50 +190,6 @@ class ReadTokens(State): return StateResult(self.name, forks) -class ManageUnrecognized(State): - def run(self, state_context) -> StateResult: - if state_context.buffer: - buffer_as_str = get_text_from_tokens(state_context.buffer) - if len(state_context.result) > 0 and isinstance(old := state_context.result[-1], UnrecognizedToken): - state_context.result[-1] = UnrecognizedToken(old.buffer + buffer_as_str, - old.start, - state_context.parser_input.pos - 1) - else: - state_context.result.append(UnrecognizedToken(buffer_as_str, - state_context.buffer_start_pos, - state_context.parser_input.pos - 1)) - - return StateResult(self.next_states[0]) - - -class ReadConcept(State): - def run(self, state_context) -> StateResult: - start = state_context.parser_input.pos - - for expected in state_context.concept_to_recognize.expected_tokens: - if not state_context.parser_input.next_token(False): - # eof before the concept is recognized - state_context.errors.append(UnexpectedEof(expected, state_context.parser_input.token)) - state_context.concept_to_recognize = None - return StateResult(self.next_states[0]) - - token = state_context.parser_input.token - if token.value != expected: - # token mismatch - state_context.errors.append(UnexpectedToken(token, expected)) - state_context.concept_to_recognize = None - return StateResult(self.next_states[0]) - - state_context.result.append(MetadataToken(state_context.concept_to_recognize.metadata, - start, - state_context.parser_input.pos, - state_context.concept_to_recognize.resolution_method, - "simple")) - - state_context.concept_to_recognize = None - return StateResult(self.next_states[0]) - - class End(State): def run(self, state_context) -> StateResult: return StateResult(None) diff --git a/src/services/SheerkaConceptManager.py b/src/services/SheerkaConceptManager.py index 92f3ec7..2dc1f56 100644 --- a/src/services/SheerkaConceptManager.py +++ b/src/services/SheerkaConceptManager.py @@ -67,7 +67,7 @@ class ConceptManager(BaseService): You can define new concept, modify or delete them There are also function to help retrieve them easily (like first token cache) - Already instantiated concepts are managed by the Memory service + Already instantiated concepts are managed by the SheerkaMemory service, not here """ NAME = "ConceptManager" diff --git a/src/services/SheerkaDummyEventManager.py b/src/services/SheerkaDummyEventManager.py index 9401103..727b336 100644 --- a/src/services/SheerkaDummyEventManager.py +++ b/src/services/SheerkaDummyEventManager.py @@ -6,7 +6,7 @@ from services.BaseService import BaseService class SheerkaDummyEventManager(BaseService): """ Manage simple publish and subscribe functions - Need to be replaced by a standard in the industry (Redis?) + Need to be replaced by a standard in the industry (Kafka, Redis?) """ NAME = "DummyEventManager" diff --git a/tests/core/test_concept.py b/tests/core/test_concept.py index d04f3e9..23f2898 100644 --- a/tests/core/test_concept.py +++ b/tests/core/test_concept.py @@ -95,10 +95,10 @@ def test_i_cannot_get_an_attribute_which_is_not_defined(): def test_i_can_repr_a_concept(): next_id = GetNextId() foo = get_concept("foo", sequence=next_id) - assert repr(foo) == "(Concept foo#1001)" + assert repr(foo) == "Concept(foo#1001)" bar = get_concept("bar", pre="is an int", sequence=next_id) - assert repr(bar) == "(Concept bar#1002, #pre=is an int)" + assert repr(bar) == "Concept(bar#1002, #pre=is an int)" baz = get_concept("baz", definition="add a b", variables=["a", "b"], sequence=next_id) - assert repr(baz) == "(Concept baz#1003, a=**NotInit**, b=**NotInit**)" + assert repr(baz) == "Concept(baz#1003, a=**NotInit**, b=**NotInit**)" diff --git a/tests/helpers.py b/tests/helpers.py index 32d1f2a..2385691 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -45,7 +45,8 @@ def get_concept(name=None, body=None, is_builtin=False, is_unique=False, autouse=False, - sequence=None) -> Concept: + sequence=None, + init_parameters=True) -> Concept: """ Create a Concept objet Caution : 'id' and 'key' are not initialized @@ -115,6 +116,10 @@ def get_concept(name=None, body=None, else: metadata.digest = ConceptManager.compute_metadata_digest(metadata) metadata.all_attrs = ConceptManager.compute_all_attrs(metadata.variables) + + if init_parameters and metadata.variables: + metadata.parameters = [v[0] if isinstance(v, tuple) else v for v in metadata.variables] + return Concept(metadata) @@ -355,13 +360,11 @@ def get_concepts(context: ExecutionContext, *concepts, **kwargs) -> list[Concept """ Simple and quick way to get initialize concepts for a test :param context: - :type context: - :param concepts: - :type concepts: - :param kwargs: - :type kwargs: - :return: - :rtype: + :param concepts: Concepts to create + :param kwargs: named parameters to tweak the creation of the concepts + use_sheerka : Adds the new concepts to Sheerka. If not simply creates concepts that do not affect Sheerka + sequence : Sequence Manager, to give a correct id to the created concepts + :return: the concepts """ res = [] use_sheerka = kwargs.pop("use_sheerka", False) diff --git a/tests/parsers/test_SimpleConceptsParser.py b/tests/parsers/test_SimpleConceptsParser.py index b69a428..79c0c26 100644 --- a/tests/parsers/test_SimpleConceptsParser.py +++ b/tests/parsers/test_SimpleConceptsParser.py @@ -28,10 +28,11 @@ class TestSimpleConceptsParser(BaseTest): get_concepts(context, "I", "I am", "I am a new concept", use_sheerka=True) pi = get_parser_input(text) - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) assert res == MultipleChoices([expected]) - assert not parser.error_sink + assert not error_sink @pytest.mark.parametrize("text, expected", [ ("foo", [_mt("1001", 0, 0)]), @@ -42,10 +43,11 @@ class TestSimpleConceptsParser(BaseTest): get_concepts(context, get_metadata(name="foo", definition="I am a new concept"), use_sheerka=True) pi = get_parser_input(text) - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) assert res == MultipleChoices([expected]) - assert not parser.error_sink + assert not error_sink @pytest.mark.parametrize("text, expected", [ ("long concept name", [_mt("1001", 0, 4)]), @@ -57,17 +59,19 @@ class TestSimpleConceptsParser(BaseTest): use_sheerka=True) pi = get_parser_input(text) - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) assert res == MultipleChoices([expected]) - assert not parser.error_sink + assert not error_sink def test_i_can_parse_a_sequence_of_concept(self, context, parser): with NewOntology(context, "test_i_can_parse_a_sequence_of_concept"): get_concepts(context, "foo bar", "baz", "qux", use_sheerka=True) pi = get_parser_input("foo bar baz foo, qux") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) expected = [_mt("1001", 0, 2), _ut(" ", 3, 3), @@ -76,40 +80,43 @@ class TestSimpleConceptsParser(BaseTest): _mt("1003", 9, 9)] assert res == MultipleChoices([expected]) - assert not parser.error_sink + assert not error_sink def test_i_can_detect_multiple_choices(self, context, parser): with NewOntology(context, "test_i_can_detect_multiple_choices"): get_concepts(context, "foo bar", "bar baz", use_sheerka=True) pi = get_parser_input("foo bar baz") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) expected1 = [_mt("1001", 0, 2), _ut(" baz", 3, 4)] expected2 = [_ut("foo ", 0, 1), _mt("1002", 2, 4)] assert res == MultipleChoices([expected1, expected2]) - assert not parser.error_sink + assert not error_sink def test_i_can_detect_multiple_choices_2(self, context, parser): with NewOntology(context, "test_i_can_detect_multiple_choices_2"): get_concepts(context, "one two", "one", "two", use_sheerka=True) pi = get_parser_input("one two") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) expected1 = [_mt("1001", 0, 2)] expected2 = [_mt("1002", 0, 0), _ut(" ", 1, 1), _mt("1003", 2, 2)] assert res == MultipleChoices([expected1, expected2]) - assert not parser.error_sink + assert not error_sink def test_i_can_detect_multiple_choices_3(self, context, parser): with NewOntology(context, "test_i_can_detect_multiple_choices_2"): get_concepts(context, "one two", "one", "two", use_sheerka=True) pi = get_parser_input("one two xxx one two") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) e1 = get_from(_mt("c:one two#1001:"), _ut(" xxx "), _mt("c:#1001:")) e2 = get_from(_mt("c:one#1002:"), _ut(" "), _mt("c:two#1003:"), _ut(" xxx "), _mt("c:one two#1001:")) @@ -118,11 +125,12 @@ class TestSimpleConceptsParser(BaseTest): _mt("c:#1003:")) assert res == MultipleChoices([e1, e2, e3, e4]) - assert not parser.error_sink + assert not error_sink def test_nothing_is_return_is_no_concept_is_recognized(self, context, parser): pi = get_parser_input("one two three") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) assert res == MultipleChoices([]) @@ -131,12 +139,12 @@ class TestSimpleConceptsParser(BaseTest): get_concepts(context, "foo", "i am a concept", use_sheerka=True) pi = get_parser_input("foo.attribute") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) expected = [_mt("1001", 0, 0), _ut(".attribute", 1, 2)] assert res == MultipleChoices([expected]) pi = get_parser_input("i am a concept.attribute") - res = parser.parse(context, pi) + res = parser.parse(context, pi, error_sink) expected = [_mt("1002", 0, 6), _ut(".attribute", 7, 8)] assert res == MultipleChoices([expected]) - diff --git a/tests/parsers/test_SyaConceptsParser.py b/tests/parsers/test_SyaConceptsParser.py index 753863d..9f329b5 100644 --- a/tests/parsers/test_SyaConceptsParser.py +++ b/tests/parsers/test_SyaConceptsParser.py @@ -29,13 +29,65 @@ class TestSyaConceptsParser(BaseTest): with comparable_tokens(): assert actual == resolved_expected_list - def test_i_can_parse_a_simple_case(self, context, parser): + @pytest.mark.parametrize("concept", [ + get_concept("a plus b", variables=["a", "b"]), + get_concept("add a b", variables=["a", "b"]), + get_concept("a b add", variables=["a", "b"]), + ]) + def test_i_can_parse_a_simple_case(self, context, parser, concept): with NewOntology(context, "test_i_can_parse_a_simple_case"): - get_concepts(context, get_concept("a plus b", variables=["a", "b"]), use_sheerka=True) + get_concepts(context, concept, use_sheerka=True) pi = get_parser_input("1 plus 2") - res = parser.parse(context, pi) + error_sink = [] + res = parser.parse(context, pi, error_sink) expected = [_mt("1001", a="1 ", b=" 2")] assert res == MultipleChoices([expected]) - assert not parser.error_sink + assert not error_sink + + def test_i_can_parse_long_names_concept(self, context, parser): + with NewOntology(context, "test_i_can_parse_a_simple_case"): + get_concepts(context, get_concept("a long named concept b", variables=["a", "b"]), use_sheerka=True) + + pi = get_parser_input("1 long named concept 2") + error_sink = [] + res = parser.parse(context, pi, error_sink) + + expected = [_mt("1001", a="1 ", b=" 2")] + assert res == MultipleChoices([expected]) + assert not error_sink + + def test_i_can_parse_sequence(self, context, parser): + with NewOntology(context, "test_i_can_parse_sequence"): + get_concepts(context, get_concept("a plus b", variables=["a", "b"]), use_sheerka=True) + + pi = get_parser_input("1 plus 2 3 plus 7") + error_sink = [] + res = parser.parse(context, pi, error_sink) + + expected = [[_mt("1001", a="1 ", b=" 2")], [_mt("1001", a=" 3 ", b=" 7")]] + assert res == MultipleChoices(expected) + assert not error_sink + + def test_not_enough_parameters(self, context, parser): + with NewOntology(context, "test_not_enough_parameters"): + get_concepts(context, get_concept("a plus b", variables=["a", "b"]), use_sheerka=True) + + pi = get_parser_input("1 plus 2 3 plus 7") + error_sink = [] + res = parser.parse(context, pi, error_sink) + + expected = [[_mt("1001", a="1 ", b=" 2")], [_mt("1001", a=" 3 ", b=" 7")]] + assert res == MultipleChoices(expected) + assert not error_sink + + def test_i_can_detect_when_name_does_not_match(self, context, parser): + with NewOntology(context, "test_i_can_detect_when_name_does_not_match"): + get_concepts(context, get_concept("a long named concept b", variables=["a", "b"]), use_sheerka=True) + + pi = get_parser_input("1 long named mismatch 2") + error_sink = [] + res = parser.parse(context, pi, error_sink) + + assert error_sink