kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed SyaNodeParser false positive recognition issue

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-05-15 10:36:05 +02:00
parent 6e343ba996
commit 5489ef00b9
24 changed files with 484 additions and 5741 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/builtin_concepts.py
+1 -19
View File
@@ -349,9 +349,6 @@ class EnumerationConcept(Concept):
self.set_value(ConceptParts.BODY, iteration)
self.metadata.is_evaluated = True
# def __iter__(self):
# return iter(self.body)
class ListConcept(Concept):
def __init__(self, items=None):
@@ -362,21 +359,6 @@ class ListConcept(Concept):
def append(self, obj):
self.body.append(obj)
# def __len__(self):
# return len(self.body)
#
# def __getitem__(self, key):
# return self.body[key]
#
# def __setitem__(self, key, value):
# self.body[key] = value
#
# def __iter__(self):
# return iter(self.body)
#
# def __contains__(self, item):
# return item in self.body
class FilteredConcept(Concept):
def __init__(self, filtered=None, iterable=None, predicate=None):
@@ -450,5 +432,5 @@ class ExplanationConcept(Concept):
self.set_value("command", command) # explain command parameters
self.set_value("title", title) # a title to the explanation
self.set_value("instructions", instructions) # instructions for SheerkaPrint
self.set_value(ConceptParts.BODY, execution_result) # list of results
self.set_value(ConceptParts.BODY, execution_result) # list of results
self.metadata.is_evaluated = True
src/core/builtin_helpers.py
+1
View File
@@ -326,6 +326,7 @@ def ensure_evaluated(context, concept):
return evaluated
def get_lexer_nodes_from_unrecognized(context, unrecognized_tokens_node, parsers):
"""
Using parsers, try to recognize concepts from source
src/core/concept.py
+3 -3
View File
@@ -221,7 +221,7 @@ class Concept:
Create the key for this concept.
Must be called only when the concept if fully initialized
The method is not called set_key to make sure that no other class set the key by mistake
The method is not called 'set_key' to make sure that no other class set the key by mistake
:param tokens:
:return:
"""
@@ -248,8 +248,8 @@ class Concept:
if token.value in variables:
key += VARIABLE_PREFIX + str(variables.index(token.value))
else:
value = token.value[1:-1] if token.type == TokenKind.STRING else token.value
key += value
#value = token.value[1:-1] if token.type == TokenKind.STRING else token.value
key += token.value
first = False
self.metadata.key = key
src/core/sheerka/Services/SheerkaCreateNewConcept.py
-24
View File
@@ -56,12 +56,6 @@ class SheerkaCreateNewConcept:
return sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
resolved_concepts_by_first_keyword = init_ret_value.body
# update concept definition by key
# init_sya_ret_value = self.bnp.initialize(context, [concept], use_sheerka=True)
# if not init_sya_ret_value.status:
# return sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
# concepts_by_first_keyword = init_sya_ret_value.body
concept.freeze_definition_hash()
cache_manager.add_concept(concept)
@@ -74,21 +68,3 @@ class SheerkaCreateNewConcept:
# process the return if needed
ret = sheerka.ret(self.logger_name, True, sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
# def load_concepts_nodes_definitions(self, context):
# """
# Gets from sdp what is need to parse nodes
# :return:
# """
# sdp = self.sheerka.sdp
#
# concepts_by_first_keyword = sdp.get(
# self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# resolved_concepts_by_first_keyword = init_ret_value.body
#
# return concepts_by_first_keyword, resolved_concepts_by_first_keyword
src/core/sheerka/Services/SheerkaExecute.py
+4 -3
View File
@@ -1,8 +1,9 @@
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
import core.utils
from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
NO_MATCH = "** No Match **"
class SheerkaExecute:
"""
Manage the execution of a process flow
@@ -58,7 +59,8 @@ class SheerkaExecute:
# else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens"
# execution_context.log(f"Parsing {debug_text}")
with execution_context.push(desc=f"Parsing using {parser.name}", logger=parser.verbose_log) as sub_context:
with execution_context.push(desc=f"Parsing using {parser.name}",
logger=parser.verbose_log) as sub_context:
sub_context.add_inputs(to_parse=to_parse)
res = parser.parse(sub_context, to_parse)
if res is not None:
@@ -86,7 +88,6 @@ class SheerkaExecute:
stop_processing = True
sub_context.add_values(return_values=res)
if stop_processing:
break # Do not try the other priorities if a match is found
src/core/sheerka/Services/SheerkaModifyConcept.py
+1 -1
View File
@@ -35,7 +35,7 @@ class SheerkaModifyConcept:
# TODO : update concept by first keyword
# TODO : update resolved by first keyword
# TODO : update concets grammars
# TODO : update concepts grammars
ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
return ret
src/core/sheerka/Services/SheerkaVariableManager.py
-2
View File
@@ -1,8 +1,6 @@
from dataclasses import dataclass
from typing import List
from sdp.sheerkaSerializer import Serializer
@dataclass
class Variable:
src/core/sheerka/Sheerka.py
-48
View File
@@ -60,10 +60,6 @@ class Sheerka(Concept):
self.bnp = None # reference to the BaseNodeParser class (to compute first keyword token)
# # Cache for concepts grammars
# # To be shared between BNFNode parsers instances
# self.concepts_grammars = {}
# a concept can be instantiated
# ex: File is a concept, but File('foo.txt') is an instance
# TODO: manage contexts
@@ -303,27 +299,6 @@ class Sheerka(Concept):
res = self.bnp.resolve_concepts_by_first_keyword(context, concepts_by_first_keyword)
self.cache_manager.put(self.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY, False, res.body)
# sya = self.bnf.resolve_sya_associativity_and_precedence()
# self.cache_manager.put(self.RESOLVED_CONCEPTS_SYA_DEFINITION_ENTRY, sya)
#
#
# self.concepts_by_first_keyword, \
# self.resolved_concepts_by_first_keyword = \
# self.create_new_concept_handler.load_concepts_nodes_definitions(context)
# self.concepts_by_first_keyword = self.sdp.get_safe(
# self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
# load_origin=False) or {}
#
# self.sya_definitions = self.sdp.get_safe(
# self.CONCEPTS_SYA_DEFINITION_ENTRY,
# load_origin=False) or {}
#
# init_ret_value = self.bnp.resolve_concepts_by_first_keyword(self, self.concepts_by_first_keyword)
# if not init_ret_value.status:
# return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
# self.resolved_concepts_by_first_keyword = init_ret_value.body
def reset(self, cache_only=False):
self.cache_manager.clear()
self.cache_manager.cache_only = cache_only
@@ -346,7 +321,6 @@ class Sheerka(Concept):
with ExecutionContext(self.key, event, self, f"Evaluating '{text}'", self.log) as execution_context:
user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
# execution_context.local_hints.add(BuiltinConcepts.EVAL_WHERE_REQUESTED)
steps = [
BuiltinConcepts.BEFORE_PARSING,
@@ -525,28 +499,6 @@ class Sheerka(Concept):
return concept
#
# def get(self, concept_key, concept_id=None):
# """
# Tries to find a concept
# What is return must be used a template for another concept.
# You must not modify the returned concept
# :param concept_key: key of the concept
# :param concept_id: when multiple concepts with the same key, use the id
# :return:
# """
#
# by_key = self.get_by_key(concept_key)
# if self.is_known(by_key):
# return by_key
#
# # else return by name
# by_name = self.get_by_name(concept_key)
# if self.is_known(by_name):
# return by_name
#
# return by_key # return not found for key
def get_by_key(self, concept_key, concept_id=None):
concept_key = str(concept_key) if isinstance(concept_key, BuiltinConcepts) else concept_key
return self.internal_get("key", concept_key, self.CONCEPTS_BY_KEY_ENTRY, concept_id)
src/core/tokenizer.py
+33 -3
View File
@@ -1,4 +1,4 @@
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
@@ -48,6 +48,7 @@ class TokenKind(Enum):
DEGREE = "degree" # °
WORD = "word"
EQUALSEQUALS = "=="
VAR_DEF = "__var__"
@dataclass()
@@ -58,6 +59,8 @@ class Token:
line: int
column: int
_str_value: str = field(default=None, repr=False, compare=False, hash=None)
def __repr__(self):
if self.type == TokenKind.IDENTIFIER:
value = str(self.value)
@@ -72,6 +75,23 @@ class Token:
return f"Token({value})"
@property
def str_value(self):
if self._str_value:
return self._str_value
if self.type == TokenKind.STRING:
self._str_value = self.value[1:-1]
elif self.type == TokenKind.KEYWORD:
self._str_value = self.value.value
else:
self._str_value = str(self.value)
return self._str_value
@staticmethod
def is_whitespace(token):
return token and token.type == TokenKind.WHITESPACE
@dataclass()
class LexerError(Exception):
@@ -101,12 +121,13 @@ class Tokenizer:
KEYWORDS = set(x.value for x in Keywords)
def __init__(self, text, parse_word=False):
def __init__(self, text, yield_eof=True, parse_word=False):
self.text = text
self.text_len = len(text)
self.column = 1
self.line = 1
self.i = 0
self.yield_eof = yield_eof
self.parse_word = parse_word
def __iter__(self):
@@ -134,6 +155,7 @@ class Tokenizer:
self.i += 1
self.column += 1
elif c == "_":
from core.concept import VARIABLE_PREFIX
if self.i + 1 < self.text_len and self.text[self.i + 1].isalpha():
identifier = self.eat_identifier(self.i)
token_type = TokenKind.KEYWORD if identifier in self.KEYWORDS else TokenKind.IDENTIFIER
@@ -141,6 +163,13 @@ class Tokenizer:
yield Token(token_type, value, self.i, self.line, self.column)
self.i += len(identifier)
self.column += len(identifier)
elif self.i + 7 < self.text_len and \
self.text[self.i: self.i + 7] == VARIABLE_PREFIX and \
self.text[self.i + 7].isdigit():
number = self.eat_number(self.i + 7)
yield Token(TokenKind.VAR_DEF, VARIABLE_PREFIX + number, self.i, self.line, self.column)
self.i += 7 + len(number)
self.column += 7 + len(number)
else:
yield Token(TokenKind.UNDERSCORE, "_", self.i, self.line, self.column)
self.i += 1
@@ -308,7 +337,8 @@ class Tokenizer:
else:
raise LexerError(f"Unknown token '{c}'", self.text, self.i, self.line, self.column)
yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
if self.yield_eof:
yield Token(TokenKind.EOF, "", self.i, self.line, self.column)
def eat_concept(self, start, line, column):
key, id, buffer = None, None, ""