kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First steps of ConceptLexer. Need to update DefaultParser before continuing

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-11-29 17:26:50 +01:00
parent 5d37addc7d
commit 5e539a4b28
21 changed files with 1409 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/ast/nodes.py
+48 -9
View File
@@ -1,6 +1,7 @@
from core.builtin_concepts import BuiltinConcepts, ListConcept
from core.concept import Concept
import ast
import core.utils
import logging
@@ -45,18 +46,18 @@ class NodeParent:
class NodeConcept(Concept):
def __init__(self, key, parent: NodeParent):
def __init__(self, key, node_type, parent: NodeParent):
super().__init__(key, True, False, key)
self.parent = parent
self.node_type = node_type
def get_node_type(self):
return self.key
return self.node_type
class GenericNodeConcept(NodeConcept):
def __init__(self, node_type, parent):
super().__init__(BuiltinConcepts.GENERIC_NODE, parent)
self.node_type = node_type
super().__init__(BuiltinConcepts.GENERIC_NODE, node_type, parent)
def __repr__(self):
return "Generic:" + self.node_type
@@ -74,17 +75,25 @@ class GenericNodeConcept(NodeConcept):
return self.body
class IdentifierConcept(NodeConcept):
class IdentifierNodeConcept(NodeConcept):
def __init__(self, parent, name):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, parent)
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Name", parent)
self.body = name
def transform(node):
class CallNodeConcept(NodeConcept):
def __init__(self, parent=None):
super().__init__(BuiltinConcepts.IDENTIFIER_NODE, "Call", parent)
def get_args_names(self, sheerka):
return sheerka.values(self.get_prop("args"))
def python_to_concept(python_node):
"""
Transform Python AST node into concept nodes
for better usage
:param node:
:param python_node:
:return:
"""
@@ -107,4 +116,34 @@ def transform(node):
concept.set_prop(field, value)
return concept
return _transform(node, None)
return _transform(python_node, None)
def concept_to_python(concept_node):
"""
Transform back concept_node to Python AST node
:param concept_node:
:return:
"""
def _transform(node):
node_type = node.get_node_type()
ast_object = core.utils.new_object("_ast." + node_type)
for field in node.props:
if field not in ast_object._fields:
continue
value = node.get_prop(field)
if isinstance(value, list) or isinstance(value, Concept) and value.key == str(BuiltinConcepts.LIST):
lst = []
for i in value:
lst.append(_transform(i))
setattr(ast_object, field, lst)
elif isinstance(value, NodeConcept):
setattr(ast_object, field, _transform(value))
else:
setattr(ast_object, field, value)
return ast_object
res = _transform(concept_node)
return res
core/ast/visitors.py
+11 -3
View File
@@ -57,13 +57,13 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
def visit_Name(self, node):
parents = get_parents(node)
if ("For", "target") in parents: # variable used by the 'for' iteration
if ("For", "target") in parents: # variable used by the 'for' iteration
return
if ("Call", "func") in parents: # name of the function
if ("Call", "func") in parents: # name of the function
return
if ("Assign", "targets") in parents: # variable which is assigned
if ("Assign", "targets") in parents: # variable which is assigned
return
if self.can_be_discarded(self.sheerka.value(node), parents):
@@ -91,6 +91,14 @@ class UnreferencedNamesVisitor(ConceptNodeVisitor):
return False
class ExtractPredicateVisitor(ConceptNodeVisitor):
def __init__(self, variable_name):
self.predicates = []
self.variable_name = variable_name
def get_parents(node):
if node.parent is None:
return []
core/builtin_helpers.py
+109
View File
@@ -1,3 +1,8 @@
import ast
import core.ast.nodes
from core.ast.nodes import CallNodeConcept, GenericNodeConcept
from core.ast.visitors import UnreferencedNamesVisitor
from core.builtin_concepts import BuiltinConcepts
@@ -81,3 +86,107 @@ def expect_one(context, return_values):
False,
sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, obj=return_values),
parents=return_values)
def get_names(sheerka, concept_node):
"""
Finds all the names referenced by the concept_node
:param sheerka:
:param concept_node:
:return:
"""
unreferenced_names_visitor = UnreferencedNamesVisitor(sheerka)
unreferenced_names_visitor.visit(concept_node)
return list(unreferenced_names_visitor.names)
def extract_predicates(sheerka, expression, variables_to_include, variables_to_exclude):
"""
from expression, tries to find all the predicates referencing a variable, and the variable only
for example
exp : isinstance(a, int) and isinstance(b, str)
will return 'isinstance(a, int)' if variable_name == 'a'
:param sheerka:
:param expression:
:param variables_to_include:
:param variables_to_exclude:
:return: list of predicates
"""
if len(variables_to_include) == 0:
return []
def _get_predicates(_nodes):
_predicates = []
for _node in _nodes:
python_node = ast.Expression(body=core.ast.nodes.concept_to_python(_node))
python_node = ast.fix_missing_locations(python_node)
_predicates.append(python_node)
return _predicates
if isinstance(expression, str):
node = ast.parse(expression, mode="eval")
else:
return NotImplementedError()
concept_node = core.ast.nodes.python_to_concept(node)
main_op = concept_node.get_prop("body")
return _get_predicates(_extract_predicates(sheerka, main_op, variables_to_include, variables_to_exclude))
def _extract_predicates(sheerka, node, variables_to_include, variables_to_exclude):
predicates = []
def _matches(_names, to_include, to_exclude):
_res = None
for n in _names:
if n in to_include and _res is None:
_res = True
if n in to_exclude:
_res = False
return _res
if node.node_type == "Compare":
if node.get_prop("left").node_type == "Name":
"""Simple case of one comparison"""
comparison_name = sheerka.value(node.get_prop("left"))
if comparison_name in variables_to_include and comparison_name not in variables_to_exclude:
predicates.append(node)
else:
"""The left part is an expression"""
res = _extract_predicates(sheerka, node.get_prop("left"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "Call":
"""Simple case predicate"""
call_node = node if isinstance(node, CallNodeConcept) else CallNodeConcept().update_from(node)
args = list(call_node.get_args_names(sheerka))
if _matches(args, variables_to_include, variables_to_exclude):
predicates.append(node)
elif node.node_type == "UnaryOp" and node.get_prop("op").node_type == "Not":
"""Simple case of negation"""
res = _extract_predicates(sheerka, node.get_prop("operand"), variables_to_include, variables_to_exclude)
if len(res) > 0:
predicates.append(node)
elif node.node_type == "BinOp":
names = get_names(sheerka, node)
if _matches(names, variables_to_include, variables_to_exclude):
predicates.append(node)
elif node.node_type == "BoolOp":
all_op = True
temp_res = []
for op in node.get_prop("values"):
res = _extract_predicates(sheerka, op, variables_to_include, variables_to_exclude)
if len(res) == 0:
all_op = False
else:
temp_res.extend(res)
if all_op:
predicates.append(node)
else:
for res in temp_res:
predicates.append(res)
return predicates
core/sheerka.py
+16 -10
View File
@@ -23,7 +23,7 @@ class Sheerka(Concept):
BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"
USER_CONCEPTS_KEYS = "User_Concepts"
def __init__(self, debug=False):
def __init__(self, debug=False, skip_builtins_in_db=False):
log.debug("Starting Sheerka.")
super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
@@ -53,6 +53,7 @@ class Sheerka(Concept):
self.parsers_prefix = None
self.debug = debug
self.skip_builtins_in_db = skip_builtins_in_db
def initialize(self, root_folder=None):
"""
@@ -109,14 +110,15 @@ class Sheerka(Concept):
if not concept.is_unique and str(key) in builtins_classes:
self.builtin_cache[key] = builtins_classes[str(key)]
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)
if from_db is None:
log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
if not self.skip_builtins_in_db:
from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.key)
if from_db is None:
log.debug(f"'{concept.name}' concept is not found in db. Adding.")
self.set_id_if_needed(concept, True)
self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
else:
log.debug(f"Found concept '{from_db}' in db. Updating.")
concept.update_from(from_db)
self.add_in_cache(concept)
@@ -125,7 +127,11 @@ class Sheerka(Concept):
Init the parsers
:return:
"""
for parser in core.utils.get_sub_classes("parsers", "parsers.BaseParser.BaseParser"):
base_class = core.utils.get_class("parsers.BaseParser.BaseParser")
for parser in core.utils.get_sub_classes("parsers", base_class):
if parser.__module__ == base_class.__module__:
continue
log.debug(f"Adding builtin parser '{parser.__name__}'")
self.parsers.append(parser)
core/tokenizer.py
+7 -5
View File
@@ -54,12 +54,14 @@ class Token:
column: int
def __repr__(self):
if type == TokenKind.IDENTIFIER:
value = "ident:" + str(self.value)
elif type == TokenKind.WHITESPACE:
value = " "
elif type == TokenKind.NEWLINE:
if self.type == TokenKind.IDENTIFIER:
value = str(self.value)
elif self.type == TokenKind.WHITESPACE:
value = "<ws>"
elif self.type == TokenKind.NEWLINE:
value = r"\n"
elif self.type == TokenKind.EOF:
value = "<EOF>"
else:
value = self.value
core/utils.py
+27 -6
View File
@@ -105,27 +105,28 @@ def get_classes_from_package(package_name):
yield c
def get_sub_classes(package_name, base_class_name):
def get_sub_classes(package_name, base_class):
pkg = __import__(package_name)
prefix = pkg.__name__ + "."
for (module_loader, name, ispkg) in pkgutil.iter_modules(pkg.__path__, prefix):
importlib.import_module(name)
base_class = get_class(base_class_name)
return base_class.__subclasses__()
base_class = get_class(base_class) if isinstance(base_class, str) else base_class
return set(base_class.__subclasses__()).union(
[s for c in base_class.__subclasses__() for s in get_sub_classes(package_name, c)])
def remove_from_list(lst, to_remove):
def remove_from_list(lst, to_remove_predicate):
"""
Removes elements from a list if they exist
:param lst:
:param to_remove:
:param to_remove_predicate:
:return:
"""
flagged = []
for item in lst:
if to_remove(item):
if to_remove_predicate(item):
flagged.append(item)
for item in flagged:
@@ -134,3 +135,23 @@ def remove_from_list(lst, to_remove):
return lst
def product(a, b):
"""
Kind of cartesian product between list a and b
knowing that a is also a list
So it's a cartesian product between a list of list and a list
"""
if a is None or len(a) == 0:
return b
if b is None or len(b) == 0:
return a
res = []
for item_b in b:
for item_a in a:
items = item_a + [item_b]
res.append(items)
return res