First version of DefaultDataFilter

src/core/Expando.py

@@ -50,6 +50,9 @@ class Expando:
   def to_dict(self, mappings: dict) -> dict:
     return {prop_name: self.get(path) for path, prop_name in mappings.items() if prop_name is not None}
   
+  def __hasattr__(self, item):
+    return item in self._props
+  
   def __repr__(self):
     if "key" in self._props:
       return f"Expando(key={self._props["key"]})"

src/core/utils.py

@@ -1,3 +1,4 @@
+import ast
 import base64
 import hashlib
 import importlib
@@ -417,3 +418,50 @@ def split_host_port(url):
       port = None
   
   return host, port
+
+
+class UnreferencedNamesVisitor(ast.NodeVisitor):
+  """
+  Try to find symbols that will be requested by the ast
+  It can be variable names, but also function names
+  """
+  
+  def __init__(self):
+    self.names = set()
+  
+  def get_names(self, node):
+    self.visit(node)
+    return self.names
+  
+  def visit_Name(self, node):
+    self.names.add(node.id)
+  
+  def visit_For(self, node: ast.For):
+    self.visit_selected(node, ["body", "orelse"])
+  
+  def visit_selected(self, node, to_visit):
+    """Called if no explicit visitor function exists for a node."""
+    for field in to_visit:
+      value = getattr(node, field)
+      if isinstance(value, list):
+        for item in value:
+          if isinstance(item, ast.AST):
+            self.visit(item)
+      elif isinstance(value, ast.AST):
+        self.visit(value)
+  
+  def visit_Call(self, node: ast.Call):
+    self.visit_selected(node, ["args", "keywords"])
+  
+  def visit_keyword(self, node: ast.keyword):
+    """
+    Keywords are parameters that are defined with a double star (**) in function / method definition
+    ex: def fun(positional, *args, **keywords)
+    :param node:
+    :type node:
+    :return:
+    :rtype:
+    """
+    self.names.add(node.arg)
+    self.visit_selected(node, ["value"])
+