Fixed #8

Fixed #12
Fixed #13
Fixed #14

src/parsers/ParserInput.py

@@ -0,0 +1,19 @@
+from parsers.tokenizer import Tokenizer
+
+
+class ParserInput:
+    def __init__(self, text, yield_oef=True):
+        self.original_text = text
+        self.yield_oef = yield_oef
+        self.tokens = None
+        self.exception = None
+
+    def init(self) -> bool:
+        try:
+            # the eof if forced, but will not be yield if not set to.
+            self.tokens = list(Tokenizer(self.original_text, yield_eof=True))
+            return True
+        except Exception as ex:
+            self.tokens = None
+            self.exception = ex
+            return False