Rewrote the lexer, parser, generator, and linker (#2)

README.md

@@ -83,12 +83,8 @@ Makes all `@mega` units move to the processor
 * for loops \
   `for (i = 0; i < 10; i += 1) { print(i) }`
 * break / continue
-* imports \
-  `%library.mpp`
 * native functions \
   `ubind(@mega)`
-* native code \
-  `.op atan a b c`
 
 ## Native functions:
 * read `result`, `cell`, `position`

examples/connected_list.mpp

@@ -1,7 +1,5 @@
 for (i = 0; i < @links; i += 1) {
-    getlink(building, i)
-
-    print(building)
+    print(getlink(i))
     print("\n")
 }
 

mlogpp/__main__.py

@@ -1,19 +1,33 @@
-from .lexer import Lexer
-from .preprocess import Preprocessor
-from .parser_ import Parser
+# from .lexer import Lexer
+# from .preprocess import Preprocessor
+# from .parser_ import Parser
+# from .error import MlogError
+# from .linker import Linker
+# from .optimizer import Optimizer
 
-tokens = Lexer.lex(Preprocessor.preprocess("""
-print.ab("Hello, World!")
-const _2a = (cell1[1.5] * a)
-a = 1
-a += 2
-print(1 + _2a)
-for (i = 0; i < 10; i += 1) {
-    print(i)
-}
-printflush(message1)
-"""))
-print("\n".join(map(repr, tokens)))
-print(Parser().parse(tokens))
+# tokens = Lexer.lex(Preprocessor.preprocess("""
+# print("Hello, World!")
+# const _2a = (cell1[3] * a)
+# cell1.enabled = cell1.x
+# a = 1
+# a += 2
+# print(1 + _2a)
+# if (1 > 2) {
+#     print("1 > 2")
+# } else {
+#     for (i = 0; i < 10; i += 1) {
+#         print(i)
+#     }
+# }
+# printflush(message1)
+# draw.color(255, 255, 255, 127)
+# """))
+# # print("\n".join(map(str, tokens)))
+# ast = Parser().parse(tokens)
+# try:
+#     print(Linker.link([Optimizer.optimize(ast.generate())]))
+# except MlogError as e:
+#     e.print()
+#     raise e
 
-# from . import cli
+from . import cli

mlogpp/cli.py

@@ -6,7 +6,7 @@
 from .lexer import Lexer
 from .preprocess import Preprocessor
 from .parser_ import Parser
-from .generator import Generator
+from .optimizer import Optimizer
 from .linker import Linker
 from . import __version__
 
@@ -24,10 +24,6 @@ class IOMethod(Enum):
 parser.add_argument("-o:s", "--output-stdout", help="write output to stdout", action="store_true")
 parser.add_argument("-o:c", "--output-clip", help="write output to clipboard (defualt)", action="store_true")
 
-parser.add_argument("-O0", "--optimize0", help="disable optimization (WARNING: creates extremely unoptimized code)", action="store_true")
-parser.add_argument("-O1", "--optimize1", help="set optimization level to 1", action="store_true")
-parser.add_argument("-O2", "--optimize2", help="set optimization level to 2 (default)", action="store_true")
-
 parser.add_argument("-v", "--verbose", help="print additional information", action="store_true")
 parser.add_argument("-l", "--lines", help="print line numbers when output to stdout is selected", action="store_true")
 
@@ -38,8 +34,6 @@ class IOMethod(Enum):
 omethod = IOMethod.CLIP
 ofile = ""
 
-optlevel = 2
-
 verbose = False
 
 # parse arguments
@@ -51,10 +45,7 @@ class IOMethod(Enum):
             omethod = IOMethod.FILE if k.endswith("file") else IOMethod.STD if k.endswith("stdout") else IOMethod.CLIP
             if omethod == IOMethod.FILE:
                 ofile = v
-        elif k.startswith("optimize"):
-            # optimization level
-
-            optlevel = int(k[-1])
+
         elif k == "verbose":
             # verbose
 
@@ -95,18 +86,18 @@ class IOMethod(Enum):
     if data[0].endswith(".mind") or data[0].endswith(".masm"):
         outs.append(data[1])
         continue
-
-    out = Preprocessor.preprocess(Lexer.lex(data[1]))
+
+    out = Preprocessor.preprocess(data[1])
+    out = Lexer.lex(out)
     out = Parser().parse(out)
-    out = Generator().generate(out, optimization_levels[optlevel])
+    out = out.generate()
+    out = Optimizer.optimize(out)
+
     # add to compiled files
     outs.append(out)
 
 # link the compiled files
-if len(outs) > 1:
-    out = Linker.link(outs).strip()
-else:
-    out = outs[0].strip()
+out = Linker.link(outs).strip()
 
 if omethod == IOMethod.FILE:
     # output to file

mlogpp/error.py

@@ -3,78 +3,54 @@
 from .formatting import Format
 from .util import Position
 
-# debug flags
-PRE_ERROR_DEBUG = False
-LEX_ERROR_DEBUG = False
-PARSE_ERROR_DEBUG = False
-LINK_ERROR_DEBUG  = False
+class MlogError(Exception):
+    def __init__(self, msg: str, pos: Position = None):
+        self.message = f"{msg}: {pos}"
 
-def pre_error(msg: str, code: str = "", pos: Position = None) -> None:
-    """
-    raise preprocessor error
-    """
-
-    # debug
-    if PRE_ERROR_DEBUG:
-        curframe = inspect.currentframe()
-        calframe = inspect.getouterframes(curframe, 2)
-        print(f"In method: {calframe[1][3]}, line: {calframe[1][2]}\n")
-
-    if pos is not None:
-        print(f"{Format.ERROR}Preprocessor error on line {pos.line}, column {pos.col}: {msg}{Format.RESET}\n")
-        print(f"Here:\n{code}\n{arrows.generate(pos.col, pos.len)}")
-    else:
-        print(f"")
-
-    sys.exit(1)
+        self.msg = msg
+        self.pos = pos
+
+    def print(self):
+        if self.pos is not None:
+            print(f"{Format.ERROR}{Format.BOLD}Error{Format.RESET}{Format.ERROR} on line {self.pos.line}, column {self.pos.start}: {self.msg}{Format.RESET}")
+            print(f"Here:\n{self.pos.code}\n{self.pos.arrows()}")
+        else:
+            print(f"{Format.ERROR}{Format.BOLD}Error{Format.RESET}{Format.ERROR}: {self.msg}{Format.RESET}")
 
-def lex_error(msg: str, code: str = "", pos: Position = None) -> None:
+def lex_error(msg: str, pos: Position = None) -> None:
     """
     raise lexer error
     """
 
-    # debug
-    if LEX_ERROR_DEBUG:
-        curframe = inspect.currentframe()
-        calframe = inspect.getouterframes(curframe, 2)
-        print(f"In method: {calframe[1][3]}, line: {calframe[1][2]}\n")
-
-    if pos is not None:
-        print(f"{Format.ERROR}Lexer error on line {pos.line + 1}, column {pos.start + 1}: {msg}{Format.RESET}\n")
-        print(f"Here:\n{code}\n{pos.arrows()}")
-    else:
-        print(f"{Format.ERROR}Lexer error: {msg}{Format.RESET}\n")
-
-    sys.exit(1)
+    raise MlogError(msg, pos)
 
 def parse_error(pos: Position, msg: str) -> None:
     """
     raise parser error
     """
 
-    # debug
-    if PARSE_ERROR_DEBUG:
-        curframe = inspect.currentframe()
-        calframe = inspect.getouterframes(curframe, 2)
-        print(f"In method: {calframe[1][3]}, line: {calframe[1][2]}\n")
-
-    print(f"{Format.ERROR}Parser error on line {pos.line}, column {pos.start}: {msg}{Format.RESET}\n")
-    print(f"Here:\n{pos.code}\n{pos.arrows()}")
-
-    sys.exit(1)
+    if msg == "Unexpected token":
+        raise MlogError(f"{msg} [\"{pos.code_section()}\"]", pos)
+    else:
+        raise MlogError(msg, pos)
 
 def link_error(pos: Position, msg: str) -> None:
     """
     raise linker error
     """
 
-    # debug
-    if LINK_ERROR_DEBUG:
-        curframe = inspect.currentframe()
-        calframe = inspect.getouterframes(curframe, 2)
-        print(f"In method: {calframe[1][3]}, line: {calframe[1][2]}\n")
-
-    print(f"{Format.ERROR}Linker error on line {pos.line}, column {pos.column}: {msg}{Format.RESET}\n")
-    print(f"Here:\n{Token._sanitize(pos.cline)}\n{arrows.generate(pos.column, pos.len)}")
+    raise MlogError(msg, pos)
+
+def gen_error(pos: Position, msg: str) -> None:
+    """
+    raise generator error
+    """
+
+    raise MlogError(msg, pos)
+
+def error(msg: str) -> None:
+    """
+    raise error
+    """
 
-    sys.exit(1)
+    raise MlogError(msg)

mlogpp/formatting.py

@@ -11,4 +11,4 @@ class Format:
 
     RESET = "\033[0m"
 
-    ERROR = "\033[1m\033[91m"
+    ERROR = "\033[91m"

mlogpp/functions.py

@@ -1,37 +1,25 @@
-def gen_signature(name: str, params: list):
-    """
-    generate a function signature
-    """
-    return f"{name}:{len(params)}"
-
 # native functions
-native = [
-    "read", "write",
-    "draw", "drawflush",
-    "print", "printflush",
-    "getlink",
-    "control",
-    "radar",
-    "sensor",
-    "set", "op",
-    "wait", "lookup",
-    "end", "jump",
-    "ubind", "ucontrol", "uradar", "ulocate"
-]
-
-# number of parameters to native functions
-native_params = {
+native = {
     "read": 3, "write": 3,
-    "draw": 7, "drawflush": 1,
+    "drawflush": 1,
     "print": 1, "printflush": 1,
     "getlink": 2,
-    "control": 6,
     "radar": 6,
     "sensor": 3,
     "set": 2, "op": 4,
-    "wait": 1, "lookup": 3,
+    "wait": 1,
     "end": 0, "jump": 4,
-    "ubind": 1, "ucontrol":6, "uradar": 6, "ulocate": 8
+    "ubind": 1, "uradar": 6
+}
+
+# return positions for native functions
+native_ret = {
+    "read": 0,
+    "getlink": 0,
+    "radar": 6,
+    "sensor": 0,
+    "uradar": 6,
+    "op": 1
 }
 
 # native subcommands
@@ -80,9 +68,37 @@ def gen_signature(name: str, params: list):
         "unit": 2,
         "item": 2,
         "liquid": 2
+    },
+    "ulocate": {
+        "ore": 1,
+        "building": 2,
+        "spawn": 0,
+        "damaged": 0
     }
 }
 
+# generate list of native subcommand combinations
+native_sublist = []
+for k, v in native_sub.items():
+    for s in v.keys():
+        native_sublist.append(f"{k}.{s}")
+
+# native subcommands return positions
+native_sub_ret = {
+    "ucontrol.getBlock": (2,),
+    "ucontrol.within": (3,),
+
+    "lookup.block": (0,),
+    "lookup.unit": (0,),
+    "lookup.item": (0,),
+    "lookup.liquid": (0,),
+
+    "ulocate.ore": (3, 4, 5, 6),
+    "ulocate.building": (3, 4, 5, 6),
+    "ulocate.spawn": (3, 4, 5, 6),
+    "ulocate.damaged": (3, 4, 5, 6)
+}
+
 # builtin operators
 builtin = [
     "mod",
@@ -111,8 +127,11 @@ def gen_signature(name: str, params: list):
     "len": 2
 }
 
+# special keywords with parentheses
+keywords_paren = ["if", "while", "for", "function", "repeat"]
+
 # special keywords
 keywords = ["if", "else", "while", "for", "function", "repeat"]
 
 # special identifiers
-special = native + builtin + keywords
+special = list(native.keys()) + builtin + keywords