Preprocessor for ULP/RTC macros

README.rst

@@ -17,6 +17,10 @@ Status
 
 The most commonly used simple stuff should work.
 
+Expressions in assembly source code are supported and get evaluated during
+assembling. Only expressions evaluating to a single integer are supported.
+Constants defined with ``.set`` are supported in expressions.
+
 We have some unit tests and also compatibility tests that compare the output
 whether it is identical with binutils-esp32ulp output.
 

esp32_ulp/__main__.py

@@ -2,14 +2,15 @@
 
 from .util import garbage_collect
 
+from .preprocess import preprocess
 from .assemble import Assembler
 from .link import make_binary
 garbage_collect('after import')
 
 
 def src_to_binary(src):
     assembler = Assembler()
-    assembler.assemble(src)
+    assembler.assemble(src, remove_comments=False)  # comments already removed by preprocessor
     garbage_collect('before symbols export')
     addrs_syms = assembler.symbols.export()
     for addr, sym in addrs_syms:
@@ -23,6 +24,7 @@ def main(fn):
     with open(fn) as f:
         src = f.read()
 
+    src = preprocess(src)
     binary = src_to_binary(src)
 
     if fn.endswith('.s') or fn.endswith('.S'):

esp32_ulp/assemble.py

@@ -3,7 +3,7 @@
 """
 
 from . import opcodes
-from .nocomment import remove_comments
+from .nocomment import remove_comments as do_remove_comments
 from .util import garbage_collect
 
 TEXT, DATA, BSS = 'text', 'data', 'bss'
@@ -12,9 +12,10 @@
 
 
 class SymbolTable:
-    def __init__(self, symbols, bases):
+    def __init__(self, symbols, bases, globals):
         self._symbols = symbols
         self._bases = bases
+        self._globals = globals
         self._pass = None
 
     def set_pass(self, _pass):
@@ -32,7 +33,7 @@ def get_from(self):
     def set_sym(self, symbol, stype, section, value):
         entry = (stype, section, value)
         if symbol in self._symbols and entry != self._symbols[symbol]:
-            raise Exception('redefining symbol %s with different value %r -> %r.' % (label, self._symbols[symbol], entry))
+            raise Exception('redefining symbol %s with different value %r -> %r.' % (symbol, self._symbols[symbol], entry))
         self._symbols[symbol] = entry
 
     def has_sym(self, symbol):
@@ -52,8 +53,10 @@ def dump(self):
         for symbol, entry in self._symbols.items():
             print(symbol, entry)
 
-    def export(self):
-        addrs_syms = [(self.resolve_absolute(entry), symbol) for symbol, entry in self._symbols.items()]
+    def export(self, incl_non_globals=False):
+        addrs_syms = [(self.resolve_absolute(entry), symbol)
+                      for symbol, entry in self._symbols.items()
+                      if incl_non_globals or symbol in self._globals]
         return sorted(addrs_syms)
 
     def to_abs_addr(self, section, offset):
@@ -93,11 +96,15 @@ def resolve_relative(self, symbol):
         from_addr = self.to_abs_addr(self._from_section, self._from_offset)
         return sym_addr - from_addr
 
+    def set_global(self, symbol):
+        self._globals[symbol] = True
+        pass
+
 
 class Assembler:
 
-    def __init__(self, symbols=None, bases=None):
-        self.symbols = SymbolTable(symbols or {}, bases or {})
+    def __init__(self, symbols=None, bases=None, globls=None):
+        self.symbols = SymbolTable(symbols or {}, bases or {}, globls or {})
         opcodes.symbols = self.symbols  # XXX dirty hack
 
     def init(self, a_pass):
@@ -118,7 +125,7 @@ def parse_line(self, line):
         """
         if not line:
             return
-        has_label = line[0] not in '\t '
+        has_label = line[0] not in '\t .'
         if has_label:
             label_line = line.split(None, 1)
             if len(label_line) == 2:
@@ -150,8 +157,10 @@ def append_section(self, value, expected_section=None):
         if expected_section is not None and s is not expected_section:
             raise TypeError('only allowed in %s section' % expected_section)
         if s is BSS:
-            # just increase BSS size by value
-            self.offsets[s] += value
+            if int.from_bytes(value, 'little') != 0:
+                raise ValueError('attempt to store non-zero value in section .bss')
+            # just increase BSS size by length of value
+            self.offsets[s] += len(value)
         else:
             self.sections[s].append(value)
             self.offsets[s] += len(value)
@@ -231,9 +240,12 @@ def d_align(self, align=4, fill=None):
             self.fill(self.section, amount, fill)
 
     def d_set(self, symbol, expr):
-        value = int(expr)  # TODO: support more than just integers
+        value = int(opcodes.eval_arg(expr))  # TODO: support more than just integers
         self.symbols.set_sym(symbol, ABS, None, value)
 
+    def d_global(self, symbol):
+        self.symbols.set_global(symbol)
+
     def append_data(self, wordlen, args):
         data = [int(arg).to_bytes(wordlen, 'little') for arg in args]
         self.append_section(b''.join(data))
@@ -263,16 +275,16 @@ def assembler_pass(self, lines):
                         continue
                 else:
                     # machine instruction
-                    func = getattr(opcodes, 'i_' + opcode, None)
+                    func = getattr(opcodes, 'i_' + opcode.lower(), None)
                     if func is not None:
-                        instruction = func(*args)
+                        instruction = 0 if self.a_pass == 1 else func(*args)
                         self.append_section(instruction.to_bytes(4, 'little'), TEXT)
                         continue
-                raise Exception('Unknown opcode or directive: %s' % opcode)
+                raise ValueError('Unknown opcode or directive: %s' % opcode)
         self.finalize_sections()
 
-    def assemble(self, text):
-        lines = remove_comments(text)
+    def assemble(self, text, remove_comments=True):
+        lines = do_remove_comments(text) if remove_comments else text.splitlines()
         self.init(1)  # pass 1 is only to get the symbol table right
         self.assembler_pass(lines)
         self.symbols.set_bases(self.compute_bases())

esp32_ulp/opcodes.py

@@ -6,6 +6,7 @@
 from uctypes import struct, addressof, LITTLE_ENDIAN, UINT32, BFUINT32, BF_POS, BF_LEN
 
 from .soc import *
+from .util import split_tokens, validate_expression
 
 # XXX dirty hack: use a global for the symbol table
 symbols = None
@@ -112,7 +113,7 @@ def make_ins(layout):
     unused : 8      # Unused
     low : 5         # Low bit
     high : 5        # High bit
-    opcode : 4      # Opcode (OPCODE_WR_REG)
+    opcode : 4      # Opcode (OPCODE_RD_REG)
 """)
 
 
@@ -267,6 +268,20 @@ def make_ins(layout):
 ARG = namedtuple('ARG', ('type', 'value', 'raw'))
 
 
+def eval_arg(arg):
+    parts = []
+    for token in split_tokens(arg):
+        if symbols.has_sym(token):
+            _, _, sym_value = symbols.get_sym(token)
+            parts.append(str(sym_value))
+        else:
+            parts.append(token)
+    parts = "".join(parts)
+    if not validate_expression(parts):
+        raise ValueError('Unsupported expression: %s' % parts)
+    return eval(parts)
+
+
 def arg_qualify(arg):
     """
     look at arg and qualify its type:
@@ -289,8 +304,12 @@ def arg_qualify(arg):
         return ARG(IMM, int(arg), arg)
     except ValueError:
         pass
-    entry = symbols.get_sym(arg)
-    return ARG(SYM, entry, arg)
+    try:
+        entry = symbols.get_sym(arg)
+        return ARG(SYM, entry, arg)
+    except KeyError:
+        pass
+    return ARG(IMM, int(eval_arg(arg)), arg)
 
 
 def get_reg(arg):

esp32_ulp/preprocess.py

@@ -0,0 +1,57 @@
+from . import nocomment
+from .util import split_tokens
+
+
+class Preprocessor:
+    def __init__(self):
+        self._defines = {}
+
+    def parse_defines(self, content):
+        result = {}
+        for line in content.splitlines():
+            line = line.strip()
+            if not line.startswith("#define"):
+                # skip lines not containing #define
+                continue
+            line = line[8:].strip()  # remove #define
+            parts = line.split(None, 1)
+            if len(parts) != 2:
+                # skip defines without value
+                continue
+            identifier, value = parts
+            tmp = identifier.split('(', 1)
+            if len(tmp) == 2:
+                # skip parameterised defines (macros)
+                continue
+            value = "".join(nocomment.remove_comments(value)).strip()
+            result[identifier] = value
+        self._defines = result
+        return result
+
+    def expand_defines(self, line):
+        found = True
+        while found:  # do as many passed as needed, until nothing was replaced anymore
+            found = False
+            tokens = split_tokens(line)
+            line = ""
+            for t in tokens:
+                lu = self._defines.get(t, t)
+                if lu != t:
+                    found = True
+                line += lu
+
+        return line
+
+    def preprocess(self, content):
+        self.parse_defines(content)
+        lines = nocomment.remove_comments(content)
+        result = []
+        for line in lines:
+            line = self.expand_defines(line)
+            result.append(line)
+        result = "\n".join(result)
+        return result
+
+
+def preprocess(content):
+    return Preprocessor().preprocess(content)

esp32_ulp/util.py

@@ -2,10 +2,68 @@
 
 import gc
 
+NORMAL, WHITESPACE = 0, 1
+
 
 def garbage_collect(msg, verbose=DEBUG):
     free_before = gc.mem_free()
     gc.collect()
     free_after = gc.mem_free()
     if verbose:
         print("%s: %d --gc--> %d bytes free" % (msg, free_before, free_after))
+
+
+def split_tokens(line):
+    buf = ""
+    tokens = []
+    state = NORMAL
+    for c in line:
+        if ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c == '_':
+            if state != NORMAL:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = NORMAL
+            buf += c
+        elif c == ' ' or c == '\t':
+            if state != WHITESPACE:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = WHITESPACE
+            buf += c
+        else:
+            if len(buf) > 0:
+                tokens.append(buf)
+                buf = ""
+            tokens.append(c)
+
+    if len(buf) > 0:
+        tokens.append(buf)
+
+    return tokens
+
+
+def validate_expression(param):
+    for token in split_tokens(param):
+        state = 0
+        for c in token:
+            if c not in ' \t+-*/%()<>&|~x0123456789abcdef':
+                return False
+
+            # the following allows hex digits a-f after 0x but not otherwise
+            if state == 0:
+                if c in 'abcdef':
+                    return False
+                if c == '0':
+                    state = 1
+                continue
+
+            if state == 1:
+                state = 2 if c == 'x' else 0
+                continue
+
+            if state == 2:
+                if c not in '0123456789abcdef':
+                    state = 0
+    return True

tests/00_unit_tests.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-for file in opcodes assemble link ; do
+for file in opcodes assemble link util preprocess; do
     echo testing $file...
     micropython $file.py
 done

tests/01_compat_tests.sh

@@ -13,12 +13,14 @@ for src_file in $(ls -1 compat/*.S); do
     log_file="${src_name}.log"
     micropython -m esp32_ulp $src_file 1>$log_file   # generates $ulp_file
 
+    pre_file="${src_name}.pre"
     obj_file="${src_name}.o"
     elf_file="${src_name}.elf"
     bin_file="${src_name}.bin"
 
     echo -e "\tBuilding using binutils"
-    esp32ulp-elf-as -o $obj_file $src_file
+    gcc -E -o ${pre_file} $src_file
+    esp32ulp-elf-as -o $obj_file ${pre_file}
     esp32ulp-elf-ld -T esp32.ulp.ld -o $elf_file $obj_file
     esp32ulp-elf-objcopy -O binary $elf_file $bin_file