refactoring, alignment

Author: Andrea Bonomi

Makefile

@@ -0,0 +1,11 @@
+
+help:
+	@echo - make coverage
+	@echo - make tests
+
+coverage:
+	python3 -m coverage run --source=cstruct --omit 'cstruct/examples/*.py' setup.py test && python3 -m coverage report -m
+
+tests:
+	python3 setup.py test
+	python2 setup.py test

cstruct/__init__.py

@@ -30,23 +30,25 @@
 __date__ = '15 August 2013'
 
 import struct
-import hashlib
 from .base import (
+    LITTLE_ENDIAN,
+    BIG_ENDIAN,
+    NATIVE_ORDER,
     STRUCTS,
     DEFINES,
     TYPEDEFS,
     C_TYPE_TO_FORMAT,
     CHAR_ZERO,
     EMPTY_BYTES_STRING
 )
-from .c_parser import Tokens
 from .abstract import CStructMeta
 from .cstruct import CStruct
 from .mem_cstruct import MemCStruct
 
 __all__ = [
     'LITTLE_ENDIAN',
     'BIG_ENDIAN',
+    'NATIVE_ORDER',
     'CHAR_ZERO',
     'EMPTY_BYTES_STRING',
     'CStruct',
@@ -55,16 +57,9 @@
     'undef',
     'typedef',
     'sizeof',
-    'factory'
+    'parse'
 ]
 
-# little-endian, std. size & alignment
-LITTLE_ENDIAN = '<'
-# big-endian, std. size & alignment
-BIG_ENDIAN = '>'
-# native order, size & alignment
-NATIVE_ORDER = '@'
-
 def define(key, value):
     """
     Define a constant that can be used in the C struct
@@ -116,9 +111,9 @@ def sizeof(type_):
         else:
             return struct.calcsize(ttype)
 
-def factory(__struct__, __cls__=None, __name__=None, **kargs):
+def parse(__struct__, __cls__=None, **kargs):
     """
-    Return a new class mapping a C struct definition.
+    Return a new class mapping a C struct/union definition.
 
     :param __struct__:     definition of the struct (or union) in C syntax
     :param __cls__:        (optional) super class - CStruct(default) or MemCStruct
@@ -127,16 +122,7 @@ def factory(__struct__, __cls__=None, __name__=None, **kargs):
     :param __is_union__:   (optional) True for union, False for struct (default)
     :returns:              __cls__ subclass
     """
-    kargs = dict(kargs)
-    if not isinstance(__struct__, Tokens):
-        __struct__ = Tokens(__struct__)
-    kargs['__struct__'] = __struct__
-    if __name__ is None: # Anonymous struct
-        __name__ = 'CStruct%s' % hashlib.sha1(str(__struct__).encode('utf-8')).hexdigest()
-        kargs['__anonymous__'] = True
-    kargs['__name__'] = __name__
     if __cls__ is None:
         __cls__ = CStruct
-    return type(__name__, (__cls__,), kargs) # TODO
-
+    return __cls__.parse(__struct__, **kargs)
 

cstruct/abstract.py

@@ -25,7 +25,8 @@
 #
 
 from .base import STRUCTS
-from .c_parser import parse_struct
+import hashlib
+from .c_parser import (parse_struct, Tokens)
 
 __all__ = [
     'CStructMeta',
@@ -73,6 +74,27 @@ def __init__(self, string=None, **kargs):
         for key, value in kargs.items():
             setattr(self, key, value)
 
+    @classmethod
+    def parse(cls, __struct__, __name__=None, **kargs):
+        """
+        Return a new class mapping a C struct/union definition.
+
+        :param __struct__:     definition of the struct (or union) in C syntax
+        :param __name__:       (optional) name of the new class. If empty, a name based on the __struct__ hash is generated
+        :param __byte_order__: (optional) byte order, valid values are LITTLE_ENDIAN, BIG_ENDIAN, NATIVE_ORDER
+        :param __is_union__:   (optional) True for union, False for struct (default)
+        :returns:              cls subclass
+        """
+        kargs = dict(kargs)
+        if not isinstance(__struct__, Tokens):
+            __struct__ = Tokens(__struct__)
+        kargs['__struct__'] = __struct__
+        if __name__ is None: # Anonymous struct
+            __name__ = cls.__name__ + '_' + hashlib.sha1(str(__struct__).encode('utf-8')).hexdigest()
+            kargs['__anonymous__'] = True
+        kargs['__name__'] = __name__
+        return type(__name__, (cls,), kargs)
+
     def unpack(self, buffer):
         """
         Unpack the string containing packed C structure data

cstruct/base.py

@@ -27,6 +27,10 @@
 import sys
 
 __all__ = [
+    'LITTLE_ENDIAN',
+    'BIG_ENDIAN',
+    'NATIVE_ORDER',
+    'CHAR_ZERO',
     'STRUCTS',
     'DEFINES',
     'TYPEDEFS',
@@ -35,51 +39,58 @@
     'CHAR_ZERO'
 ]
 
+# little-endian, std. size & alignment
+LITTLE_ENDIAN = '<'
+# big-endian, std. size & alignment
+BIG_ENDIAN = '>'
+# native order, size & alignment
+NATIVE_ORDER = '@'
+
 STRUCTS = {
 }
 
 DEFINES = {
 }
 
 TYPEDEFS = {
+    'short int':            'short',
+    'unsigned short int':   'unsigned short',
+    'ushort':               'unsigned short',
+    'long int':             'long',
+    'unsigned long int':    'unsigned long',
+    'int8_t':               'int8',
+    'uint8_t':              'uint8',
+    'int16_t':              'int16',
+    'uint16_t':             'uint16',
+    'int32_t':              'int32',
+    'uint32_t':             'uint32',
+    'int64_t':              'int64',
+    'uint64_t':             'uint64',
 }
 
 C_TYPE_TO_FORMAT = {
     'char':                 's',
     'signed char':          'b',
     'unsigned char':        'B',
     'short':                'h',
-    'short int':            'h',
-    'ushort':               'H',
     'unsigned short':       'H',
-    'unsigned short int':   'H',
     'int':                  'i',
     'unsigned int':         'I',
     'long':                 'l',
-    'long int':             'l',
     'unsigned long':        'L',
-    'unsigned long int':    'L',
     'long long':            'q',
     'unsigned long long':   'Q',
     'float':                'f',
     'double':               'd',
     'void *':               'P',
     'int8':                 'b',
-    'int8_t':               'b',
     'uint8':                'B',
-    'uint8_t':              'B',
     'int16':                'h',
-    'int16_t':              'h',
     'uint16':               'H',
-    'uint16_t':             'H',
     'int32':                'i',
-    'int32_t':              'i',
     'uint32':               'I',
-    'uint32_t':             'I',
     'int64':                'q',
-    'int64_t':              'q',
     'uint64':               'Q',
-    'uint64_t':             'Q',
 }
 
 EMPTY_BYTES_STRING = bytes()

cstruct/c_parser.py

@@ -27,6 +27,7 @@
 import re
 import struct
 from .base import (
+    NATIVE_ORDER,
     DEFINES,
     TYPEDEFS,
     STRUCTS,
@@ -39,6 +40,9 @@
     'parse_struct'
 ]
 
+def align(__byte_order__):
+    return __byte_order__ is None or __byte_order__ == NATIVE_ORDER
+
 class FieldType(object):
 
     def __init__(self, vtype, vlen, vsize, fmt, offset, flexible_array):
@@ -121,17 +125,16 @@ def __str__(self):
         return str(self.tokens)
 
 
-def parse_type(tokens, __cls__):
-    from . import factory
+def parse_type(tokens, __cls__, __byte_order__):
     if len(tokens) < 2:
         raise Exception("Parsing error")
     vtype = tokens.pop()
     # signed/unsigned/struct
-    if vtype == 'unsigned' or vtype == 'signed' or vtype == 'struct' and len(tokens) > 1:
+    if vtype in ['signed', 'unsigned', 'struct', 'union'] and len(tokens) > 1:
         vtype = vtype + " " + tokens.pop()
     next_token = tokens.pop()
     # short int, long int, or long long
-    if next_token == 'int' or next_token == 'long':
+    if next_token in ['int', 'long']:
         vtype = vtype + " " + next_token
         next_token = tokens.pop()
     # void *
@@ -170,22 +173,28 @@ def parse_type(tokens, __cls__):
         kind, vtype = vtype.split(' ', 1)
         if tokens.get() == '{': # Named nested struct
             tokens.pop()
-            vtype = factory(tokens, __name__=vtype, __cls__=__cls__,  __is_union__=__is_union__)
+            vtype = __cls__.parse(tokens, __name__=vtype, __is_union__=__is_union__, __byte_order__=__byte_order__)
         elif vtype == '{': # Unnamed nested struct
-            vtype = factory(tokens, __is_union__=__is_union__)
+            vtype = __cls__.parse(tokens, __is_union__=__is_union__, __byte_order__=__byte_order__)
         else:
             try:
                 vtype = STRUCTS[vtype]
             except KeyError:
                 raise Exception("Unknow %s \"%s\"" % (kind, vtype))
         ttype = "c"
         fmt = str(vlen * vtype.size) + ttype
+        # alignment/
+        alignment = vtype.__alignment__
     else: # other types
         ttype = C_TYPE_TO_FORMAT.get(vtype, None)
         if ttype is None:
             raise Exception("Unknow type \"" + vtype + "\"")
         fmt = (str(vlen) if vlen > 1 or flexible_array else '') + ttype
-    return vtype, vlen, fmt, flexible_array
+        # alignment
+        alignment = struct.calcsize((__byte_order__ + ttype) if __byte_order__ is not None else ttype)
+    fmt = (__byte_order__ + fmt) if __byte_order__ is not None else fmt
+    vsize = struct.calcsize(fmt)
+    return vtype, vlen, vsize, fmt, flexible_array, alignment
 
 
 def parse_struct(__struct__, __cls__=None, __fields__=None, __is_union__=False, __byte_order__=None, **kargs):
@@ -195,6 +204,7 @@ def parse_struct(__struct__, __cls__=None, __fields__=None, __is_union__=False,
     fields_types = {}
     flexible_array = False
     offset = 0
+    max_alignment = 0
     if isinstance(__struct__, Tokens):
         tokens = __struct__
     else:
@@ -206,11 +216,17 @@ def parse_struct(__struct__, __cls__=None, __fields__=None, __is_union__=False,
         # flexible array member must be the last member of such a struct
         if flexible_array:
             raise Exception("Flexible array member must be the last member of such a struct")
-        vtype, vlen, fmt, flexible_array = parse_type(tokens, __cls__)
+        vtype, vlen, vsize, fmt, flexible_array, alignment = parse_type(tokens, __cls__, __byte_order__)
         vname = tokens.pop()
         fields.append(vname)
-        fmt = (__byte_order__ + fmt) if __byte_order__ is not None else fmt
-        vsize = struct.calcsize(fmt)
+        # calculate the max field size (for the alignment)
+        max_alignment = max(max_alignment, alignment)
+        # align stuct if byte order is native
+        if not __is_union__ and align(__byte_order__) and vtype != 'char':
+            modulo = offset % alignment
+            if modulo: # not aligned to the field size
+                delta = alignment - modulo
+                offset = offset + delta
         fields_types[vname] = FieldType(vtype, vlen, vsize, fmt, offset, flexible_array)
         if not __is_union__: # C struct
             offset = offset + vsize
@@ -224,6 +240,12 @@ def parse_struct(__struct__, __cls__=None, __fields__=None, __is_union__=False,
         fmt = '%ds' % size
     else: # C struct
         fmt = "".join(fmt)
+        # add padding to stuct if byte order is native
+        if not __is_union__ and align(__byte_order__):
+            modulo = offset % max_alignment
+            if modulo: # not aligned to the max field size
+                delta = max_alignment - modulo
+                offset = offset + delta
         size = offset # (offset is calculated as size sum)
 
     # Add the byte order as prefix
@@ -236,7 +258,8 @@ def parse_struct(__struct__, __cls__=None, __fields__=None, __is_union__=False,
         '__fields_types__': fields_types,
         '__size__': size,
         '__is_union__': __is_union__,
-        '__byte_order__': __byte_order__
+        '__byte_order__': __byte_order__,
+        '__alignment__': max_alignment
     }
 
     # Add the missing fields to the class