Title: A higher level struct module
Submitter: Brian McErlean (other recipes)Brian McErlean (other recipes)
Last Updated: 2006/10/26
Version no: 1.3
Category: Databases

Description:

This recipe provides a higher level wrapper around the struct module. It provides a more convenient syntax for defining and using structs, and adds additional features such as:
- Allows embedding structures within other structures
- Allows defining arrays of items (or other structures)
- Class based syntax, allowing access and updates by field name, not position
- Extension of structures by inheritance

Source: Text Source

import struct

class Format(object):
    """Endianness and size format for structures."""
    Native          = "@"       # Native format, native size
    StandardNative  = "="       # Native format, standard size
    LittleEndian    = "<"       # Standard size
    BigEndian       = ">"       # Standard size
    
class Element(object):
    """A single element in a struct."""
    id=0
    def __init__(self, typecode):
        Element.id+=1           # Note: not thread safe
        self.id = Element.id
        self.typecode = typecode
        self.size = struct.calcsize(typecode)

    def __len__(self):
        return self.size

    def decode(self, format, s):
        """Additional decode steps once converted via struct.unpack"""
        return s

    def encode(self, format, val):
        """Additional encode steps to allow packing with struct.pack"""
        return val

    def __str__(self):
        return self.typecode

    def __call__(self, num):
        """Define this as an array of elements."""
        # Special case - strings already handled as one blob.
        if self.typecode in 'sp':
            # Strings handled specially - only one item
            return Element('%ds' % num)
        else:
            return ArrayElement(self, num)

    def __getitem__(self, num): return self(num)

class ArrayElement(Element):
    def __init__(self, basic_element, num):
        Element.__init__(self, '%ds' % (len(basic_element) * num))
        self.num = num
        self.basic_element = basic_element

    def decode(self, format, s):
        # NB. We use typecode * size, not %s%s' % (size, typecode), 
        # so we deal with typecodes that already have numbers,  
        # ie 2*'4s' != '24s'
        return [self.basic_element.decode(format, x) for x in  
                    struct.unpack('%s%s' % (format, 
                            self.num * self.basic_element.typecode),s)]

    def encode(self, format, vals):
        fmt = format + (self.basic_element.typecode * self.num)
        return struct.pack(fmt, *[self.basic_element.encode(format,v) 
                                  for v in vals])

class EmbeddedStructElement(Element):
    def __init__(self, structure):
        Element.__init__(self, '%ds' % structure._struct_size)
        self.struct = structure

    # Note: Structs use their own endianness format, not their parent's
    def decode(self, format, s):
        return self.struct(s)

    def encode(self, format, s):
        return self.struct._pack(s)

name_to_code = {
    'Char'             : 'c',
    'Byte'             : 'b',
    'UnsignedByte'     : 'B',
    'Int'              : 'i',
    'UnsignedInt'      : 'I',
    'Short'            : 'h',
    'UnsignedShort'    : 'H',
    'Long'             : 'l',
    'UnsignedLong'     : 'L',
    'String'           : 's',  
    'PascalString'     : 'p',  
    'Pointer'          : 'P',
    'Float'            : 'f',
    'Double'           : 'd',
    'LongLong'         : 'q',
    'UnsignedLongLong' : 'Q',
    }

class Type(object):
    def __getattr__(self, name):
        return Element(name_to_code[name])

    def Struct(self, struct):
        return EmbeddedStructElement(struct)
        
Type=Type()

class MetaStruct(type):
    def __init__(cls, name, bases, d):
        type.__init__(cls, name, bases, d)
        if hasattr(cls, '_struct_data'):  # Allow extending by inheritance
            cls._struct_info = list(cls._struct_info) # use copy.
        else:
            cls._struct_data=''
            cls._struct_info=[]     # name / element pairs

        # Get each Element field, sorted by id.
        elems = sorted(((k,v) for (k,v) in d.iteritems() 
                        if isinstance(v, Element)),
                        key=lambda x:x[1].id)

        cls._struct_data += ''.join(str(v) for (k,v) in elems)
        cls._struct_info += elems
        cls._struct_size = struct.calcsize(cls._format + cls._struct_data)

class Struct(object):
    """Represent a binary structure."""
    __metaclass__=MetaStruct
    _format = Format.Native  # Default to native format, native size

    def __init__(self, _data=None, **kwargs):
        if _data is None:
            _data ='\0' * self._struct_size
            
        fieldvals = zip(self._struct_info, struct.unpack(self._format + 
                                             self._struct_data, _data))
        for (name, elem), val in fieldvals:
            setattr(self, name, elem.decode(self._format, val))
        
        for k,v in kwargs.iteritems():
            setattr(self, k, v)

    def _pack(self):
        return struct.pack(self._format + self._struct_data, 
            *[elem.encode(self._format, getattr(self, name)) 
                for (name,elem) in self._struct_info])                

    def __str__(self):
        return self._pack()
    
    def __repr__(self):
        return "%s(%r)" % (self.__class__.__name__, self._pack())
    
###################################################################
#  End of implementation - usage examples follow:
###################################################################

###################################################################
#
# Usage
#
# Using the above code, we can now define structures in a
# more readable class based syntax.  For example:
###################################################################
    
class Point(Struct):
    _format = Format.LittleEndian
    x = Type.Short
    y = Type.Short
    
p = Point('\x01\x00\x02\x00')

print p.x, p.y   # Prints 1,2
p.x, p.y = 100,200
print repr(p)     # Prints "Point('d\x00\xc8\x00')

assert(struct.pack('<hh',100,200) == str(p))
    
###################################################################
#
# Arrays and Embedded structures
#
# You can also embed arrays, (and arrays of arrays), as well
# as other structures within your struct definition.
###################################################################

class Shape(Struct):
    _format = Format.BigEndian
    name      = Type.String[8]
    numpoints = Type.Int
    points    = Type.Struct(Point)[4] # Array of 4 points.

s=Shape('Triangle\x00\x00\x00\x03\x00\x00\x00\x00\x05\x00\x05\x00\n\x00'
        '\x00\x00\x00\x00\x00\x00')

# This will print "Triangle [(0,0), (5,5), (10,0)]"
print s.name, [(p.x, p.y) for p in s.points[:s.numpoints]]

# The same structure could be created as:
s2=Shape(name='Triangle', numpoints=3, points=[
                                         Point(x=0,y=0),
                                         Point(x=5,y=5),
                                         Point(x=10,y=0),
                                         Point(x=0,y=0)])

assert str(s2) == str(s)

# Note that even though Shape is in BigEndian format, the Points
# keep their LittleEndian setting, so mixing formats is possible,
# and the same struct will always have the same representation
# regardless of its context.  Hence the following is true:

assert str(s.points[1]) == str( Point(x=5, y=5))

# It is also possible to define multi-dimensional arrays,
# which will be unpacked as lists of lists.
# In addition, it is possible to add methods and non-struct
# instance variables without interfering with the structure
# (Unless you overwrite structure field names of course)

class TicTacToe(Struct):
    board = Type.Char[3][3] # 3x3 array of chars

    ignored = 'This is not packed / unpacked by the structure'
    
    def display(self):
        print '\n'.join(''.join(row) for row in self.board)

game = TicTacToe('X.O.X...O')
print game.board  # [['X', '.', 'O'], ['.', 'X', '.'], ['.', '.', 'O']]

game.display()
# Prints: X.O
#         .X.
#         ..O

game.board[0][1] = 'X'
game.display()
# Prints: XXO
#         .X.
#         ..O
print str(game) # prints 'XXO.X...O'


###################################################################
#
# Inheritance
#
# Structures may also be inherited from, in which case, additional
# fields will occur after the existing ones.
#
###################################################################

class Point3D(Point):
    z = Type.Short

p = Point3D(x=1, y=2, z=3)

print repr(p)   # prints Point3D('\x01\x00\x02\x00\x03\x00')

Discussion:

The standard struct module is useful when dealing with C structs, and various file and network formats, but is rather awkward to work with. It is fairly low-level, using strings of character codes to describe the structure, and unpacks to a tuple, rather than an object with appropriately named fields. This means that access to items is always by position, rather than the field name, as would be used in the C code.

It also fares badly when dealing with arrays, and embedded structures. Dealing with these within the struct module means each element becomes effectively the same as a top-level field, flattening their structure and losing the organisation of elements. This all results in code that is much messier and hard to maintain than the equivalent C code.

This recipe builds on top of the struct module and provides syntax for declaration of structures that is closer to the corresponding C code, and allows for more complex structures to be defined.

Structures are read by instantiating the appropriate class with the binary struct data, and packed by calling str(aStruct). I use the above code as a module named "structure".

Implementation:

Structs are defined by lists of Element objects. Each element contains the struct code for the type, and a unique id, incremented after each instantiation, which is used so that the elements can be sorted into the same order that they were defined within a struct. They also define __getitem__, returning another Element subclass, ArrayElement, dealing with arrays of simple elements. Another Element subclass, EmbeddedStructElement is used to represent substructures.

The Type object provides some syntax sugar for constructing these elements, allowing "x = Type.Int", instead of "x = Element("i")"

The MetaStruct metaclass checks for all fields which are subclasses of Element, sorts them into the definition order, and uses them to create the corresponding struct format string, and the corresponding list of Elements

Finally, the Struct class defines appropriate __init__ and __str__ methods to use this generated by the metaclass to encode and decode the structure. Arrays and substructures are treated as strings of the appropriate size, and implement an encode() and decode() method which will transform from the string to the appropriate data, or vice-versa.


Warnings and Caveats:

There are a few flaws in the above code currently. The main one is that there is no validation when setting struct fields. For instance, given a struct like:

>>> class MyStruct(Struct):
... name = Type.String[8]
... values = Type.Int[4]

>>> a=MyStruct() # Initialises with everything zeroed.


There is nothing preventing you from doing:

>>> a.values = [1,2,3,4,5] # One too many items.

This will now fail when trying to pack the structure with a struct error which doesn't give you any clue as to which field is corrupt. Probably the main thing that should be done is either some kind of pre-validation of items, or at least error handling that mentions what field is invalid.
Worse is that doing:

>>> a.name = "thisnameistoobig"

will not raise an error even when packing the structure, but will instead silently truncate the name in the packed representation.


Another thing to be careful of is that duplicating names will cause the last definition to be used, rather than resulting in multiple elements, or giving an error. For instance:

>>> class MyStruct(Struct):
... item1=Type.Char[5]
... pad = Type.Char[3]
... item2=Type.Char[1]
... pad = Type.Char[3]
... item3=Type.Char[5]

Here the first pad bytes will not be used, as the second defintion of pad will override the name.


Also, note that you must create element objects (ie Type.Int) within the struct, rather than reusing them. For instance:

>>> myArrayType = Type.Int[8] # define an 8 element array
>>> class MyStruct(Struct):
... name = Type.String
... items = myArrayType # Don't do this

Here items will actually be packed before name, as it was created earlier, so the definition order will not match the packed order.

Finally, note that padding and alignment will be handled the same as the struct module.