Lib/compiler/pyassem.py

"""Assembler for Python bytecode

The new module is used to create the code object.  The following
attribute definitions are included from the reference manual:

co_name gives the function name
co_argcount is the number of positional arguments (including
    arguments with default values) 
co_nlocals is the number of local variables used by the function
    (including arguments)  
co_varnames is a tuple containing the names of the local variables
    (starting with the argument names) 
co_code is a string representing the sequence of bytecode instructions 
co_consts is a tuple containing the literals used by the bytecode
co_names is a tuple containing the names used by the bytecode
co_filename is the filename from which the code was compiled
co_firstlineno is the first line number of the function
co_lnotab is a string encoding the mapping from byte code offsets
    to line numbers.  see LineAddrTable below.
co_stacksize is the required stack size (including local variables)
co_flags is an integer encoding a number of flags for the
    interpreter.  There are four flags:
    CO_OPTIMIZED -- uses load fast
    CO_NEWLOCALS -- everything?
    CO_VARARGS -- use *args
    CO_VARKEYWORDS -- uses **args

If a code object represents a function, the first item in co_consts is
the documentation string of the function, or None if undefined.
"""

import sys
import dis
import new
import string

import misc

# flags for code objects
CO_OPTIMIZED = 0x0001
CO_NEWLOCALS = 0x0002
CO_VARARGS = 0x0004
CO_VARKEYWORDS = 0x0008

class TupleArg:
    def __init__(self, count, names):
        self.count = count
        self.names = names
    def __repr__(self):
        return "TupleArg(%s, %s)" % (self.count, self.names)
    def getName(self):
        return ".nested%d" % self.count

class PyAssembler:
    """Creates Python code objects
    """

    # XXX this class needs to major refactoring

    def __init__(self, args=(), name='?', filename='<?>',
                 docstring=None):
        # XXX why is the default value for flags 3?
        self.insts = []
        # used by makeCodeObject
        self._getArgCount(args)
        self.code = ''
        self.consts = [docstring]
        self.filename = filename
        self.flags = CO_NEWLOCALS
        self.name = name
        self.names = []
        self.varnames = list(args) or []
        for i in range(len(self.varnames)):
            var = self.varnames[i]
            if isinstance(var, TupleArg):
                self.varnames[i] = var.getName()
        # lnotab support
        self.firstlineno = 0
        self.lastlineno = 0
        self.last_addr = 0
        self.lnotab = ''

    def _getArgCount(self, args):
        self.argcount = len(args)
        if args:
            for arg in args:
                if isinstance(arg, TupleArg):
                    numNames = len(misc.flatten(arg.names))
                    self.argcount = self.argcount - numNames 

    def __repr__(self):
        return "<bytecode: %d instrs>" % len(self.insts)

    def setFlags(self, val):
        """XXX for module's function"""
        self.flags = val

    def setOptimized(self):
        self.flags = self.flags | CO_OPTIMIZED

    def setVarArgs(self):
        if not self.flags & CO_VARARGS:
            self.flags = self.flags | CO_VARARGS
            self.argcount = self.argcount - 1

    def setKWArgs(self):
        self.flags = self.flags | CO_VARKEYWORDS

    def getCurInst(self):
        return len(self.insts)

    def getNextInst(self):
        return len(self.insts) + 1

    def dump(self, io=sys.stdout):
        i = 0
        for inst in self.insts:
            if inst[0] == 'SET_LINENO':
                io.write("\n")
            io.write("    %3d " % i)
            if len(inst) == 1:
                io.write("%s\n" % inst)
            else:
                io.write("%-15.15s\t%s\n" % inst)
            i = i + 1

    def makeCodeObject(self):
        """Make a Python code object

        This creates a Python code object using the new module.  This
        seems simpler than reverse-engineering the way marshal dumps
        code objects into .pyc files.  One of the key difficulties is
        figuring out how to layout references to code objects that
        appear on the VM stack; e.g.
          3 SET_LINENO          1
          6 LOAD_CONST          0 (<code object fact at 8115878 [...]
          9 MAKE_FUNCTION       0
         12 STORE_NAME          0 (fact)
        """
        
        self._findOffsets()
        lnotab = LineAddrTable()
        for t in self.insts:
            opname = t[0]
            if len(t) == 1:
                lnotab.addCode(chr(self.opnum[opname]))
            elif len(t) == 2:
                oparg = self._convertArg(opname, t[1])
                if opname == 'SET_LINENO':
                    lnotab.nextLine(oparg)
                try:
                    hi, lo = divmod(oparg, 256)
                except TypeError:
                    raise TypeError, "untranslated arg: %s, %s" % (opname, oparg)
                lnotab.addCode(chr(self.opnum[opname]) + chr(lo) +
                               chr(hi))
        # why is a module a special case?
        if self.flags == 0:
            nlocals = 0
        else:
            nlocals = len(self.varnames)
        # XXX danger! can't pass through here twice
        if self.flags & CO_VARKEYWORDS:
            self.argcount = self.argcount - 1
        stacksize = findDepth(self.insts)
        try:
            co = new.code(self.argcount, nlocals, stacksize,
                          self.flags, lnotab.getCode(), self._getConsts(),
                          tuple(self.names), tuple(self.varnames),
                          self.filename, self.name, self.firstlineno,
                          lnotab.getTable())
        except SystemError, err:
            print err
            print repr(self.argcount)
            print repr(nlocals)
            print repr(stacksize)
            print repr(self.flags)
            print repr(lnotab.getCode())
            print repr(self._getConsts())
            print repr(self.names)
            print repr(self.varnames)
            print repr(self.filename)
            print repr(self.name)
            print repr(self.firstlineno)
            print repr(lnotab.getTable())
            raise
        return co

    def _getConsts(self):
        """Return a tuple for the const slot of a code object

        Converts PythonVMCode objects to code objects
        """
        l = []
        for elt in self.consts:
            # XXX might be clearer to just as isinstance(CodeGen)
            if hasattr(elt, 'asConst'):
                l.append(elt.asConst())
            else:
                l.append(elt)
        return tuple(l)

    def _findOffsets(self):
        """Find offsets for use in resolving StackRefs"""
        self.offsets = []
        cur = 0
        for t in self.insts:
            self.offsets.append(cur)
            l = len(t)
            if l == 1:
                cur = cur + 1
            elif l == 2:
                cur = cur + 3
                arg = t[1]
                # XXX this is a total hack: for a reference used
                # multiple times, we create a list of offsets and
                # expect that we when we pass through the code again
                # to actually generate the offsets, we'll pass in the
                # same order.
                if isinstance(arg, StackRef):
                    try:
                        arg.__offset.append(cur)
                    except AttributeError:
                        arg.__offset = [cur]

    def _convertArg(self, op, arg):
        """Convert the string representation of an arg to a number

        The specific handling depends on the opcode.

        XXX This first implementation isn't going to be very
        efficient. 
        """
        if op == 'SET_LINENO':
            return arg
        if op == 'LOAD_CONST':
            return self._lookupName(arg, self.consts)
        if op in self.localOps:
            # make sure it's in self.names, but use the bytecode offset
            self._lookupName(arg, self.names)
            return self._lookupName(arg, self.varnames)
        if op in self.globalOps:
            return self._lookupName(arg, self.names)
        if op in self.nameOps:
            return self._lookupName(arg, self.names)
        if op == 'COMPARE_OP':
            return self.cmp_op.index(arg)
        if self.hasjrel.has_elt(op):
            offset = arg.__offset[0]
            del arg.__offset[0]
            return self.offsets[arg.resolve()] - offset
        if self.hasjabs.has_elt(op):
            return self.offsets[arg.resolve()]
        return arg

    nameOps = ('STORE_NAME', 'IMPORT_NAME', 'IMPORT_FROM',
               'STORE_ATTR', 'LOAD_ATTR', 'LOAD_NAME', 'DELETE_NAME',
               'DELETE_ATTR')
    localOps = ('LOAD_FAST', 'STORE_FAST', 'DELETE_FAST')
    globalOps = ('LOAD_GLOBAL', 'STORE_GLOBAL', 'DELETE_GLOBAL')

    def _lookupName(self, name, list):
        """Return index of name in list, appending if necessary"""
        if name in list:
            i = list.index(name)
            # this is cheap, but incorrect in some cases, e.g 2 vs. 2L
            if type(name) == type(list[i]):
                return i
            for i in range(len(list)):
                elt = list[i]
                if type(elt) == type(name) and elt == name:
                    return i
        end = len(list)
        list.append(name)
        return end

    # Convert some stuff from the dis module for local use
    
    cmp_op = list(dis.cmp_op)
    hasjrel = misc.Set()
    for i in dis.hasjrel:
        hasjrel.add(dis.opname[i])
    hasjabs = misc.Set()
    for i in dis.hasjabs:
        hasjabs.add(dis.opname[i])
    
    opnum = {}
    for num in range(len(dis.opname)):
        opnum[dis.opname[num]] = num

    # this version of emit + arbitrary hooks might work, but it's damn
    # messy.

    def emit(self, *args):
        self._emitDispatch(args[0], args[1:])
        self.insts.append(args)

    def _emitDispatch(self, type, args):
        for func in self._emit_hooks.get(type, []):
            func(self, args)

    _emit_hooks = {}

class LineAddrTable:
    """lnotab
    
    This class builds the lnotab, which is undocumented but described
    by com_set_lineno in compile.c.  Here's an attempt at explanation:

    For each SET_LINENO instruction after the first one, two bytes are
    added to lnotab.  (In some cases, multiple two-byte entries are
    added.)  The first byte is the distance in bytes between the
    instruction for the last SET_LINENO and the current SET_LINENO.
    The second byte is offset in line numbers.  If either offset is
    greater than 255, multiple two-byte entries are added -- one entry
    for each factor of 255.
    """

    def __init__(self):
        self.code = []
        self.codeOffset = 0
        self.firstline = 0
        self.lastline = 0
        self.lastoff = 0
        self.lnotab = []

    def addCode(self, code):
        self.code.append(code)
        self.codeOffset = self.codeOffset + len(code)

    def nextLine(self, lineno):
        if self.firstline == 0:
            self.firstline = lineno
            self.lastline = lineno
        else:
            # compute deltas
            addr = self.codeOffset - self.lastoff
            line = lineno - self.lastline
            while addr > 0 or line > 0:
                # write the values in 1-byte chunks that sum
                # to desired value
                trunc_addr = addr
                trunc_line = line
                if trunc_addr > 255:
                    trunc_addr = 255
                if trunc_line > 255:
                    trunc_line = 255
                self.lnotab.append(trunc_addr)
                self.lnotab.append(trunc_line)
                addr = addr - trunc_addr
                line = line - trunc_line
            self.lastline = lineno
            self.lastoff = self.codeOffset

    def getCode(self):
        return string.join(self.code, '')

    def getTable(self):
        return string.join(map(chr, self.lnotab), '')
    
class StackRef:
    """Manage stack locations for jumps, loops, etc."""
    count = 0

    def __init__(self, id=None, val=None):
        if id is None:
            id = StackRef.count
            StackRef.count = StackRef.count + 1
        self.id = id
        self.val = val

    def __repr__(self):
        if self.val:
            return "StackRef(val=%d)" % self.val
        else:
            return "StackRef(id=%d)" % self.id

    def bind(self, inst):
        self.val = inst

    def resolve(self):
        if self.val is None:
            print "UNRESOLVE REF", self
            return 0
        return self.val

class StackDepthTracker:
    # XXX need to keep track of stack depth on jumps

    def findDepth(self, insts):
        depth = 0
        maxDepth = 0
        for i in insts:
            opname = i[0]
            delta = self.effect.get(opname, 0)
            if delta > 1:
                depth = depth + delta
            elif delta < 0:
                if depth > maxDepth:
                    maxDepth = depth
                depth = depth + delta
            else:
                if depth > maxDepth:
                    maxDepth = depth
                # now check patterns
                for pat, delta in self.patterns:
                    if opname[:len(pat)] == pat:
                        depth = depth + delta
                        break
                # if we still haven't found a match
                if delta == 0:
                    meth = getattr(self, opname)
                    depth = depth + meth(i[1])
            if depth < 0:
                depth = 0
        return maxDepth

    effect = {
        'POP_TOP': -1,
        'DUP_TOP': 1,
        'SLICE+1': -1,
        'SLICE+2': -1,
        'SLICE+3': -2,
        'STORE_SLICE+0': -1,
        'STORE_SLICE+1': -2,
        'STORE_SLICE+2': -2,
        'STORE_SLICE+3': -3,
        'DELETE_SLICE+0': -1,
        'DELETE_SLICE+1': -2,
        'DELETE_SLICE+2': -2,
        'DELETE_SLICE+3': -3,
        'STORE_SUBSCR': -3,
        'DELETE_SUBSCR': -2,
        # PRINT_EXPR?
        'PRINT_ITEM': -1,
        'LOAD_LOCALS': 1,
        'RETURN_VALUE': -1,
        'EXEC_STMT': -2,
        'BUILD_CLASS': -2,
        'STORE_NAME': -1,
        'STORE_ATTR': -2,
        'DELETE_ATTR': -1,
        'STORE_GLOBAL': -1,
        'BUILD_MAP': 1,
        'COMPARE_OP': -1,
        'STORE_FAST': -1,
        }
    # use pattern match
    patterns = [
        ('BINARY_', -1),
        ('LOAD_', 1),
        ('IMPORT_', 1),
        ]
    # special cases

    #: UNPACK_TUPLE, UNPACK_LIST, BUILD_TUPLE,
    # BUILD_LIST, CALL_FUNCTION, MAKE_FUNCTION, BUILD_SLICE
    def UNPACK_TUPLE(self, count):
        return count
    def UNPACK_LIST(self, count):
        return count
    def BUILD_TUPLE(self, count):
        return -count
    def BUILD_LIST(self, count):
        return -count
    def CALL_FUNCTION(self, argc):
        hi, lo = divmod(argc, 256)
        return lo + hi * 2
    def MAKE_FUNCTION(self, argc):
        return -argc
    def BUILD_SLICE(self, argc):
        if argc == 2:
            return -1
        elif argc == 3:
            return -2
    
findDepth = StackDepthTracker().findDepth