Issue #4753: By enabling a configure option named '--with-computed-gotos'
on compilers that support it (notably: gcc, SunPro, icc), the bytecode
evaluation loop is compiled with a new dispatch mechanism which gives
speedups of up to 20%, depending on the system, on various benchmarks.
diff --git a/Python/ceval.c b/Python/ceval.c
index fcf8efb..e0ce92e 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -705,6 +705,114 @@
char *filename;
#endif
+/* Computed GOTOs, or
+ the-optimization-commonly-but-improperly-known-as-"threaded code"
+ using gcc's labels-as-values extension
+ (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
+
+ The traditional bytecode evaluation loop uses a "switch" statement, which
+ decent compilers will optimize as a single indirect branch instruction
+ combined with a lookup table of jump addresses. However, since the
+ indirect jump instruction is shared by all opcodes, the CPU will have a
+ hard time making the right prediction for where to jump next (actually,
+ it will be always wrong except in the uncommon case of a sequence of
+ several identical opcodes).
+
+ "Threaded code" in contrast, uses an explicit jump table and an explicit
+ indirect jump instruction at the end of each opcode. Since the jump
+ instruction is at a different address for each opcode, the CPU will make a
+ separate prediction for each of these instructions, which is equivalent to
+ predicting the second opcode of each opcode pair. These predictions have
+ a much better chance to turn out valid, especially in small bytecode loops.
+
+ A mispredicted branch on a modern CPU flushes the whole pipeline and
+ can cost several CPU cycles (depending on the pipeline depth),
+ and potentially many more instructions (depending on the pipeline width).
+ A correctly predicted branch, however, is nearly free.
+
+ At the time of this writing, the "threaded code" version is up to 15-20%
+ faster than the normal "switch" version, depending on the compiler and the
+ CPU architecture.
+
+ We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
+ because it would render the measurements invalid.
+
+
+ NOTE: care must be taken that the compiler doesn't try to "optimize" the
+ indirect jumps by sharing them between all opcodes. Such optimizations
+ can be disabled on gcc by using the -fno-gcse flag (or possibly
+ -fno-crossjumping).
+*/
+
+#if defined(USE_COMPUTED_GOTOS) && defined(DYNAMIC_EXECUTION_PROFILE)
+#undef USE_COMPUTED_GOTOS
+#endif
+
+#ifdef USE_COMPUTED_GOTOS
+/* Import the static jump table */
+#include "opcode_targets.h"
+
+/* This macro is used when several opcodes defer to the same implementation
+ (e.g. SETUP_LOOP, SETUP_FINALLY) */
+#define TARGET_WITH_IMPL(op, impl) \
+ TARGET_##op: \
+ opcode = op; \
+ if (HAS_ARG(op)) \
+ oparg = NEXTARG(); \
+ case op: \
+ goto impl; \
+
+#define TARGET(op) \
+ TARGET_##op: \
+ opcode = op; \
+ if (HAS_ARG(op)) \
+ oparg = NEXTARG(); \
+ case op:
+
+
+#define DISPATCH() \
+ { \
+ /* Avoid multiple loads from _Py_Ticker despite `volatile` */ \
+ int _tick = _Py_Ticker - 1; \
+ _Py_Ticker = _tick; \
+ if (_tick >= 0) { \
+ FAST_DISPATCH(); \
+ } \
+ continue; \
+ }
+
+#ifdef LLTRACE
+#define FAST_DISPATCH() \
+ { \
+ if (!lltrace && !_Py_TracingPossible) { \
+ f->f_lasti = INSTR_OFFSET(); \
+ goto *opcode_targets[*next_instr++]; \
+ } \
+ goto fast_next_opcode; \
+ }
+#else
+#define FAST_DISPATCH() \
+ { \
+ if (!_Py_TracingPossible) { \
+ f->f_lasti = INSTR_OFFSET(); \
+ goto *opcode_targets[*next_instr++]; \
+ } \
+ goto fast_next_opcode; \
+ }
+#endif
+
+#else
+#define TARGET(op) \
+ case op:
+#define TARGET_WITH_IMPL(op, impl) \
+ /* silence compiler warnings about `impl` unused */ \
+ if (0) goto impl; \
+ case op:
+#define DISPATCH() continue
+#define FAST_DISPATCH() goto fast_next_opcode
+#endif
+
+
/* Tuple access macros */
#ifndef Py_DEBUG
@@ -782,16 +890,23 @@
predictions turned-on and interpret the results as if some opcodes
had been combined or turn-off predictions so that the opcode frequency
counter updates for both opcodes.
+
+ Opcode prediction is disabled with threaded code, since the latter allows
+ the CPU to record separate branch prediction information for each
+ opcode.
+
*/
-#ifdef DYNAMIC_EXECUTION_PROFILE
+#if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS)
#define PREDICT(op) if (0) goto PRED_##op
+#define PREDICTED(op) PRED_##op:
+#define PREDICTED_WITH_ARG(op) PRED_##op:
#else
#define PREDICT(op) if (*next_instr == op) goto PRED_##op
-#endif
-
#define PREDICTED(op) PRED_##op: next_instr++
#define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3
+#endif
+
/* Stack manipulation macros */
@@ -1153,56 +1268,56 @@
/* case STOP_CODE: this is an error! */
- case NOP:
- goto fast_next_opcode;
+ TARGET(NOP)
+ FAST_DISPATCH();
- case LOAD_FAST:
+ TARGET(LOAD_FAST)
x = GETLOCAL(oparg);
if (x != NULL) {
Py_INCREF(x);
PUSH(x);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
format_exc_check_arg(PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(co->co_varnames, oparg));
break;
- case LOAD_CONST:
+ TARGET(LOAD_CONST)
x = GETITEM(consts, oparg);
Py_INCREF(x);
PUSH(x);
- goto fast_next_opcode;
+ FAST_DISPATCH();
PREDICTED_WITH_ARG(STORE_FAST);
- case STORE_FAST:
+ TARGET(STORE_FAST)
v = POP();
SETLOCAL(oparg, v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
PREDICTED(POP_TOP);
- case POP_TOP:
+ TARGET(POP_TOP)
v = POP();
Py_DECREF(v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
- case ROT_TWO:
+ TARGET(ROT_TWO)
v = TOP();
w = SECOND();
SET_TOP(w);
SET_SECOND(v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
- case ROT_THREE:
+ TARGET(ROT_THREE)
v = TOP();
w = SECOND();
x = THIRD();
SET_TOP(w);
SET_SECOND(x);
SET_THIRD(v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
- case ROT_FOUR:
+ TARGET(ROT_FOUR)
u = TOP();
v = SECOND();
w = THIRD();
@@ -1211,15 +1326,15 @@
SET_SECOND(w);
SET_THIRD(x);
SET_FOURTH(u);
- goto fast_next_opcode;
+ FAST_DISPATCH();
- case DUP_TOP:
+ TARGET(DUP_TOP)
v = TOP();
Py_INCREF(v);
PUSH(v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
- case DUP_TOPX:
+ TARGET(DUP_TOPX)
if (oparg == 2) {
x = TOP();
Py_INCREF(x);
@@ -1228,7 +1343,7 @@
STACKADJ(2);
SET_TOP(x);
SET_SECOND(w);
- goto fast_next_opcode;
+ FAST_DISPATCH();
} else if (oparg == 3) {
x = TOP();
Py_INCREF(x);
@@ -1240,106 +1355,106 @@
SET_TOP(x);
SET_SECOND(w);
SET_THIRD(v);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
Py_FatalError("invalid argument to DUP_TOPX"
" (bytecode corruption?)");
/* Never returns, so don't bother to set why. */
break;
- case UNARY_POSITIVE:
+ TARGET(UNARY_POSITIVE)
v = TOP();
x = PyNumber_Positive(v);
Py_DECREF(v);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case UNARY_NEGATIVE:
+ TARGET(UNARY_NEGATIVE)
v = TOP();
x = PyNumber_Negative(v);
Py_DECREF(v);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case UNARY_NOT:
+ TARGET(UNARY_NOT)
v = TOP();
err = PyObject_IsTrue(v);
Py_DECREF(v);
if (err == 0) {
Py_INCREF(Py_True);
SET_TOP(Py_True);
- continue;
+ DISPATCH();
}
else if (err > 0) {
Py_INCREF(Py_False);
SET_TOP(Py_False);
err = 0;
- continue;
+ DISPATCH();
}
STACKADJ(-1);
break;
- case UNARY_INVERT:
+ TARGET(UNARY_INVERT)
v = TOP();
x = PyNumber_Invert(v);
Py_DECREF(v);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_POWER:
+ TARGET(BINARY_POWER)
w = POP();
v = TOP();
x = PyNumber_Power(v, w, Py_None);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_MULTIPLY:
+ TARGET(BINARY_MULTIPLY)
w = POP();
v = TOP();
x = PyNumber_Multiply(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_TRUE_DIVIDE:
+ TARGET(BINARY_TRUE_DIVIDE)
w = POP();
v = TOP();
x = PyNumber_TrueDivide(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_FLOOR_DIVIDE:
+ TARGET(BINARY_FLOOR_DIVIDE)
w = POP();
v = TOP();
x = PyNumber_FloorDivide(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_MODULO:
+ TARGET(BINARY_MODULO)
w = POP();
v = TOP();
x = PyNumber_Remainder(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_ADD:
+ TARGET(BINARY_ADD)
w = POP();
v = TOP();
if (PyUnicode_CheckExact(v) &&
@@ -1355,152 +1470,152 @@
skip_decref_vx:
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_SUBTRACT:
+ TARGET(BINARY_SUBTRACT)
w = POP();
v = TOP();
x = PyNumber_Subtract(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_SUBSCR:
+ TARGET(BINARY_SUBSCR)
w = POP();
v = TOP();
x = PyObject_GetItem(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_LSHIFT:
+ TARGET(BINARY_LSHIFT)
w = POP();
v = TOP();
x = PyNumber_Lshift(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_RSHIFT:
+ TARGET(BINARY_RSHIFT)
w = POP();
v = TOP();
x = PyNumber_Rshift(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_AND:
+ TARGET(BINARY_AND)
w = POP();
v = TOP();
x = PyNumber_And(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_XOR:
+ TARGET(BINARY_XOR)
w = POP();
v = TOP();
x = PyNumber_Xor(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case BINARY_OR:
+ TARGET(BINARY_OR)
w = POP();
v = TOP();
x = PyNumber_Or(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case LIST_APPEND:
+ TARGET(LIST_APPEND)
w = POP();
v = stack_pointer[-oparg];
err = PyList_Append(v, w);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
- continue;
+ DISPATCH();
}
break;
- case SET_ADD:
+ TARGET(SET_ADD)
w = POP();
v = stack_pointer[-oparg];
err = PySet_Add(v, w);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
- continue;
+ DISPATCH();
}
break;
- case INPLACE_POWER:
+ TARGET(INPLACE_POWER)
w = POP();
v = TOP();
x = PyNumber_InPlacePower(v, w, Py_None);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_MULTIPLY:
+ TARGET(INPLACE_MULTIPLY)
w = POP();
v = TOP();
x = PyNumber_InPlaceMultiply(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_TRUE_DIVIDE:
+ TARGET(INPLACE_TRUE_DIVIDE)
w = POP();
v = TOP();
x = PyNumber_InPlaceTrueDivide(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_FLOOR_DIVIDE:
+ TARGET(INPLACE_FLOOR_DIVIDE)
w = POP();
v = TOP();
x = PyNumber_InPlaceFloorDivide(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_MODULO:
+ TARGET(INPLACE_MODULO)
w = POP();
v = TOP();
x = PyNumber_InPlaceRemainder(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_ADD:
+ TARGET(INPLACE_ADD)
w = POP();
v = TOP();
if (PyUnicode_CheckExact(v) &&
@@ -1516,70 +1631,70 @@
skip_decref_v:
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_SUBTRACT:
+ TARGET(INPLACE_SUBTRACT)
w = POP();
v = TOP();
x = PyNumber_InPlaceSubtract(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_LSHIFT:
+ TARGET(INPLACE_LSHIFT)
w = POP();
v = TOP();
x = PyNumber_InPlaceLshift(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_RSHIFT:
+ TARGET(INPLACE_RSHIFT)
w = POP();
v = TOP();
x = PyNumber_InPlaceRshift(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_AND:
+ TARGET(INPLACE_AND)
w = POP();
v = TOP();
x = PyNumber_InPlaceAnd(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_XOR:
+ TARGET(INPLACE_XOR)
w = POP();
v = TOP();
x = PyNumber_InPlaceXor(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case INPLACE_OR:
+ TARGET(INPLACE_OR)
w = POP();
v = TOP();
x = PyNumber_InPlaceOr(v, w);
Py_DECREF(v);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case STORE_SUBSCR:
+ TARGET(STORE_SUBSCR)
w = TOP();
v = SECOND();
u = THIRD();
@@ -1589,10 +1704,10 @@
Py_DECREF(u);
Py_DECREF(v);
Py_DECREF(w);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case DELETE_SUBSCR:
+ TARGET(DELETE_SUBSCR)
w = TOP();
v = SECOND();
STACKADJ(-2);
@@ -1600,10 +1715,10 @@
err = PyObject_DelItem(v, w);
Py_DECREF(v);
Py_DECREF(w);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case PRINT_EXPR:
+ TARGET(PRINT_EXPR)
v = POP();
w = PySys_GetObject("displayhook");
if (w == NULL) {
@@ -1630,7 +1745,7 @@
#ifdef CASE_TOO_BIG
default: switch (opcode) {
#endif
- case RAISE_VARARGS:
+ TARGET(RAISE_VARARGS)
v = w = NULL;
switch (oparg) {
case 2:
@@ -1648,19 +1763,19 @@
}
break;
- case STORE_LOCALS:
+ TARGET(STORE_LOCALS)
x = POP();
v = f->f_locals;
Py_XDECREF(v);
f->f_locals = x;
- continue;
+ DISPATCH();
- case RETURN_VALUE:
+ TARGET(RETURN_VALUE)
retval = POP();
why = WHY_RETURN;
goto fast_block_end;
- case YIELD_VALUE:
+ TARGET(YIELD_VALUE)
retval = POP();
f->f_stacktop = stack_pointer;
why = WHY_YIELD;
@@ -1670,7 +1785,7 @@
SWAP_EXC_STATE();
goto fast_yield;
- case POP_EXCEPT:
+ TARGET(POP_EXCEPT)
{
PyTryBlock *b = PyFrame_BlockPop(f);
if (b->b_type != EXCEPT_HANDLER) {
@@ -1681,17 +1796,17 @@
}
UNWIND_EXCEPT_HANDLER(b);
}
- continue;
+ DISPATCH();
- case POP_BLOCK:
+ TARGET(POP_BLOCK)
{
PyTryBlock *b = PyFrame_BlockPop(f);
UNWIND_BLOCK(b);
}
- continue;
+ DISPATCH();
PREDICTED(END_FINALLY);
- case END_FINALLY:
+ TARGET(END_FINALLY)
v = POP();
if (PyLong_Check(v)) {
why = (enum why_code) PyLong_AS_LONG(v);
@@ -1731,7 +1846,7 @@
Py_DECREF(v);
break;
- case LOAD_BUILD_CLASS:
+ TARGET(LOAD_BUILD_CLASS)
x = PyDict_GetItemString(f->f_builtins,
"__build_class__");
if (x == NULL) {
@@ -1743,7 +1858,7 @@
PUSH(x);
break;
- case STORE_NAME:
+ TARGET(STORE_NAME)
w = GETITEM(names, oparg);
v = POP();
if ((x = f->f_locals) != NULL) {
@@ -1752,14 +1867,14 @@
else
err = PyObject_SetItem(x, w, v);
Py_DECREF(v);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
}
PyErr_Format(PyExc_SystemError,
"no locals found when storing %R", w);
break;
- case DELETE_NAME:
+ TARGET(DELETE_NAME)
w = GETITEM(names, oparg);
if ((x = f->f_locals) != NULL) {
if ((err = PyObject_DelItem(x, w)) != 0)
@@ -1773,7 +1888,7 @@
break;
PREDICTED_WITH_ARG(UNPACK_SEQUENCE);
- case UNPACK_SEQUENCE:
+ TARGET(UNPACK_SEQUENCE)
v = POP();
if (PyTuple_CheckExact(v) &&
PyTuple_GET_SIZE(v) == oparg) {
@@ -1785,7 +1900,7 @@
PUSH(w);
}
Py_DECREF(v);
- continue;
+ DISPATCH();
} else if (PyList_CheckExact(v) &&
PyList_GET_SIZE(v) == oparg) {
PyObject **items = \
@@ -1805,7 +1920,7 @@
Py_DECREF(v);
break;
- case UNPACK_EX:
+ TARGET(UNPACK_EX)
{
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
v = POP();
@@ -1820,7 +1935,7 @@
break;
}
- case STORE_ATTR:
+ TARGET(STORE_ATTR)
w = GETITEM(names, oparg);
v = TOP();
u = SECOND();
@@ -1828,10 +1943,10 @@
err = PyObject_SetAttr(v, w, u); /* v.w = u */
Py_DECREF(v);
Py_DECREF(u);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case DELETE_ATTR:
+ TARGET(DELETE_ATTR)
w = GETITEM(names, oparg);
v = POP();
err = PyObject_SetAttr(v, w, (PyObject *)NULL);
@@ -1839,22 +1954,22 @@
Py_DECREF(v);
break;
- case STORE_GLOBAL:
+ TARGET(STORE_GLOBAL)
w = GETITEM(names, oparg);
v = POP();
err = PyDict_SetItem(f->f_globals, w, v);
Py_DECREF(v);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case DELETE_GLOBAL:
+ TARGET(DELETE_GLOBAL)
w = GETITEM(names, oparg);
if ((err = PyDict_DelItem(f->f_globals, w)) != 0)
format_exc_check_arg(
PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w);
break;
- case LOAD_NAME:
+ TARGET(LOAD_NAME)
w = GETITEM(names, oparg);
if ((v = f->f_locals) == NULL) {
PyErr_Format(PyExc_SystemError,
@@ -1889,9 +2004,9 @@
Py_INCREF(x);
}
PUSH(x);
- continue;
+ DISPATCH();
- case LOAD_GLOBAL:
+ TARGET(LOAD_GLOBAL)
w = GETITEM(names, oparg);
if (PyUnicode_CheckExact(w)) {
/* Inline the PyDict_GetItem() calls.
@@ -1911,7 +2026,7 @@
if (x != NULL) {
Py_INCREF(x);
PUSH(x);
- continue;
+ DISPATCH();
}
d = (PyDictObject *)(f->f_builtins);
e = d->ma_lookup(d, w, hash);
@@ -1923,7 +2038,7 @@
if (x != NULL) {
Py_INCREF(x);
PUSH(x);
- continue;
+ DISPATCH();
}
goto load_global_error;
}
@@ -1942,13 +2057,13 @@
}
Py_INCREF(x);
PUSH(x);
- continue;
+ DISPATCH();
- case DELETE_FAST:
+ TARGET(DELETE_FAST)
x = GETLOCAL(oparg);
if (x != NULL) {
SETLOCAL(oparg, NULL);
- continue;
+ DISPATCH();
}
format_exc_check_arg(
PyExc_UnboundLocalError,
@@ -1957,19 +2072,19 @@
);
break;
- case LOAD_CLOSURE:
+ TARGET(LOAD_CLOSURE)
x = freevars[oparg];
Py_INCREF(x);
PUSH(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case LOAD_DEREF:
+ TARGET(LOAD_DEREF)
x = freevars[oparg];
w = PyCell_Get(x);
if (w != NULL) {
PUSH(w);
- continue;
+ DISPATCH();
}
err = -1;
/* Don't stomp existing exception */
@@ -1990,14 +2105,14 @@
}
break;
- case STORE_DEREF:
+ TARGET(STORE_DEREF)
w = POP();
x = freevars[oparg];
PyCell_Set(x, w);
Py_DECREF(w);
- continue;
+ DISPATCH();
- case BUILD_TUPLE:
+ TARGET(BUILD_TUPLE)
x = PyTuple_New(oparg);
if (x != NULL) {
for (; --oparg >= 0;) {
@@ -2005,11 +2120,11 @@
PyTuple_SET_ITEM(x, oparg, w);
}
PUSH(x);
- continue;
+ DISPATCH();
}
break;
- case BUILD_LIST:
+ TARGET(BUILD_LIST)
x = PyList_New(oparg);
if (x != NULL) {
for (; --oparg >= 0;) {
@@ -2017,11 +2132,11 @@
PyList_SET_ITEM(x, oparg, w);
}
PUSH(x);
- continue;
+ DISPATCH();
}
break;
- case BUILD_SET:
+ TARGET(BUILD_SET)
x = PySet_New(NULL);
if (x != NULL) {
for (; --oparg >= 0;) {
@@ -2035,17 +2150,17 @@
break;
}
PUSH(x);
- continue;
+ DISPATCH();
}
break;
- case BUILD_MAP:
+ TARGET(BUILD_MAP)
x = _PyDict_NewPresized((Py_ssize_t)oparg);
PUSH(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case STORE_MAP:
+ TARGET(STORE_MAP)
w = TOP(); /* key */
u = SECOND(); /* value */
v = THIRD(); /* dict */
@@ -2054,10 +2169,10 @@
err = PyDict_SetItem(v, w, u); /* v[w] = u */
Py_DECREF(u);
Py_DECREF(w);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case MAP_ADD:
+ TARGET(MAP_ADD)
w = TOP(); /* key */
u = SECOND(); /* value */
STACKADJ(-2);
@@ -2068,20 +2183,20 @@
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
- continue;
+ DISPATCH();
}
break;
- case LOAD_ATTR:
+ TARGET(LOAD_ATTR)
w = GETITEM(names, oparg);
v = TOP();
x = PyObject_GetAttr(v, w);
Py_DECREF(v);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case COMPARE_OP:
+ TARGET(COMPARE_OP)
w = POP();
v = TOP();
x = cmp_outcome(oparg, v, w);
@@ -2091,9 +2206,9 @@
if (x == NULL) break;
PREDICT(JUMP_IF_FALSE);
PREDICT(JUMP_IF_TRUE);
- continue;
+ DISPATCH();
- case IMPORT_NAME:
+ TARGET(IMPORT_NAME)
w = GETITEM(names, oparg);
x = PyDict_GetItemString(f->f_builtins, "__import__");
if (x == NULL) {
@@ -2134,10 +2249,10 @@
READ_TIMESTAMP(intr1);
Py_DECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case IMPORT_STAR:
+ TARGET(IMPORT_STAR)
v = POP();
PyFrame_FastToLocals(f);
if ((x = f->f_locals) == NULL) {
@@ -2150,33 +2265,33 @@
READ_TIMESTAMP(intr1);
PyFrame_LocalsToFast(f, 0);
Py_DECREF(v);
- if (err == 0) continue;
+ if (err == 0) DISPATCH();
break;
- case IMPORT_FROM:
+ TARGET(IMPORT_FROM)
w = GETITEM(names, oparg);
v = TOP();
READ_TIMESTAMP(intr0);
x = import_from(v, w);
READ_TIMESTAMP(intr1);
PUSH(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case JUMP_FORWARD:
+ TARGET(JUMP_FORWARD)
JUMPBY(oparg);
- goto fast_next_opcode;
+ FAST_DISPATCH();
PREDICTED_WITH_ARG(JUMP_IF_FALSE);
- case JUMP_IF_FALSE:
+ TARGET(JUMP_IF_FALSE)
w = TOP();
if (w == Py_True) {
PREDICT(POP_TOP);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
if (w == Py_False) {
JUMPBY(oparg);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
err = PyObject_IsTrue(w);
if (err > 0)
@@ -2185,18 +2300,18 @@
JUMPBY(oparg);
else
break;
- continue;
+ DISPATCH();
PREDICTED_WITH_ARG(JUMP_IF_TRUE);
- case JUMP_IF_TRUE:
+ TARGET(JUMP_IF_TRUE)
w = TOP();
if (w == Py_False) {
PREDICT(POP_TOP);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
if (w == Py_True) {
JUMPBY(oparg);
- goto fast_next_opcode;
+ FAST_DISPATCH();
}
err = PyObject_IsTrue(w);
if (err > 0) {
@@ -2207,10 +2322,10 @@
;
else
break;
- continue;
+ DISPATCH();
PREDICTED_WITH_ARG(JUMP_ABSOLUTE);
- case JUMP_ABSOLUTE:
+ TARGET(JUMP_ABSOLUTE)
JUMPTO(oparg);
#if FAST_LOOPS
/* Enabling this path speeds-up all while and for-loops by bypassing
@@ -2218,14 +2333,14 @@
because it prevents detection of a control-break in tight loops like
"while 1: pass". Compile with this option turned-on when you need
the speed-up and do not need break checking inside tight loops (ones
- that contain only instructions ending with goto fast_next_opcode).
+ that contain only instructions ending with FAST_DISPATCH).
*/
- goto fast_next_opcode;
+ FAST_DISPATCH();
#else
- continue;
+ DISPATCH();
#endif
- case GET_ITER:
+ TARGET(GET_ITER)
/* before: [obj]; after [getiter(obj)] */
v = TOP();
x = PyObject_GetIter(v);
@@ -2233,13 +2348,13 @@
if (x != NULL) {
SET_TOP(x);
PREDICT(FOR_ITER);
- continue;
+ DISPATCH();
}
STACKADJ(-1);
break;
PREDICTED_WITH_ARG(FOR_ITER);
- case FOR_ITER:
+ TARGET(FOR_ITER)
/* before: [iter]; after: [iter, iter()] *or* [] */
v = TOP();
x = (*v->ob_type->tp_iternext)(v);
@@ -2247,7 +2362,7 @@
PUSH(x);
PREDICT(STORE_FAST);
PREDICT(UNPACK_SEQUENCE);
- continue;
+ DISPATCH();
}
if (PyErr_Occurred()) {
if (!PyErr_ExceptionMatches(
@@ -2259,13 +2374,13 @@
x = v = POP();
Py_DECREF(v);
JUMPBY(oparg);
- continue;
+ DISPATCH();
- case BREAK_LOOP:
+ TARGET(BREAK_LOOP)
why = WHY_BREAK;
goto fast_block_end;
- case CONTINUE_LOOP:
+ TARGET(CONTINUE_LOOP)
retval = PyLong_FromLong(oparg);
if (!retval) {
x = NULL;
@@ -2274,9 +2389,10 @@
why = WHY_CONTINUE;
goto fast_block_end;
- case SETUP_LOOP:
- case SETUP_EXCEPT:
- case SETUP_FINALLY:
+ TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally)
+ TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally)
+ TARGET(SETUP_FINALLY)
+ _setup_finally:
/* NOTE: If you add any new block-setup opcodes that
are not try/except/finally handlers, you may need
to update the PyGen_NeedsFinalizing() function.
@@ -2284,9 +2400,9 @@
PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg,
STACK_LEVEL());
- continue;
+ DISPATCH();
- case WITH_CLEANUP:
+ TARGET(WITH_CLEANUP)
{
/* At the top of the stack are 1-3 values indicating
how/why we entered the finally clause:
@@ -2350,7 +2466,7 @@
break;
}
- case CALL_FUNCTION:
+ TARGET(CALL_FUNCTION)
{
PyObject **sp;
PCALL(PCALL_ALL);
@@ -2363,13 +2479,14 @@
stack_pointer = sp;
PUSH(x);
if (x != NULL)
- continue;
+ DISPATCH();
break;
}
- case CALL_FUNCTION_VAR:
- case CALL_FUNCTION_KW:
- case CALL_FUNCTION_VAR_KW:
+ TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw)
+ TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw)
+ TARGET(CALL_FUNCTION_VAR_KW)
+ _call_function_var_kw:
{
int na = oparg & 0xff;
int nk = (oparg>>8) & 0xff;
@@ -2409,12 +2526,13 @@
}
PUSH(x);
if (x != NULL)
- continue;
+ DISPATCH();
break;
}
- case MAKE_CLOSURE:
- case MAKE_FUNCTION:
+ TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function)
+ TARGET(MAKE_FUNCTION)
+ _make_function:
{
int posdefaults = oparg & 0xff;
int kwdefaults = (oparg>>8) & 0xff;
@@ -2507,7 +2625,7 @@
break;
}
- case BUILD_SLICE:
+ TARGET(BUILD_SLICE)
if (oparg == 3)
w = POP();
else
@@ -2519,14 +2637,17 @@
Py_DECREF(v);
Py_XDECREF(w);
SET_TOP(x);
- if (x != NULL) continue;
+ if (x != NULL) DISPATCH();
break;
- case EXTENDED_ARG:
+ TARGET(EXTENDED_ARG)
opcode = NEXTOP();
oparg = oparg<<16 | NEXTARG();
goto dispatch_opcode;
+#ifdef USE_COMPUTED_GOTOS
+ _unknown_opcode:
+#endif
default:
fprintf(stderr,
"XXX lineno: %d, opcode: %d\n",
diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py
new file mode 100644
index 0000000..6b1ab43
--- /dev/null
+++ b/Python/makeopcodetargets.py
@@ -0,0 +1,43 @@
+#! /usr/bin/env python
+"""Generate C code for the jump table of the threaded code interpreter
+(for compilers supporting computed gotos or "labels-as-values", such as gcc).
+"""
+
+import imp
+import os
+
+
+def find_module(modname):
+ """Finds and returns a module in the local dist/checkout.
+ """
+ modpath = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)), "Lib")
+ return imp.load_module(modname, *imp.find_module(modname, [modpath]))
+
+def write_contents(f):
+ """Write C code contents to the target file object.
+ """
+ opcode = find_module("opcode")
+ targets = ['_unknown_opcode'] * 256
+ for opname, op in opcode.opmap.items():
+ if opname == "STOP_CODE":
+ # XXX opcode not implemented
+ continue
+ targets[op] = "TARGET_%s" % opname
+ f.write("static void *opcode_targets[256] = {\n")
+ f.write(",\n".join("\t&&%s" % s for s in targets))
+ f.write("\n};\n")
+
+
+if __name__ == "__main__":
+ import sys
+ assert len(sys.argv) < 3, "Too many arguments"
+ if len(sys.argv) == 2:
+ target = sys.argv[1]
+ else:
+ target = "Python/opcode_targets.h"
+ f = open(target, "w")
+ try:
+ write_contents(f)
+ finally:
+ f.close()
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
new file mode 100644
index 0000000..c662717
--- /dev/null
+++ b/Python/opcode_targets.h
@@ -0,0 +1,258 @@
+static void *opcode_targets[256] = {
+ &&_unknown_opcode,
+ &&TARGET_POP_TOP,
+ &&TARGET_ROT_TWO,
+ &&TARGET_ROT_THREE,
+ &&TARGET_DUP_TOP,
+ &&TARGET_ROT_FOUR,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_NOP,
+ &&TARGET_UNARY_POSITIVE,
+ &&TARGET_UNARY_NEGATIVE,
+ &&TARGET_UNARY_NOT,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_UNARY_INVERT,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_BINARY_POWER,
+ &&TARGET_BINARY_MULTIPLY,
+ &&_unknown_opcode,
+ &&TARGET_BINARY_MODULO,
+ &&TARGET_BINARY_ADD,
+ &&TARGET_BINARY_SUBTRACT,
+ &&TARGET_BINARY_SUBSCR,
+ &&TARGET_BINARY_FLOOR_DIVIDE,
+ &&TARGET_BINARY_TRUE_DIVIDE,
+ &&TARGET_INPLACE_FLOOR_DIVIDE,
+ &&TARGET_INPLACE_TRUE_DIVIDE,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_STORE_MAP,
+ &&TARGET_INPLACE_ADD,
+ &&TARGET_INPLACE_SUBTRACT,
+ &&TARGET_INPLACE_MULTIPLY,
+ &&_unknown_opcode,
+ &&TARGET_INPLACE_MODULO,
+ &&TARGET_STORE_SUBSCR,
+ &&TARGET_DELETE_SUBSCR,
+ &&TARGET_BINARY_LSHIFT,
+ &&TARGET_BINARY_RSHIFT,
+ &&TARGET_BINARY_AND,
+ &&TARGET_BINARY_XOR,
+ &&TARGET_BINARY_OR,
+ &&TARGET_INPLACE_POWER,
+ &&TARGET_GET_ITER,
+ &&TARGET_STORE_LOCALS,
+ &&TARGET_PRINT_EXPR,
+ &&TARGET_LOAD_BUILD_CLASS,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_INPLACE_LSHIFT,
+ &&TARGET_INPLACE_RSHIFT,
+ &&TARGET_INPLACE_AND,
+ &&TARGET_INPLACE_XOR,
+ &&TARGET_INPLACE_OR,
+ &&TARGET_BREAK_LOOP,
+ &&TARGET_WITH_CLEANUP,
+ &&_unknown_opcode,
+ &&TARGET_RETURN_VALUE,
+ &&TARGET_IMPORT_STAR,
+ &&_unknown_opcode,
+ &&TARGET_YIELD_VALUE,
+ &&TARGET_POP_BLOCK,
+ &&TARGET_END_FINALLY,
+ &&TARGET_POP_EXCEPT,
+ &&TARGET_STORE_NAME,
+ &&TARGET_DELETE_NAME,
+ &&TARGET_UNPACK_SEQUENCE,
+ &&TARGET_FOR_ITER,
+ &&TARGET_UNPACK_EX,
+ &&TARGET_STORE_ATTR,
+ &&TARGET_DELETE_ATTR,
+ &&TARGET_STORE_GLOBAL,
+ &&TARGET_DELETE_GLOBAL,
+ &&TARGET_DUP_TOPX,
+ &&TARGET_LOAD_CONST,
+ &&TARGET_LOAD_NAME,
+ &&TARGET_BUILD_TUPLE,
+ &&TARGET_BUILD_LIST,
+ &&TARGET_BUILD_SET,
+ &&TARGET_BUILD_MAP,
+ &&TARGET_LOAD_ATTR,
+ &&TARGET_COMPARE_OP,
+ &&TARGET_IMPORT_NAME,
+ &&TARGET_IMPORT_FROM,
+ &&TARGET_JUMP_FORWARD,
+ &&TARGET_JUMP_IF_FALSE,
+ &&TARGET_JUMP_IF_TRUE,
+ &&TARGET_JUMP_ABSOLUTE,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_LOAD_GLOBAL,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_CONTINUE_LOOP,
+ &&TARGET_SETUP_LOOP,
+ &&TARGET_SETUP_EXCEPT,
+ &&TARGET_SETUP_FINALLY,
+ &&_unknown_opcode,
+ &&TARGET_LOAD_FAST,
+ &&TARGET_STORE_FAST,
+ &&TARGET_DELETE_FAST,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_RAISE_VARARGS,
+ &&TARGET_CALL_FUNCTION,
+ &&TARGET_MAKE_FUNCTION,
+ &&TARGET_BUILD_SLICE,
+ &&TARGET_MAKE_CLOSURE,
+ &&TARGET_LOAD_CLOSURE,
+ &&TARGET_LOAD_DEREF,
+ &&TARGET_STORE_DEREF,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&TARGET_CALL_FUNCTION_VAR,
+ &&TARGET_CALL_FUNCTION_KW,
+ &&TARGET_CALL_FUNCTION_VAR_KW,
+ &&TARGET_EXTENDED_ARG,
+ &&_unknown_opcode,
+ &&TARGET_LIST_APPEND,
+ &&TARGET_SET_ADD,
+ &&TARGET_MAP_ADD,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode
+};