Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 1 | """This module includes tests of the code object representation. |
| 2 | |
| 3 | >>> def f(x): |
| 4 | ... def g(y): |
| 5 | ... return x + y |
| 6 | ... return g |
| 7 | ... |
| 8 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 9 | >>> dump(f.__code__) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 10 | name: f |
| 11 | argcount: 1 |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 12 | kwonlyargcount: 0 |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 13 | names: () |
| 14 | varnames: ('x', 'g') |
| 15 | cellvars: ('x',) |
| 16 | freevars: () |
| 17 | nlocals: 2 |
| 18 | flags: 3 |
Antoine Pitrou | 86a36b5 | 2011-11-25 18:56:07 +0100 | [diff] [blame] | 19 | consts: ('None', '<code object g>', "'f.<locals>.g'") |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 20 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 21 | >>> dump(f(4).__code__) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 22 | name: g |
| 23 | argcount: 1 |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 24 | kwonlyargcount: 0 |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 25 | names: () |
| 26 | varnames: ('y',) |
| 27 | cellvars: () |
| 28 | freevars: ('x',) |
| 29 | nlocals: 1 |
| 30 | flags: 19 |
| 31 | consts: ('None',) |
| 32 | |
| 33 | >>> def h(x, y): |
| 34 | ... a = x + y |
| 35 | ... b = x - y |
| 36 | ... c = a * b |
| 37 | ... return c |
Tim Peters | 536cf99 | 2005-12-25 23:18:31 +0000 | [diff] [blame] | 38 | ... |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 39 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 40 | >>> dump(h.__code__) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 41 | name: h |
| 42 | argcount: 2 |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 43 | kwonlyargcount: 0 |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 44 | names: () |
| 45 | varnames: ('x', 'y', 'a', 'b', 'c') |
| 46 | cellvars: () |
| 47 | freevars: () |
| 48 | nlocals: 5 |
| 49 | flags: 67 |
| 50 | consts: ('None',) |
| 51 | |
| 52 | >>> def attrs(obj): |
Guido van Rossum | 7131f84 | 2007-02-09 20:13:25 +0000 | [diff] [blame] | 53 | ... print(obj.attr1) |
| 54 | ... print(obj.attr2) |
| 55 | ... print(obj.attr3) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 56 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 57 | >>> dump(attrs.__code__) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 58 | name: attrs |
| 59 | argcount: 1 |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 60 | kwonlyargcount: 0 |
Georg Brandl | 88fc664 | 2007-02-09 21:28:07 +0000 | [diff] [blame] | 61 | names: ('print', 'attr1', 'attr2', 'attr3') |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 62 | varnames: ('obj',) |
| 63 | cellvars: () |
| 64 | freevars: () |
| 65 | nlocals: 1 |
| 66 | flags: 67 |
| 67 | consts: ('None',) |
| 68 | |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 69 | >>> def optimize_away(): |
| 70 | ... 'doc string' |
| 71 | ... 'not a docstring' |
| 72 | ... 53 |
Guido van Rossum | e2a383d | 2007-01-15 16:59:06 +0000 | [diff] [blame] | 73 | ... 0x53 |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 74 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 75 | >>> dump(optimize_away.__code__) |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 76 | name: optimize_away |
| 77 | argcount: 0 |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 78 | kwonlyargcount: 0 |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 79 | names: () |
| 80 | varnames: () |
| 81 | cellvars: () |
| 82 | freevars: () |
| 83 | nlocals: 0 |
| 84 | flags: 67 |
| 85 | consts: ("'doc string'", 'None') |
| 86 | |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 87 | >>> def keywordonly_args(a,b,*,k1): |
| 88 | ... return a,b,k1 |
| 89 | ... |
| 90 | |
Neal Norwitz | 221085d | 2007-02-25 20:55:47 +0000 | [diff] [blame] | 91 | >>> dump(keywordonly_args.__code__) |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 92 | name: keywordonly_args |
| 93 | argcount: 2 |
| 94 | kwonlyargcount: 1 |
| 95 | names: () |
| 96 | varnames: ('a', 'b', 'k1') |
| 97 | cellvars: () |
| 98 | freevars: () |
| 99 | nlocals: 3 |
| 100 | flags: 67 |
| 101 | consts: ('None',) |
| 102 | |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 103 | """ |
| 104 | |
Nick Coghlan | 078f181 | 2017-12-03 11:12:20 +1000 | [diff] [blame] | 105 | import inspect |
Serhiy Storchaka | 00a0fc1 | 2016-09-30 10:07:26 +0300 | [diff] [blame] | 106 | import sys |
Dino Viehland | f3cffd2 | 2017-06-21 14:44:36 -0700 | [diff] [blame] | 107 | import threading |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 108 | import unittest |
Collin Winter | 4222e9c | 2010-03-18 22:46:40 +0000 | [diff] [blame] | 109 | import weakref |
Victor Stinner | a4b091e | 2017-06-23 15:08:55 +0200 | [diff] [blame] | 110 | try: |
| 111 | import ctypes |
| 112 | except ImportError: |
| 113 | ctypes = None |
Dino Viehland | f3cffd2 | 2017-06-21 14:44:36 -0700 | [diff] [blame] | 114 | from test.support import (run_doctest, run_unittest, cpython_only, |
| 115 | check_impl_detail) |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 116 | |
Collin Winter | 4222e9c | 2010-03-18 22:46:40 +0000 | [diff] [blame] | 117 | |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 118 | def consts(t): |
| 119 | """Yield a doctest-safe sequence of object reprs.""" |
| 120 | for elt in t: |
| 121 | r = repr(elt) |
| 122 | if r.startswith("<code object"): |
| 123 | yield "<code object %s>" % elt.co_name |
| 124 | else: |
| 125 | yield r |
| 126 | |
| 127 | def dump(co): |
| 128 | """Print out a text representation of a code object.""" |
Guido van Rossum | 4f72a78 | 2006-10-27 23:31:49 +0000 | [diff] [blame] | 129 | for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", |
| 130 | "cellvars", "freevars", "nlocals", "flags"]: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 131 | print("%s: %s" % (attr, getattr(co, "co_" + attr))) |
| 132 | print("consts:", tuple(consts(co.co_consts))) |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 133 | |
Nick Coghlan | 078f181 | 2017-12-03 11:12:20 +1000 | [diff] [blame] | 134 | # Needed for test_closure_injection below |
| 135 | # Defined at global scope to avoid implicitly closing over __class__ |
| 136 | def external_getitem(self, i): |
| 137 | return f"Foreign getitem: {super().__getitem__(i)}" |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 138 | |
| 139 | class CodeTest(unittest.TestCase): |
| 140 | |
Serhiy Storchaka | 5cfc79d | 2014-02-07 10:06:39 +0200 | [diff] [blame] | 141 | @cpython_only |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 142 | def test_newempty(self): |
Serhiy Storchaka | 5cfc79d | 2014-02-07 10:06:39 +0200 | [diff] [blame] | 143 | import _testcapi |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 144 | co = _testcapi.code_newempty("filename", "funcname", 15) |
Ezio Melotti | b3aedd4 | 2010-11-20 19:04:17 +0000 | [diff] [blame] | 145 | self.assertEqual(co.co_filename, "filename") |
| 146 | self.assertEqual(co.co_name, "funcname") |
| 147 | self.assertEqual(co.co_firstlineno, 15) |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 148 | |
Nick Coghlan | 078f181 | 2017-12-03 11:12:20 +1000 | [diff] [blame] | 149 | @cpython_only |
| 150 | def test_closure_injection(self): |
| 151 | # From https://bugs.python.org/issue32176 |
| 152 | from types import FunctionType, CodeType |
| 153 | |
| 154 | def create_closure(__class__): |
| 155 | return (lambda: __class__).__closure__ |
| 156 | |
| 157 | def new_code(c): |
| 158 | '''A new code object with a __class__ cell added to freevars''' |
| 159 | return CodeType( |
| 160 | c.co_argcount, c.co_kwonlyargcount, c.co_nlocals, |
| 161 | c.co_stacksize, c.co_flags, c.co_code, c.co_consts, c.co_names, |
| 162 | c.co_varnames, c.co_filename, c.co_name, c.co_firstlineno, |
| 163 | c.co_lnotab, c.co_freevars + ('__class__',), c.co_cellvars) |
| 164 | |
| 165 | def add_foreign_method(cls, name, f): |
| 166 | code = new_code(f.__code__) |
| 167 | assert not f.__closure__ |
| 168 | closure = create_closure(cls) |
| 169 | defaults = f.__defaults__ |
| 170 | setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) |
| 171 | |
| 172 | class List(list): |
| 173 | pass |
| 174 | |
| 175 | add_foreign_method(List, "__getitem__", external_getitem) |
| 176 | |
| 177 | # Ensure the closure injection actually worked |
| 178 | function = List.__getitem__ |
| 179 | class_ref = function.__closure__[0].cell_contents |
| 180 | self.assertIs(class_ref, List) |
| 181 | |
| 182 | # Ensure the code correctly indicates it accesses a free variable |
| 183 | self.assertFalse(function.__code__.co_flags & inspect.CO_NOFREE, |
| 184 | hex(function.__code__.co_flags)) |
| 185 | |
| 186 | # Ensure the zero-arg super() call in the injected method works |
| 187 | obj = List([1, 2, 3]) |
| 188 | self.assertEqual(obj[0], "Foreign getitem: 1") |
Serhiy Storchaka | 09f3d08 | 2016-10-04 18:17:22 +0300 | [diff] [blame] | 189 | |
| 190 | def isinterned(s): |
| 191 | return s is sys.intern(('_' + s + '_')[1:-1]) |
| 192 | |
Serhiy Storchaka | 00a0fc1 | 2016-09-30 10:07:26 +0300 | [diff] [blame] | 193 | class CodeConstsTest(unittest.TestCase): |
| 194 | |
| 195 | def find_const(self, consts, value): |
| 196 | for v in consts: |
| 197 | if v == value: |
| 198 | return v |
Serhiy Storchaka | 09f3d08 | 2016-10-04 18:17:22 +0300 | [diff] [blame] | 199 | self.assertIn(value, consts) # raises an exception |
| 200 | self.fail('Should never be reached') |
Serhiy Storchaka | 00a0fc1 | 2016-09-30 10:07:26 +0300 | [diff] [blame] | 201 | |
| 202 | def assertIsInterned(self, s): |
Serhiy Storchaka | 09f3d08 | 2016-10-04 18:17:22 +0300 | [diff] [blame] | 203 | if not isinterned(s): |
Serhiy Storchaka | 00a0fc1 | 2016-09-30 10:07:26 +0300 | [diff] [blame] | 204 | self.fail('String %r is not interned' % (s,)) |
| 205 | |
Serhiy Storchaka | 09f3d08 | 2016-10-04 18:17:22 +0300 | [diff] [blame] | 206 | def assertIsNotInterned(self, s): |
| 207 | if isinterned(s): |
| 208 | self.fail('String %r is interned' % (s,)) |
| 209 | |
Serhiy Storchaka | 00a0fc1 | 2016-09-30 10:07:26 +0300 | [diff] [blame] | 210 | @cpython_only |
| 211 | def test_interned_string(self): |
| 212 | co = compile('res = "str_value"', '?', 'exec') |
| 213 | v = self.find_const(co.co_consts, 'str_value') |
| 214 | self.assertIsInterned(v) |
| 215 | |
| 216 | @cpython_only |
| 217 | def test_interned_string_in_tuple(self): |
| 218 | co = compile('res = ("str_value",)', '?', 'exec') |
| 219 | v = self.find_const(co.co_consts, ('str_value',)) |
| 220 | self.assertIsInterned(v[0]) |
| 221 | |
| 222 | @cpython_only |
| 223 | def test_interned_string_in_frozenset(self): |
| 224 | co = compile('res = a in {"str_value"}', '?', 'exec') |
| 225 | v = self.find_const(co.co_consts, frozenset(('str_value',))) |
| 226 | self.assertIsInterned(tuple(v)[0]) |
| 227 | |
| 228 | @cpython_only |
| 229 | def test_interned_string_default(self): |
| 230 | def f(a='str_value'): |
| 231 | return a |
| 232 | self.assertIsInterned(f()) |
| 233 | |
Serhiy Storchaka | 09f3d08 | 2016-10-04 18:17:22 +0300 | [diff] [blame] | 234 | @cpython_only |
| 235 | def test_interned_string_with_null(self): |
| 236 | co = compile(r'res = "str\0value!"', '?', 'exec') |
| 237 | v = self.find_const(co.co_consts, 'str\0value!') |
| 238 | self.assertIsNotInterned(v) |
| 239 | |
Alexandre Vassalotti | 7b82b40 | 2009-07-21 04:30:03 +0000 | [diff] [blame] | 240 | |
Collin Winter | 4222e9c | 2010-03-18 22:46:40 +0000 | [diff] [blame] | 241 | class CodeWeakRefTest(unittest.TestCase): |
| 242 | |
| 243 | def test_basic(self): |
| 244 | # Create a code object in a clean environment so that we know we have |
| 245 | # the only reference to it left. |
| 246 | namespace = {} |
| 247 | exec("def f(): pass", globals(), namespace) |
| 248 | f = namespace["f"] |
| 249 | del namespace |
| 250 | |
| 251 | self.called = False |
| 252 | def callback(code): |
| 253 | self.called = True |
| 254 | |
| 255 | # f is now the last reference to the function, and through it, the code |
| 256 | # object. While we hold it, check that we can create a weakref and |
| 257 | # deref it. Then delete it, and check that the callback gets called and |
| 258 | # the reference dies. |
| 259 | coderef = weakref.ref(f.__code__, callback) |
| 260 | self.assertTrue(bool(coderef())) |
| 261 | del f |
| 262 | self.assertFalse(bool(coderef())) |
| 263 | self.assertTrue(self.called) |
| 264 | |
| 265 | |
Victor Stinner | a4b091e | 2017-06-23 15:08:55 +0200 | [diff] [blame] | 266 | if check_impl_detail(cpython=True) and ctypes is not None: |
Dino Viehland | f3cffd2 | 2017-06-21 14:44:36 -0700 | [diff] [blame] | 267 | py = ctypes.pythonapi |
| 268 | freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) |
| 269 | |
| 270 | RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex |
| 271 | RequestCodeExtraIndex.argtypes = (freefunc,) |
| 272 | RequestCodeExtraIndex.restype = ctypes.c_ssize_t |
| 273 | |
| 274 | SetExtra = py._PyCode_SetExtra |
| 275 | SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) |
| 276 | SetExtra.restype = ctypes.c_int |
| 277 | |
| 278 | GetExtra = py._PyCode_GetExtra |
| 279 | GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, |
| 280 | ctypes.POINTER(ctypes.c_voidp)) |
| 281 | GetExtra.restype = ctypes.c_int |
| 282 | |
| 283 | LAST_FREED = None |
| 284 | def myfree(ptr): |
| 285 | global LAST_FREED |
| 286 | LAST_FREED = ptr |
| 287 | |
| 288 | FREE_FUNC = freefunc(myfree) |
| 289 | FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) |
| 290 | |
| 291 | class CoExtra(unittest.TestCase): |
| 292 | def get_func(self): |
| 293 | # Defining a function causes the containing function to have a |
| 294 | # reference to the code object. We need the code objects to go |
| 295 | # away, so we eval a lambda. |
| 296 | return eval('lambda:42') |
| 297 | |
| 298 | def test_get_non_code(self): |
| 299 | f = self.get_func() |
| 300 | |
| 301 | self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, |
| 302 | ctypes.c_voidp(100)) |
| 303 | self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, |
| 304 | ctypes.c_voidp(100)) |
| 305 | |
| 306 | def test_bad_index(self): |
| 307 | f = self.get_func() |
| 308 | self.assertRaises(SystemError, SetExtra, f.__code__, |
| 309 | FREE_INDEX+100, ctypes.c_voidp(100)) |
| 310 | self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, |
| 311 | ctypes.c_voidp(100)), 0) |
| 312 | |
| 313 | def test_free_called(self): |
| 314 | # Verify that the provided free function gets invoked |
| 315 | # when the code object is cleaned up. |
| 316 | f = self.get_func() |
| 317 | |
| 318 | SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) |
| 319 | del f |
| 320 | self.assertEqual(LAST_FREED, 100) |
| 321 | |
| 322 | def test_get_set(self): |
| 323 | # Test basic get/set round tripping. |
| 324 | f = self.get_func() |
| 325 | |
| 326 | extra = ctypes.c_voidp() |
| 327 | |
| 328 | SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) |
| 329 | # reset should free... |
| 330 | SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) |
| 331 | self.assertEqual(LAST_FREED, 200) |
| 332 | |
| 333 | extra = ctypes.c_voidp() |
| 334 | GetExtra(f.__code__, FREE_INDEX, extra) |
| 335 | self.assertEqual(extra.value, 300) |
| 336 | del f |
| 337 | |
| 338 | def test_free_different_thread(self): |
| 339 | # Freeing a code object on a different thread then |
| 340 | # where the co_extra was set should be safe. |
| 341 | f = self.get_func() |
| 342 | class ThreadTest(threading.Thread): |
| 343 | def __init__(self, f, test): |
| 344 | super().__init__() |
| 345 | self.f = f |
| 346 | self.test = test |
| 347 | def run(self): |
| 348 | del self.f |
| 349 | self.test.assertEqual(LAST_FREED, 500) |
| 350 | |
| 351 | SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) |
| 352 | tt = ThreadTest(f, self) |
| 353 | del f |
| 354 | tt.start() |
| 355 | tt.join() |
| 356 | self.assertEqual(LAST_FREED, 500) |
| 357 | |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 358 | def test_main(verbose=None): |
Jeremy Hylton | 3e0055f | 2005-10-20 19:59:25 +0000 | [diff] [blame] | 359 | from test import test_code |
| 360 | run_doctest(test_code, verbose) |
Dino Viehland | f3cffd2 | 2017-06-21 14:44:36 -0700 | [diff] [blame] | 361 | tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] |
Victor Stinner | a4b091e | 2017-06-23 15:08:55 +0200 | [diff] [blame] | 362 | if check_impl_detail(cpython=True) and ctypes is not None: |
Dino Viehland | f3cffd2 | 2017-06-21 14:44:36 -0700 | [diff] [blame] | 363 | tests.append(CoExtra) |
| 364 | run_unittest(*tests) |
Benjamin Peterson | ad9d48d | 2008-04-02 21:49:44 +0000 | [diff] [blame] | 365 | |
Collin Winter | 4222e9c | 2010-03-18 22:46:40 +0000 | [diff] [blame] | 366 | if __name__ == "__main__": |
Benjamin Peterson | ad9d48d | 2008-04-02 21:49:44 +0000 | [diff] [blame] | 367 | test_main() |