Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 1 | import errno |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 2 | import os |
| 3 | import re |
| 4 | import sys |
| 5 | import warnings |
| 6 | from inspect import isabstract |
| 7 | from test import support |
| 8 | |
| 9 | |
| 10 | def dash_R(the_module, test, indirect_test, huntrleaks): |
| 11 | """Run a test multiple times, looking for reference leaks. |
| 12 | |
| 13 | Returns: |
| 14 | False if the test didn't leak references; True if we detected refleaks. |
| 15 | """ |
| 16 | # This code is hackish and inelegant, but it seems to do the job. |
| 17 | import copyreg |
| 18 | import collections.abc |
| 19 | |
| 20 | if not hasattr(sys, 'gettotalrefcount'): |
| 21 | raise Exception("Tracking reference leaks requires a debug build " |
| 22 | "of Python") |
| 23 | |
| 24 | # Save current values for dash_R_cleanup() to restore. |
| 25 | fs = warnings.filters[:] |
| 26 | ps = copyreg.dispatch_table.copy() |
| 27 | pic = sys.path_importer_cache.copy() |
| 28 | try: |
| 29 | import zipimport |
| 30 | except ImportError: |
| 31 | zdc = None # Run unmodified on platforms without zipimport support |
| 32 | else: |
| 33 | zdc = zipimport._zip_directory_cache.copy() |
| 34 | abcs = {} |
| 35 | for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: |
| 36 | if not isabstract(abc): |
| 37 | continue |
| 38 | for obj in abc.__subclasses__() + [abc]: |
| 39 | abcs[obj] = obj._abc_registry.copy() |
| 40 | |
Victor Stinner | 6c2feab | 2017-09-01 13:05:27 +0200 | [diff] [blame] | 41 | # bpo-31217: Integer pool to get a single integer object for the same |
| 42 | # value. The pool is used to prevent false alarm when checking for memory |
| 43 | # block leaks. Fill the pool with values in -1000..1000 which are the most |
| 44 | # common (reference, memory block, file descriptor) differences. |
| 45 | int_pool = {value: value for value in range(-1000, 1000)} |
| 46 | def get_pooled_int(value): |
| 47 | return int_pool.setdefault(value, value) |
| 48 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 49 | nwarmup, ntracked, fname = huntrleaks |
| 50 | fname = os.path.join(support.SAVEDCWD, fname) |
| 51 | repcount = nwarmup + ntracked |
| 52 | rc_deltas = [0] * repcount |
| 53 | alloc_deltas = [0] * repcount |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 54 | fd_deltas = [0] * repcount |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 55 | |
| 56 | print("beginning", repcount, "repetitions", file=sys.stderr) |
Victor Stinner | f33536c | 2015-09-30 00:48:27 +0200 | [diff] [blame] | 57 | print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, |
| 58 | flush=True) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 59 | # initialize variables to make pyflakes quiet |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 60 | rc_before = alloc_before = fd_before = 0 |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 61 | for i in range(repcount): |
| 62 | indirect_test() |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 63 | alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, |
| 64 | abcs) |
Victor Stinner | 17a63e2 | 2017-02-08 13:06:08 +0100 | [diff] [blame] | 65 | print('.', end='', file=sys.stderr, flush=True) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 66 | if i >= nwarmup: |
Victor Stinner | 6c2feab | 2017-09-01 13:05:27 +0200 | [diff] [blame] | 67 | rc_deltas[i] = get_pooled_int(rc_after - rc_before) |
| 68 | alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) |
| 69 | fd_deltas[i] = get_pooled_int(fd_after - fd_before) |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 70 | alloc_before = alloc_after |
| 71 | rc_before = rc_after |
| 72 | fd_before = fd_after |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 73 | print(file=sys.stderr) |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 74 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 75 | # These checkers return False on success, True on failure |
| 76 | def check_rc_deltas(deltas): |
Victor Stinner | beeca6e | 2017-06-29 10:32:49 +0200 | [diff] [blame] | 77 | # Checker for reference counters and memomry blocks. |
| 78 | # |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 79 | # bpo-30776: Try to ignore false positives: |
| 80 | # |
| 81 | # [3, 0, 0] |
| 82 | # [0, 1, 0] |
| 83 | # [8, -8, 1] |
| 84 | # |
| 85 | # Expected leaks: |
| 86 | # |
| 87 | # [5, 5, 6] |
| 88 | # [10, 1, 1] |
| 89 | return all(delta >= 1 for delta in deltas) |
| 90 | |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 91 | def check_fd_deltas(deltas): |
| 92 | return any(deltas) |
| 93 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 94 | failed = False |
| 95 | for deltas, item_name, checker in [ |
| 96 | (rc_deltas, 'references', check_rc_deltas), |
Victor Stinner | beeca6e | 2017-06-29 10:32:49 +0200 | [diff] [blame] | 97 | (alloc_deltas, 'memory blocks', check_rc_deltas), |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 98 | (fd_deltas, 'file descriptors', check_fd_deltas) |
| 99 | ]: |
| 100 | # ignore warmup runs |
| 101 | deltas = deltas[nwarmup:] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 102 | if checker(deltas): |
| 103 | msg = '%s leaked %s %s, sum=%s' % ( |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 104 | test, deltas, item_name, sum(deltas)) |
Victor Stinner | f33536c | 2015-09-30 00:48:27 +0200 | [diff] [blame] | 105 | print(msg, file=sys.stderr, flush=True) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 106 | with open(fname, "a") as refrep: |
| 107 | print(msg, file=refrep) |
| 108 | refrep.flush() |
| 109 | failed = True |
| 110 | return failed |
| 111 | |
| 112 | |
| 113 | def dash_R_cleanup(fs, ps, pic, zdc, abcs): |
| 114 | import gc, copyreg |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 115 | import collections.abc |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 116 | from weakref import WeakSet |
| 117 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 118 | # Restore some original values. |
| 119 | warnings.filters[:] = fs |
| 120 | copyreg.dispatch_table.clear() |
| 121 | copyreg.dispatch_table.update(ps) |
| 122 | sys.path_importer_cache.clear() |
| 123 | sys.path_importer_cache.update(pic) |
| 124 | try: |
| 125 | import zipimport |
| 126 | except ImportError: |
| 127 | pass # Run unmodified on platforms without zipimport support |
| 128 | else: |
| 129 | zipimport._zip_directory_cache.clear() |
| 130 | zipimport._zip_directory_cache.update(zdc) |
| 131 | |
| 132 | # clear type cache |
| 133 | sys._clear_type_cache() |
| 134 | |
| 135 | # Clear ABC registries, restoring previously saved ABC registries. |
Ivan Levkivskyi | 7acffa2 | 2017-03-05 19:15:20 +0100 | [diff] [blame] | 136 | abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] |
| 137 | abs_classes = filter(isabstract, abs_classes) |
| 138 | if 'typing' in sys.modules: |
| 139 | t = sys.modules['typing'] |
| 140 | # These classes require special treatment because they do not appear |
| 141 | # in direct subclasses of collections.abc classes |
| 142 | abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] |
| 143 | for abc in abs_classes: |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 144 | for obj in abc.__subclasses__() + [abc]: |
| 145 | obj._abc_registry = abcs.get(obj, WeakSet()).copy() |
| 146 | obj._abc_cache.clear() |
| 147 | obj._abc_negative_cache.clear() |
| 148 | |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 149 | clear_caches() |
| 150 | |
| 151 | # Collect cyclic trash and read memory statistics immediately after. |
| 152 | func1 = sys.getallocatedblocks |
| 153 | func2 = sys.gettotalrefcount |
| 154 | gc.collect() |
Antoine Pitrou | 896145d | 2017-07-22 13:22:54 +0200 | [diff] [blame] | 155 | return func1(), func2(), support.fd_count() |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 156 | |
| 157 | |
| 158 | def clear_caches(): |
| 159 | import gc |
| 160 | |
| 161 | # Clear the warnings registry, so they can be displayed again |
| 162 | for mod in sys.modules.values(): |
| 163 | if hasattr(mod, '__warningregistry__'): |
| 164 | del mod.__warningregistry__ |
| 165 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 166 | # Flush standard output, so that buffered data is sent to the OS and |
| 167 | # associated Python objects are reclaimed. |
| 168 | for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): |
| 169 | if stream is not None: |
| 170 | stream.flush() |
| 171 | |
| 172 | # Clear assorted module caches. |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 173 | # Don't worry about resetting the cache if the module is not loaded |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 174 | try: |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 175 | distutils_dir_util = sys.modules['distutils.dir_util'] |
| 176 | except KeyError: |
| 177 | pass |
| 178 | else: |
| 179 | distutils_dir_util._path_created.clear() |
| 180 | re.purge() |
| 181 | |
| 182 | try: |
| 183 | _strptime = sys.modules['_strptime'] |
| 184 | except KeyError: |
| 185 | pass |
| 186 | else: |
| 187 | _strptime._regex_cache.clear() |
| 188 | |
| 189 | try: |
| 190 | urllib_parse = sys.modules['urllib.parse'] |
| 191 | except KeyError: |
| 192 | pass |
| 193 | else: |
| 194 | urllib_parse.clear_cache() |
| 195 | |
| 196 | try: |
| 197 | urllib_request = sys.modules['urllib.request'] |
| 198 | except KeyError: |
| 199 | pass |
| 200 | else: |
| 201 | urllib_request.urlcleanup() |
| 202 | |
| 203 | try: |
| 204 | linecache = sys.modules['linecache'] |
| 205 | except KeyError: |
| 206 | pass |
| 207 | else: |
| 208 | linecache.clearcache() |
| 209 | |
| 210 | try: |
| 211 | mimetypes = sys.modules['mimetypes'] |
| 212 | except KeyError: |
| 213 | pass |
| 214 | else: |
| 215 | mimetypes._default_mime_types() |
| 216 | |
| 217 | try: |
| 218 | filecmp = sys.modules['filecmp'] |
| 219 | except KeyError: |
| 220 | pass |
| 221 | else: |
| 222 | filecmp._cache.clear() |
| 223 | |
| 224 | try: |
| 225 | struct = sys.modules['struct'] |
| 226 | except KeyError: |
| 227 | pass |
| 228 | else: |
| 229 | struct._clearcache() |
| 230 | |
| 231 | try: |
| 232 | doctest = sys.modules['doctest'] |
| 233 | except KeyError: |
| 234 | pass |
| 235 | else: |
| 236 | doctest.master = None |
| 237 | |
| 238 | try: |
| 239 | ctypes = sys.modules['ctypes'] |
| 240 | except KeyError: |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 241 | pass |
| 242 | else: |
| 243 | ctypes._reset_cache() |
| 244 | |
Serhiy Storchaka | 04c954d | 2016-11-09 23:51:54 +0200 | [diff] [blame] | 245 | try: |
| 246 | typing = sys.modules['typing'] |
| 247 | except KeyError: |
| 248 | pass |
| 249 | else: |
| 250 | for f in typing._cleanups: |
| 251 | f() |
| 252 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 253 | gc.collect() |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 254 | |
| 255 | |
| 256 | def warm_caches(): |
| 257 | # char cache |
| 258 | s = bytes(range(256)) |
| 259 | for i in range(256): |
| 260 | s[i:i+1] |
| 261 | # unicode cache |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 262 | [chr(i) for i in range(256)] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 263 | # int cache |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 264 | list(range(-5, 257)) |