blob: adedfc5fb44b571e2829a8b803c6625cda6eb784 [file] [log] [blame]
Victor Stinner8ce8ff92014-03-10 11:05:07 +01001from collections import Sequence, Iterable
Victor Stinnered3b0bc2013-11-23 12:27:24 +01002from functools import total_ordering
3import fnmatch
Victor Stinner23f628d2014-02-16 23:53:38 +01004import linecache
Victor Stinnered3b0bc2013-11-23 12:27:24 +01005import os.path
6import pickle
7
8# Import types and functions implemented in C
9from _tracemalloc import *
10from _tracemalloc import _get_object_traceback, _get_traces
11
12
13def _format_size(size, sign):
14 for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
15 if abs(size) < 100 and unit != 'B':
16 # 3 digits (xx.x UNIT)
17 if sign:
18 return "%+.1f %s" % (size, unit)
19 else:
20 return "%.1f %s" % (size, unit)
21 if abs(size) < 10 * 1024 or unit == 'TiB':
22 # 4 or 5 digits (xxxx UNIT)
23 if sign:
24 return "%+.0f %s" % (size, unit)
25 else:
26 return "%.0f %s" % (size, unit)
27 size /= 1024
28
29
30class Statistic:
31 """
32 Statistic difference on memory allocations between two Snapshot instance.
33 """
34
35 __slots__ = ('traceback', 'size', 'count')
36
37 def __init__(self, traceback, size, count):
38 self.traceback = traceback
39 self.size = size
40 self.count = count
41
42 def __hash__(self):
Victor Stinner802a4842013-11-26 10:16:25 +010043 return hash((self.traceback, self.size, self.count))
Victor Stinnered3b0bc2013-11-23 12:27:24 +010044
45 def __eq__(self, other):
46 return (self.traceback == other.traceback
47 and self.size == other.size
48 and self.count == other.count)
49
50 def __str__(self):
51 text = ("%s: size=%s, count=%i"
52 % (self.traceback,
53 _format_size(self.size, False),
54 self.count))
55 if self.count:
56 average = self.size / self.count
57 text += ", average=%s" % _format_size(average, False)
58 return text
59
60 def __repr__(self):
61 return ('<Statistic traceback=%r size=%i count=%i>'
62 % (self.traceback, self.size, self.count))
63
64 def _sort_key(self):
65 return (self.size, self.count, self.traceback)
66
67
68class StatisticDiff:
69 """
70 Statistic difference on memory allocations between an old and a new
71 Snapshot instance.
72 """
73 __slots__ = ('traceback', 'size', 'size_diff', 'count', 'count_diff')
74
75 def __init__(self, traceback, size, size_diff, count, count_diff):
76 self.traceback = traceback
77 self.size = size
78 self.size_diff = size_diff
79 self.count = count
80 self.count_diff = count_diff
81
82 def __hash__(self):
Victor Stinner802a4842013-11-26 10:16:25 +010083 return hash((self.traceback, self.size, self.size_diff,
84 self.count, self.count_diff))
Victor Stinnered3b0bc2013-11-23 12:27:24 +010085
86 def __eq__(self, other):
87 return (self.traceback == other.traceback
88 and self.size == other.size
89 and self.size_diff == other.size_diff
90 and self.count == other.count
91 and self.count_diff == other.count_diff)
92
93 def __str__(self):
94 text = ("%s: size=%s (%s), count=%i (%+i)"
95 % (self.traceback,
96 _format_size(self.size, False),
97 _format_size(self.size_diff, True),
98 self.count,
99 self.count_diff))
100 if self.count:
101 average = self.size / self.count
102 text += ", average=%s" % _format_size(average, False)
103 return text
104
105 def __repr__(self):
106 return ('<StatisticDiff traceback=%r size=%i (%+i) count=%i (%+i)>'
107 % (self.traceback, self.size, self.size_diff,
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100108 self.count, self.count_diff))
109
110 def _sort_key(self):
111 return (abs(self.size_diff), self.size,
112 abs(self.count_diff), self.count,
113 self.traceback)
114
115
116def _compare_grouped_stats(old_group, new_group):
117 statistics = []
118 for traceback, stat in new_group.items():
119 previous = old_group.pop(traceback, None)
120 if previous is not None:
121 stat = StatisticDiff(traceback,
Victor Stinnerd81999a2014-03-06 17:06:04 +0100122 stat.size, stat.size - previous.size,
123 stat.count, stat.count - previous.count)
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100124 else:
125 stat = StatisticDiff(traceback,
Victor Stinnerd81999a2014-03-06 17:06:04 +0100126 stat.size, stat.size,
127 stat.count, stat.count)
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100128 statistics.append(stat)
129
130 for traceback, stat in old_group.items():
131 stat = StatisticDiff(traceback, 0, -stat.size, 0, -stat.count)
132 statistics.append(stat)
133 return statistics
134
135
136@total_ordering
137class Frame:
138 """
139 Frame of a traceback.
140 """
141 __slots__ = ("_frame",)
142
143 def __init__(self, frame):
Victor Stinner733e50a2014-03-06 17:06:41 +0100144 # frame is a tuple: (filename: str, lineno: int)
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100145 self._frame = frame
146
147 @property
148 def filename(self):
149 return self._frame[0]
150
151 @property
152 def lineno(self):
153 return self._frame[1]
154
155 def __eq__(self, other):
156 return (self._frame == other._frame)
157
158 def __lt__(self, other):
159 return (self._frame < other._frame)
160
161 def __hash__(self):
162 return hash(self._frame)
163
164 def __str__(self):
165 return "%s:%s" % (self.filename, self.lineno)
166
167 def __repr__(self):
168 return "<Frame filename=%r lineno=%r>" % (self.filename, self.lineno)
169
170
171@total_ordering
172class Traceback(Sequence):
173 """
174 Sequence of Frame instances sorted from the most recent frame
175 to the oldest frame.
176 """
177 __slots__ = ("_frames",)
178
179 def __init__(self, frames):
180 Sequence.__init__(self)
Victor Stinner733e50a2014-03-06 17:06:41 +0100181 # frames is a tuple of frame tuples: see Frame constructor for the
182 # format of a frame tuple
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100183 self._frames = frames
184
185 def __len__(self):
186 return len(self._frames)
187
188 def __getitem__(self, index):
Victor Stinner524be302014-02-01 04:07:02 +0100189 if isinstance(index, slice):
190 return tuple(Frame(trace) for trace in self._frames[index])
191 else:
192 return Frame(self._frames[index])
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100193
194 def __contains__(self, frame):
195 return frame._frame in self._frames
196
197 def __hash__(self):
198 return hash(self._frames)
199
200 def __eq__(self, other):
201 return (self._frames == other._frames)
202
203 def __lt__(self, other):
204 return (self._frames < other._frames)
205
206 def __str__(self):
207 return str(self[0])
208
209 def __repr__(self):
210 return "<Traceback %r>" % (tuple(self),)
211
Victor Stinner23f628d2014-02-16 23:53:38 +0100212 def format(self, limit=None):
213 lines = []
214 if limit is not None and limit < 0:
215 return lines
216 for frame in self[:limit]:
217 lines.append(' File "%s", line %s'
218 % (frame.filename, frame.lineno))
219 line = linecache.getline(frame.filename, frame.lineno).strip()
220 if line:
221 lines.append(' %s' % line)
222 return lines
223
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100224
225def get_object_traceback(obj):
226 """
227 Get the traceback where the Python object *obj* was allocated.
228 Return a Traceback instance.
229
230 Return None if the tracemalloc module is not tracing memory allocations or
231 did not trace the allocation of the object.
232 """
233 frames = _get_object_traceback(obj)
234 if frames is not None:
235 return Traceback(frames)
236 else:
237 return None
238
239
240class Trace:
241 """
242 Trace of a memory block.
243 """
244 __slots__ = ("_trace",)
245
246 def __init__(self, trace):
Victor Stinner733e50a2014-03-06 17:06:41 +0100247 # trace is a tuple: (size, traceback), see Traceback constructor
248 # for the format of the traceback tuple
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100249 self._trace = trace
250
251 @property
252 def size(self):
253 return self._trace[0]
254
255 @property
256 def traceback(self):
257 return Traceback(self._trace[1])
258
259 def __eq__(self, other):
260 return (self._trace == other._trace)
261
262 def __hash__(self):
263 return hash(self._trace)
264
265 def __str__(self):
266 return "%s: %s" % (self.traceback, _format_size(self.size, False))
267
268 def __repr__(self):
269 return ("<Trace size=%s, traceback=%r>"
270 % (_format_size(self.size, False), self.traceback))
271
272
273class _Traces(Sequence):
274 def __init__(self, traces):
275 Sequence.__init__(self)
Victor Stinner733e50a2014-03-06 17:06:41 +0100276 # traces is a tuple of trace tuples: see Trace constructor
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100277 self._traces = traces
278
279 def __len__(self):
280 return len(self._traces)
281
282 def __getitem__(self, index):
Victor Stinner524be302014-02-01 04:07:02 +0100283 if isinstance(index, slice):
284 return tuple(Trace(trace) for trace in self._traces[index])
285 else:
286 return Trace(self._traces[index])
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100287
288 def __contains__(self, trace):
289 return trace._trace in self._traces
290
291 def __eq__(self, other):
292 return (self._traces == other._traces)
293
294 def __repr__(self):
295 return "<Traces len=%s>" % len(self)
296
297
298def _normalize_filename(filename):
299 filename = os.path.normcase(filename)
300 if filename.endswith(('.pyc', '.pyo')):
301 filename = filename[:-1]
302 return filename
303
304
305class Filter:
306 def __init__(self, inclusive, filename_pattern,
307 lineno=None, all_frames=False):
308 self.inclusive = inclusive
309 self._filename_pattern = _normalize_filename(filename_pattern)
310 self.lineno = lineno
311 self.all_frames = all_frames
312
313 @property
314 def filename_pattern(self):
315 return self._filename_pattern
316
317 def __match_frame(self, filename, lineno):
318 filename = _normalize_filename(filename)
319 if not fnmatch.fnmatch(filename, self._filename_pattern):
320 return False
321 if self.lineno is None:
322 return True
323 else:
324 return (lineno == self.lineno)
325
326 def _match_frame(self, filename, lineno):
327 return self.__match_frame(filename, lineno) ^ (not self.inclusive)
328
329 def _match_traceback(self, traceback):
330 if self.all_frames:
331 if any(self.__match_frame(filename, lineno)
332 for filename, lineno in traceback):
333 return self.inclusive
334 else:
335 return (not self.inclusive)
336 else:
337 filename, lineno = traceback[0]
338 return self._match_frame(filename, lineno)
339
340
341class Snapshot:
342 """
343 Snapshot of traces of memory blocks allocated by Python.
344 """
345
346 def __init__(self, traces, traceback_limit):
Victor Stinner733e50a2014-03-06 17:06:41 +0100347 # traces is a tuple of trace tuples: see _Traces constructor for
348 # the exact format
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100349 self.traces = _Traces(traces)
350 self.traceback_limit = traceback_limit
351
352 def dump(self, filename):
353 """
354 Write the snapshot into a file.
355 """
356 with open(filename, "wb") as fp:
357 pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL)
358
359 @staticmethod
360 def load(filename):
361 """
362 Load a snapshot from a file.
363 """
364 with open(filename, "rb") as fp:
365 return pickle.load(fp)
366
367 def _filter_trace(self, include_filters, exclude_filters, trace):
368 traceback = trace[1]
369 if include_filters:
370 if not any(trace_filter._match_traceback(traceback)
371 for trace_filter in include_filters):
372 return False
373 if exclude_filters:
374 if any(not trace_filter._match_traceback(traceback)
375 for trace_filter in exclude_filters):
376 return False
377 return True
378
379 def filter_traces(self, filters):
380 """
381 Create a new Snapshot instance with a filtered traces sequence, filters
382 is a list of Filter instances. If filters is an empty list, return a
383 new Snapshot instance with a copy of the traces.
384 """
Victor Stinner8ce8ff92014-03-10 11:05:07 +0100385 if not isinstance(filters, Iterable):
386 raise TypeError("filters must be a list of filters, not %s"
387 % type(filters).__name__)
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100388 if filters:
389 include_filters = []
390 exclude_filters = []
391 for trace_filter in filters:
392 if trace_filter.inclusive:
393 include_filters.append(trace_filter)
394 else:
395 exclude_filters.append(trace_filter)
396 new_traces = [trace for trace in self.traces._traces
397 if self._filter_trace(include_filters,
398 exclude_filters,
399 trace)]
400 else:
401 new_traces = self.traces._traces.copy()
402 return Snapshot(new_traces, self.traceback_limit)
403
404 def _group_by(self, key_type, cumulative):
405 if key_type not in ('traceback', 'filename', 'lineno'):
406 raise ValueError("unknown key_type: %r" % (key_type,))
407 if cumulative and key_type not in ('lineno', 'filename'):
408 raise ValueError("cumulative mode cannot by used "
409 "with key type %r" % key_type)
Victor Stinnered3b0bc2013-11-23 12:27:24 +0100410
411 stats = {}
412 tracebacks = {}
413 if not cumulative:
414 for trace in self.traces._traces:
415 size, trace_traceback = trace
416 try:
417 traceback = tracebacks[trace_traceback]
418 except KeyError:
419 if key_type == 'traceback':
420 frames = trace_traceback
421 elif key_type == 'lineno':
422 frames = trace_traceback[:1]
423 else: # key_type == 'filename':
424 frames = ((trace_traceback[0][0], 0),)
425 traceback = Traceback(frames)
426 tracebacks[trace_traceback] = traceback
427 try:
428 stat = stats[traceback]
429 stat.size += size
430 stat.count += 1
431 except KeyError:
432 stats[traceback] = Statistic(traceback, size, 1)
433 else:
434 # cumulative statistics
435 for trace in self.traces._traces:
436 size, trace_traceback = trace
437 for frame in trace_traceback:
438 try:
439 traceback = tracebacks[frame]
440 except KeyError:
441 if key_type == 'lineno':
442 frames = (frame,)
443 else: # key_type == 'filename':
444 frames = ((frame[0], 0),)
445 traceback = Traceback(frames)
446 tracebacks[frame] = traceback
447 try:
448 stat = stats[traceback]
449 stat.size += size
450 stat.count += 1
451 except KeyError:
452 stats[traceback] = Statistic(traceback, size, 1)
453 return stats
454
455 def statistics(self, key_type, cumulative=False):
456 """
457 Group statistics by key_type. Return a sorted list of Statistic
458 instances.
459 """
460 grouped = self._group_by(key_type, cumulative)
461 statistics = list(grouped.values())
462 statistics.sort(reverse=True, key=Statistic._sort_key)
463 return statistics
464
465 def compare_to(self, old_snapshot, key_type, cumulative=False):
466 """
467 Compute the differences with an old snapshot old_snapshot. Get
468 statistics as a sorted list of StatisticDiff instances, grouped by
469 group_by.
470 """
471 new_group = self._group_by(key_type, cumulative)
472 old_group = old_snapshot._group_by(key_type, cumulative)
473 statistics = _compare_grouped_stats(old_group, new_group)
474 statistics.sort(reverse=True, key=StatisticDiff._sort_key)
475 return statistics
476
477
478def take_snapshot():
479 """
480 Take a snapshot of traces of memory blocks allocated by Python.
481 """
482 if not is_tracing():
483 raise RuntimeError("the tracemalloc module must be tracing memory "
484 "allocations to take a snapshot")
485 traces = _get_traces()
486 traceback_limit = get_traceback_limit()
487 return Snapshot(traces, traceback_limit)