Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # Copyright 2010 the V8 project authors. All rights reserved. |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following |
| 12 | # disclaimer in the documentation and/or other materials provided |
| 13 | # with the distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived |
| 16 | # from this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | # |
| 30 | |
| 31 | # |
| 32 | # This is an utility for plotting charts based on GC traces produced by V8 when |
| 33 | # run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual |
| 34 | # plotting. |
| 35 | # |
| 36 | # Usage: gc-nvp-trace-processor.py <GC-trace-filename> |
| 37 | # |
| 38 | |
| 39 | |
| 40 | from __future__ import with_statement |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 41 | import sys, types, subprocess, math |
| 42 | import gc_nvp_common |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 43 | |
| 44 | def flatten(l): |
| 45 | flat = [] |
| 46 | for i in l: flat.extend(i) |
| 47 | return flat |
| 48 | |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 49 | def gnuplot(script): |
| 50 | gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE) |
| 51 | gnuplot.stdin.write(script) |
| 52 | gnuplot.stdin.close() |
| 53 | gnuplot.wait() |
| 54 | |
| 55 | x1y1 = 'x1y1' |
| 56 | x1y2 = 'x1y2' |
| 57 | x2y1 = 'x2y1' |
| 58 | x2y2 = 'x2y2' |
| 59 | |
| 60 | class Item(object): |
| 61 | def __init__(self, title, field, axis = x1y1, **keywords): |
| 62 | self.title = title |
| 63 | self.axis = axis |
| 64 | self.props = keywords |
| 65 | if type(field) is types.ListType: |
| 66 | self.field = field |
| 67 | else: |
| 68 | self.field = [field] |
| 69 | |
| 70 | def fieldrefs(self): |
| 71 | return self.field |
| 72 | |
| 73 | def to_gnuplot(self, context): |
| 74 | args = ['"%s"' % context.datafile, |
| 75 | 'using %s' % context.format_fieldref(self.field), |
| 76 | 'title "%s"' % self.title, |
| 77 | 'axis %s' % self.axis] |
| 78 | if 'style' in self.props: |
| 79 | args.append('with %s' % self.props['style']) |
| 80 | if 'lc' in self.props: |
| 81 | args.append('lc rgb "%s"' % self.props['lc']) |
| 82 | if 'fs' in self.props: |
| 83 | args.append('fs %s' % self.props['fs']) |
| 84 | return ' '.join(args) |
| 85 | |
| 86 | class Plot(object): |
| 87 | def __init__(self, *items): |
| 88 | self.items = items |
| 89 | |
| 90 | def fieldrefs(self): |
| 91 | return flatten([item.fieldrefs() for item in self.items]) |
| 92 | |
| 93 | def to_gnuplot(self, ctx): |
| 94 | return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items]) |
| 95 | |
| 96 | class Set(object): |
| 97 | def __init__(self, value): |
| 98 | self.value = value |
| 99 | |
| 100 | def to_gnuplot(self, ctx): |
| 101 | return 'set ' + self.value |
| 102 | |
| 103 | def fieldrefs(self): |
| 104 | return [] |
| 105 | |
| 106 | class Context(object): |
| 107 | def __init__(self, datafile, field_to_index): |
| 108 | self.datafile = datafile |
| 109 | self.field_to_index = field_to_index |
| 110 | |
| 111 | def format_fieldref(self, fieldref): |
| 112 | return ':'.join([str(self.field_to_index[field]) for field in fieldref]) |
| 113 | |
| 114 | def collect_fields(plot): |
| 115 | field_to_index = {} |
| 116 | fields = [] |
| 117 | |
| 118 | def add_field(field): |
| 119 | if field not in field_to_index: |
| 120 | fields.append(field) |
| 121 | field_to_index[field] = len(fields) |
| 122 | |
| 123 | for field in flatten([item.fieldrefs() for item in plot]): |
| 124 | add_field(field) |
| 125 | |
| 126 | return (fields, field_to_index) |
| 127 | |
| 128 | def is_y2_used(plot): |
| 129 | for subplot in plot: |
| 130 | if isinstance(subplot, Plot): |
| 131 | for item in subplot.items: |
| 132 | if item.axis == x1y2 or item.axis == x2y2: |
| 133 | return True |
| 134 | return False |
| 135 | |
| 136 | def get_field(trace_line, field): |
| 137 | t = type(field) |
| 138 | if t is types.StringType: |
| 139 | return trace_line[field] |
| 140 | elif t is types.FunctionType: |
| 141 | return field(trace_line) |
| 142 | |
| 143 | def generate_datafile(datafile_name, trace, fields): |
| 144 | with open(datafile_name, 'w') as datafile: |
| 145 | for line in trace: |
| 146 | data_line = [str(get_field(line, field)) for field in fields] |
| 147 | datafile.write('\t'.join(data_line)) |
| 148 | datafile.write('\n') |
| 149 | |
| 150 | def generate_script_and_datafile(plot, trace, datafile, output): |
| 151 | (fields, field_to_index) = collect_fields(plot) |
| 152 | generate_datafile(datafile, trace, fields) |
| 153 | script = [ |
| 154 | 'set terminal png', |
| 155 | 'set output "%s"' % output, |
| 156 | 'set autoscale', |
| 157 | 'set ytics nomirror', |
| 158 | 'set xtics nomirror', |
| 159 | 'set key below' |
| 160 | ] |
| 161 | |
| 162 | if is_y2_used(plot): |
| 163 | script.append('set autoscale y2') |
| 164 | script.append('set y2tics') |
| 165 | |
| 166 | context = Context(datafile, field_to_index) |
| 167 | |
| 168 | for item in plot: |
| 169 | script.append(item.to_gnuplot(context)) |
| 170 | |
| 171 | return '\n'.join(script) |
| 172 | |
| 173 | def plot_all(plots, trace, prefix): |
| 174 | charts = [] |
| 175 | |
| 176 | for plot in plots: |
| 177 | outfilename = "%s_%d.png" % (prefix, len(charts)) |
| 178 | charts.append(outfilename) |
| 179 | script = generate_script_and_datafile(plot, trace, '~datafile', outfilename) |
| 180 | print 'Plotting %s...' % outfilename |
| 181 | gnuplot(script) |
| 182 | |
| 183 | return charts |
| 184 | |
| 185 | def reclaimed_bytes(row): |
| 186 | return row['total_size_before'] - row['total_size_after'] |
| 187 | |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 188 | def other_scope(r): |
Ben Murdoch | 257744e | 2011-11-30 15:57:28 +0000 | [diff] [blame] | 189 | if r['gc'] == 's': |
| 190 | # there is no 'other' scope for scavenging collections. |
| 191 | return 0 |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 192 | return r['pause'] - r['mark'] - r['sweep'] - r['external'] |
Ben Murdoch | 257744e | 2011-11-30 15:57:28 +0000 | [diff] [blame] | 193 | |
| 194 | def scavenge_scope(r): |
| 195 | if r['gc'] == 's': |
| 196 | return r['pause'] - r['external'] |
| 197 | return 0 |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 198 | |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 199 | |
| 200 | def real_mutator(r): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 201 | return r['mutator'] - r['steps_took'] |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 202 | |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 203 | plots = [ |
| 204 | [ |
| 205 | Set('style fill solid 0.5 noborder'), |
| 206 | Set('style histogram rowstacked'), |
| 207 | Set('style data histograms'), |
Ben Murdoch | 257744e | 2011-11-30 15:57:28 +0000 | [diff] [blame] | 208 | Plot(Item('Scavenge', scavenge_scope, lc = 'green'), |
| 209 | Item('Marking', 'mark', lc = 'purple'), |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 210 | Item('Sweep', 'sweep', lc = 'blue'), |
Ben Murdoch | 257744e | 2011-11-30 15:57:28 +0000 | [diff] [blame] | 211 | Item('External', 'external', lc = '#489D43'), |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 212 | Item('Other', other_scope, lc = 'grey'), |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 213 | Item('IGC Steps', 'steps_took', lc = '#FF6347')) |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 214 | ], |
| 215 | [ |
| 216 | Set('style fill solid 0.5 noborder'), |
| 217 | Set('style histogram rowstacked'), |
| 218 | Set('style data histograms'), |
| 219 | Plot(Item('Scavenge', scavenge_scope, lc = 'green'), |
| 220 | Item('Marking', 'mark', lc = 'purple'), |
| 221 | Item('Sweep', 'sweep', lc = 'blue'), |
| 222 | Item('External', 'external', lc = '#489D43'), |
| 223 | Item('Other', other_scope, lc = '#ADD8E6'), |
| 224 | Item('External', 'external', lc = '#D3D3D3')) |
| 225 | ], |
| 226 | |
| 227 | [ |
| 228 | Plot(Item('Mutator', real_mutator, lc = 'black', style = 'lines')) |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 229 | ], |
| 230 | [ |
| 231 | Set('style histogram rowstacked'), |
| 232 | Set('style data histograms'), |
| 233 | Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2, |
| 234 | fs = 'solid 0.4 noborder', |
| 235 | lc = 'green'), |
| 236 | Item('Total holes (after GC)', 'holes_size_before', x1y2, |
| 237 | fs = 'solid 0.4 noborder', |
| 238 | lc = 'red'), |
| 239 | Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red')) |
| 240 | ], |
| 241 | [ |
| 242 | Set('style histogram rowstacked'), |
| 243 | Set('style data histograms'), |
| 244 | Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2, |
| 245 | fs = 'solid 0.4 noborder', |
| 246 | lc = 'green'), |
| 247 | Item('Total holes (after GC)', 'holes_size_after', x1y2, |
| 248 | fs = 'solid 0.4 noborder', |
| 249 | lc = 'red'), |
| 250 | Item('GC Time', ['i', 'pause'], |
| 251 | style = 'lines', |
| 252 | lc = 'red')) |
| 253 | ], |
| 254 | [ |
| 255 | Set('style fill solid 0.5 noborder'), |
| 256 | Set('style data histograms'), |
| 257 | Plot(Item('Allocated', 'allocated'), |
| 258 | Item('Reclaimed', reclaimed_bytes), |
| 259 | Item('Promoted', 'promoted', style = 'lines', lc = 'black')) |
| 260 | ], |
| 261 | ] |
| 262 | |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 263 | def freduce(f, field, trace, init): |
| 264 | return reduce(lambda t,r: f(t, r[field]), trace, init) |
| 265 | |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 266 | def calc_total(trace, field): |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 267 | return freduce(lambda t,v: t + long(v), field, trace, long(0)) |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 268 | |
| 269 | def calc_max(trace, field): |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 270 | return freduce(lambda t,r: max(t, r), field, trace, 0) |
| 271 | |
| 272 | def count_nonzero(trace, field): |
| 273 | return freduce(lambda t,r: t if r == 0 else t + 1, field, trace, 0) |
| 274 | |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 275 | |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 276 | def process_trace(filename): |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 277 | trace = gc_nvp_common.parse_gc_trace(filename) |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 278 | |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 279 | marksweeps = filter(lambda r: r['gc'] == 'ms', trace) |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 280 | scavenges = filter(lambda r: r['gc'] == 's', trace) |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 281 | globalgcs = filter(lambda r: r['gc'] != 's', trace) |
| 282 | |
Kristian Monsen | 50ef84f | 2010-07-29 15:18:00 +0100 | [diff] [blame] | 283 | |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 284 | charts = plot_all(plots, trace, filename) |
| 285 | |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 286 | def stats(out, prefix, trace, field): |
| 287 | n = len(trace) |
| 288 | total = calc_total(trace, field) |
| 289 | max = calc_max(trace, field) |
| 290 | if n > 0: |
| 291 | avg = total / n |
| 292 | else: |
| 293 | avg = 0 |
| 294 | if n > 1: |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 295 | dev = math.sqrt(freduce(lambda t,r: t + (r - avg) ** 2, field, trace, 0) / |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 296 | (n - 1)) |
| 297 | else: |
| 298 | dev = 0 |
| 299 | |
| 300 | out.write('<tr><td>%s</td><td>%d</td><td>%d</td>' |
| 301 | '<td>%d</td><td>%d [dev %f]</td></tr>' % |
| 302 | (prefix, n, total, max, avg, dev)) |
| 303 | |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 304 | def HumanReadable(size): |
| 305 | suffixes = ['bytes', 'kB', 'MB', 'GB'] |
| 306 | power = 1 |
| 307 | for i in range(len(suffixes)): |
| 308 | if size < power*1024: |
| 309 | return "%.1f" % (float(size) / power) + " " + suffixes[i] |
| 310 | power *= 1024 |
| 311 | |
| 312 | def throughput(name, trace): |
| 313 | total_live_after = calc_total(trace, 'total_size_after') |
| 314 | total_live_before = calc_total(trace, 'total_size_before') |
| 315 | total_gc = calc_total(trace, 'pause') |
| 316 | if total_gc == 0: |
| 317 | return |
| 318 | out.write('GC %s Throughput (after): %s / %s ms = %s/ms<br/>' % |
| 319 | (name, |
| 320 | HumanReadable(total_live_after), |
| 321 | total_gc, |
| 322 | HumanReadable(total_live_after / total_gc))) |
| 323 | out.write('GC %s Throughput (before): %s / %s ms = %s/ms<br/>' % |
| 324 | (name, |
| 325 | HumanReadable(total_live_before), |
| 326 | total_gc, |
| 327 | HumanReadable(total_live_before / total_gc))) |
| 328 | |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 329 | |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 330 | with open(filename + '.html', 'w') as out: |
| 331 | out.write('<html><body>') |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 332 | out.write('<table>') |
| 333 | out.write('<tr><td>Phase</td><td>Count</td><td>Time (ms)</td>') |
| 334 | out.write('<td>Max</td><td>Avg</td></tr>') |
| 335 | stats(out, 'Total in GC', trace, 'pause') |
| 336 | stats(out, 'Scavenge', scavenges, 'pause') |
| 337 | stats(out, 'MarkSweep', marksweeps, 'pause') |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 338 | stats(out, 'Mark', filter(lambda r: r['mark'] != 0, trace), 'mark') |
| 339 | stats(out, 'Sweep', filter(lambda r: r['sweep'] != 0, trace), 'sweep') |
Ben Murdoch | 257744e | 2011-11-30 15:57:28 +0000 | [diff] [blame] | 340 | stats(out, |
| 341 | 'External', |
| 342 | filter(lambda r: r['external'] != 0, trace), |
| 343 | 'external') |
Iain Merrick | 7568138 | 2010-08-19 15:07:18 +0100 | [diff] [blame] | 344 | out.write('</table>') |
Ben Murdoch | 3ef787d | 2012-04-12 10:51:47 +0100 | [diff] [blame] | 345 | throughput('TOTAL', trace) |
| 346 | throughput('MS', marksweeps) |
| 347 | throughput('OLDSPACE', globalgcs) |
| 348 | out.write('<br/>') |
Leon Clarke | f7060e2 | 2010-06-03 12:02:55 +0100 | [diff] [blame] | 349 | for chart in charts: |
| 350 | out.write('<img src="%s">' % chart) |
| 351 | out.write('</body></html>') |
| 352 | |
| 353 | print "%s generated." % (filename + '.html') |
| 354 | |
| 355 | if len(sys.argv) != 2: |
| 356 | print "Usage: %s <GC-trace-filename>" % sys.argv[0] |
| 357 | sys.exit(1) |
| 358 | |
| 359 | process_trace(sys.argv[1]) |