Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / a01bccdbe613f5e65fa8da2fb07d6b9b66d203bf / . / clang / tools / scan-build-py / libscanbuild / report.py
blob: 766ddef7199093dcba906cf85eb2838733bf7193 [file] [log] [blame]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]1# -*- coding: utf-8 -*-
2# The LLVM Compiler Infrastructure
3#
4# This file is distributed under the University of Illinois Open Source
5# License. See LICENSE.TXT for details.
6""" This module is responsible to generate 'index.html' for the report.
7
8The input for this step is the output directory, where individual reports
9could be found. It parses those reports and generates 'index.html'. """
10
11import re
12import os
13import os.path
14import sys
15import shutil
16import time
17import tempfile
18import itertools
19import plistlib
20import glob
21import json
22import logging
23import contextlib
Devin Coughlin80e74cd2016-09-06 23:42:51 +0000[diff] [blame]24import datetime
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]25from libscanbuild import duplicate_check
26from libscanbuild.clang import get_version
27
28__all__ = ['report_directory', 'document']
29
30
31@contextlib.contextmanager
32def report_directory(hint, keep):
33 """ Responsible for the report directory.
34
35 hint -- could specify the parent directory of the output directory.
36 keep -- a boolean value to keep or delete the empty report directory. """
37
Devin Coughlin80e74cd2016-09-06 23:42:51 +0000[diff] [blame]38 stamp_format = 'scan-build-%Y-%m-%d-%H-%M-%S-%f-'
39 stamp = datetime.datetime.now().strftime(stamp_format)
Anton Yartsev6e135f42016-02-21 17:04:26 +0000[diff] [blame]40
41 parentdir = os.path.abspath(hint)
42 if not os.path.exists(parentdir):
43 os.makedirs(parentdir)
44
45 name = tempfile.mkdtemp(prefix=stamp, dir=parentdir)
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]46
47 logging.info('Report directory created: %s', name)
48
49 try:
50 yield name
51 finally:
52 if os.listdir(name):
53 msg = "Run 'scan-view %s' to examine bug reports."
54 keep = True
55 else:
56 if keep:
57 msg = "Report directory '%s' contans no report, but kept."
58 else:
59 msg = "Removing directory '%s' because it contains no report."
60 logging.warning(msg, name)
61
62 if not keep:
63 os.rmdir(name)
64
65
66def document(args, output_dir, use_cdb):
67 """ Generates cover report and returns the number of bugs/crashes. """
68
69 html_reports_available = args.output_format in {'html', 'plist-html'}
70
71 logging.debug('count crashes and bugs')
72 crash_count = sum(1 for _ in read_crashes(output_dir))
73 bug_counter = create_counters()
74 for bug in read_bugs(output_dir, html_reports_available):
75 bug_counter(bug)
76 result = crash_count + bug_counter.total
77
78 if html_reports_available and result:
79 logging.debug('generate index.html file')
80 # common prefix for source files to have sort filenames
81 prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
82 # assemble the cover from multiple fragments
83 try:
84 fragments = []
85 if bug_counter.total:
86 fragments.append(bug_summary(output_dir, bug_counter))
87 fragments.append(bug_report(output_dir, prefix))
88 if crash_count:
89 fragments.append(crash_report(output_dir, prefix))
90 assemble_cover(output_dir, prefix, args, fragments)
91 # copy additinal files to the report
92 copy_resource_files(output_dir)
93 if use_cdb:
94 shutil.copy(args.cdb, output_dir)
95 finally:
96 for fragment in fragments:
97 os.remove(fragment)
98 return result
99
100
101def assemble_cover(output_dir, prefix, args, fragments):
102 """ Put together the fragments into a final report. """
103
104 import getpass
105 import socket
106 import datetime
107
108 if args.html_title is None:
109 args.html_title = os.path.basename(prefix) + ' - analyzer results'
110
111 with open(os.path.join(output_dir, 'index.html'), 'w') as handle:
112 indent = 0
113 handle.write(reindent("""
114 |<!DOCTYPE html>
115 |<html>
116 | <head>
117 | <title>{html_title}</title>
118 | <link type="text/css" rel="stylesheet" href="scanview.css"/>
119 | <script type='text/javascript' src="sorttable.js"></script>
120 | <script type='text/javascript' src='selectable.js'></script>
121 | </head>""", indent).format(html_title=args.html_title))
122 handle.write(comment('SUMMARYENDHEAD'))
123 handle.write(reindent("""
124 | <body>
125 | <h1>{html_title}</h1>
126 | <table>
127 | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr>
128 | <tr><th>Working Directory:</th><td>{current_dir}</td></tr>
129 | <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
130 | <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
131 | <tr><th>Date:</th><td>{date}</td></tr>
132 | </table>""", indent).format(html_title=args.html_title,
133 user_name=getpass.getuser(),
134 host_name=socket.gethostname(),
135 current_dir=prefix,
136 cmd_args=' '.join(sys.argv),
137 clang_version=get_version(args.clang),
138 date=datetime.datetime.today(
139 ).strftime('%c')))
140 for fragment in fragments:
141 # copy the content of fragments
142 with open(fragment, 'r') as input_handle:
143 shutil.copyfileobj(input_handle, handle)
144 handle.write(reindent("""
145 | </body>
146 |</html>""", indent))
147
148
149def bug_summary(output_dir, bug_counter):
150 """ Bug summary is a HTML table to give a better overview of the bugs. """
151
152 name = os.path.join(output_dir, 'summary.html.fragment')
153 with open(name, 'w') as handle:
154 indent = 4
155 handle.write(reindent("""
156 |<h2>Bug Summary</h2>
157 |<table>
158 | <thead>
159 | <tr>
160 | <td>Bug Type</td>
161 | <td>Quantity</td>
162 | <td class="sorttable_nosort">Display?</td>
163 | </tr>
164 | </thead>
165 | <tbody>""", indent))
166 handle.write(reindent("""
167 | <tr style="font-weight:bold">
168 | <td class="SUMM_DESC">All Bugs</td>
169 | <td class="Q">{0}</td>
170 | <td>
171 | <center>
172 | <input checked type="checkbox" id="AllBugsCheck"
173 | onClick="CopyCheckedStateToCheckButtons(this);"/>
174 | </center>
175 | </td>
176 | </tr>""", indent).format(bug_counter.total))
177 for category, types in bug_counter.categories.items():
178 handle.write(reindent("""
179 | <tr>
180 | <th>{0}</th><th colspan=2></th>
181 | </tr>""", indent).format(category))
182 for bug_type in types.values():
183 handle.write(reindent("""
184 | <tr>
185 | <td class="SUMM_DESC">{bug_type}</td>
186 | <td class="Q">{bug_count}</td>
187 | <td>
188 | <center>
189 | <input checked type="checkbox"
190 | onClick="ToggleDisplay(this,'{bug_type_class}');"/>
191 | </center>
192 | </td>
193 | </tr>""", indent).format(**bug_type))
194 handle.write(reindent("""
195 | </tbody>
196 |</table>""", indent))
197 handle.write(comment('SUMMARYBUGEND'))
198 return name
199
200
201def bug_report(output_dir, prefix):
202 """ Creates a fragment from the analyzer reports. """
203
204 pretty = prettify_bug(prefix, output_dir)
205 bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
206
207 name = os.path.join(output_dir, 'bugs.html.fragment')
208 with open(name, 'w') as handle:
209 indent = 4
210 handle.write(reindent("""
211 |<h2>Reports</h2>
212 |<table class="sortable" style="table-layout:automatic">
213 | <thead>
214 | <tr>
215 | <td>Bug Group</td>
216 | <td class="sorttable_sorted">
217 | Bug Type
218 | <span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span>
219 | </td>
220 | <td>File</td>
221 | <td>Function/Method</td>
222 | <td class="Q">Line</td>
223 | <td class="Q">Path Length</td>
224 | <td class="sorttable_nosort"></td>
225 | </tr>
226 | </thead>
227 | <tbody>""", indent))
228 handle.write(comment('REPORTBUGCOL'))
229 for current in bugs:
230 handle.write(reindent("""
231 | <tr class="{bug_type_class}">
232 | <td class="DESC">{bug_category}</td>
233 | <td class="DESC">{bug_type}</td>
234 | <td>{bug_file}</td>
235 | <td class="DESC">{bug_function}</td>
236 | <td class="Q">{bug_line}</td>
237 | <td class="Q">{bug_path_length}</td>
238 | <td><a href="{report_file}#EndPath">View Report</a></td>
239 | </tr>""", indent).format(**current))
240 handle.write(comment('REPORTBUG', {'id': current['report_file']}))
241 handle.write(reindent("""
242 | </tbody>
243 |</table>""", indent))
244 handle.write(comment('REPORTBUGEND'))
245 return name
246
247
248def crash_report(output_dir, prefix):
249 """ Creates a fragment from the compiler crashes. """
250
251 pretty = prettify_crash(prefix, output_dir)
252 crashes = (pretty(crash) for crash in read_crashes(output_dir))
253
254 name = os.path.join(output_dir, 'crashes.html.fragment')
255 with open(name, 'w') as handle:
256 indent = 4
257 handle.write(reindent("""
258 |<h2>Analyzer Failures</h2>
259 |<p>The analyzer had problems processing the following files:</p>
260 |<table>
261 | <thead>
262 | <tr>
263 | <td>Problem</td>
264 | <td>Source File</td>
265 | <td>Preprocessed File</td>
266 | <td>STDERR Output</td>
267 | </tr>
268 | </thead>
269 | <tbody>""", indent))
270 for current in crashes:
271 handle.write(reindent("""
272 | <tr>
273 | <td>{problem}</td>
274 | <td>{source}</td>
275 | <td><a href="{file}">preprocessor output</a></td>
276 | <td><a href="{stderr}">analyzer std err</a></td>
277 | </tr>""", indent).format(**current))
278 handle.write(comment('REPORTPROBLEM', current))
279 handle.write(reindent("""
280 | </tbody>
281 |</table>""", indent))
282 handle.write(comment('REPORTCRASHES'))
283 return name
284
285
286def read_crashes(output_dir):
287 """ Generate a unique sequence of crashes from given output directory. """
288
289 return (parse_crash(filename)
290 for filename in glob.iglob(os.path.join(output_dir, 'failures',
291 '*.info.txt')))
292
293
294def read_bugs(output_dir, html):
295 """ Generate a unique sequence of bugs from given output directory.
296
297 Duplicates can be in a project if the same module was compiled multiple
298 times with different compiler options. These would be better to show in
299 the final report (cover) only once. """
300
301 parser = parse_bug_html if html else parse_bug_plist
302 pattern = '*.html' if html else '*.plist'
303
304 duplicate = duplicate_check(
305 lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
306
307 bugs = itertools.chain.from_iterable(
308 # parser creates a bug generator not the bug itself
309 parser(filename)
310 for filename in glob.iglob(os.path.join(output_dir, pattern)))
311
312 return (bug for bug in bugs if not duplicate(bug))
313
314
315def parse_bug_plist(filename):
316 """ Returns the generator of bugs from a single .plist file. """
317
318 content = plistlib.readPlist(filename)
319 files = content.get('files')
320 for bug in content.get('diagnostics', []):
321 if len(files) <= int(bug['location']['file']):
322 logging.warning('Parsing bug from "%s" failed', filename)
323 continue
324
325 yield {
326 'result': filename,
327 'bug_type': bug['type'],
328 'bug_category': bug['category'],
329 'bug_line': int(bug['location']['line']),
330 'bug_path_length': int(bug['location']['col']),
331 'bug_file': files[int(bug['location']['file'])]
332 }
333
334
335def parse_bug_html(filename):
336 """ Parse out the bug information from HTML output. """
337
338 patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
339 re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
340 re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
341 re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
342 re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
343 re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
344 re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
345 endsign = re.compile(r'<!-- BUGMETAEND -->')
346
347 bug = {
348 'report_file': filename,
349 'bug_function': 'n/a', # compatibility with < clang-3.5
350 'bug_category': 'Other',
351 'bug_line': 0,
352 'bug_path_length': 1
353 }
354
355 with open(filename) as handler:
356 for line in handler.readlines():
357 # do not read the file further
358 if endsign.match(line):
359 break
360 # search for the right lines
361 for regex in patterns:
362 match = regex.match(line.strip())
363 if match:
364 bug.update(match.groupdict())
365 break
366
367 encode_value(bug, 'bug_line', int)
368 encode_value(bug, 'bug_path_length', int)
369
370 yield bug
371
372
373def parse_crash(filename):
374 """ Parse out the crash information from the report file. """
375
376 match = re.match(r'(.*)\.info\.txt', filename)
377 name = match.group(1) if match else None
378 with open(filename) as handler:
379 lines = handler.readlines()
380 return {
381 'source': lines[0].rstrip(),
382 'problem': lines[1].rstrip(),
383 'file': name,
384 'info': name + '.info.txt',
385 'stderr': name + '.stderr.txt'
386 }
387
388
389def category_type_name(bug):
390 """ Create a new bug attribute from bug by category and type.
391
392 The result will be used as CSS class selector in the final report. """
393
394 def smash(key):
395 """ Make value ready to be HTML attribute value. """
396
397 return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
398
399 return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
400
401
402def create_counters():
403 """ Create counters for bug statistics.
404
405 Two entries are maintained: 'total' is an integer, represents the
406 number of bugs. The 'categories' is a two level categorisation of bug
407 counters. The first level is 'bug category' the second is 'bug type'.
408 Each entry in this classification is a dictionary of 'count', 'type'
409 and 'label'. """
410
411 def predicate(bug):
412 bug_category = bug['bug_category']
413 bug_type = bug['bug_type']
414 current_category = predicate.categories.get(bug_category, dict())
415 current_type = current_category.get(bug_type, {
416 'bug_type': bug_type,
417 'bug_type_class': category_type_name(bug),
418 'bug_count': 0
419 })
420 current_type.update({'bug_count': current_type['bug_count'] + 1})
421 current_category.update({bug_type: current_type})
422 predicate.categories.update({bug_category: current_category})
423 predicate.total += 1
424
425 predicate.total = 0
426 predicate.categories = dict()
427 return predicate
428
429
430def prettify_bug(prefix, output_dir):
431 def predicate(bug):
432 """ Make safe this values to embed into HTML. """
433
434 bug['bug_type_class'] = category_type_name(bug)
435
436 encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
437 encode_value(bug, 'bug_category', escape)
438 encode_value(bug, 'bug_type', escape)
439 encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
440 return bug
441
442 return predicate
443
444
445def prettify_crash(prefix, output_dir):
446 def predicate(crash):
447 """ Make safe this values to embed into HTML. """
448
449 encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
450 encode_value(crash, 'problem', escape)
451 encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
452 encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
453 encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
454 return crash
455
456 return predicate
457
458
459def copy_resource_files(output_dir):
460 """ Copy the javascript and css files to the report directory. """
461
462 this_dir = os.path.dirname(os.path.realpath(__file__))
463 for resource in os.listdir(os.path.join(this_dir, 'resources')):
464 shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
465
466
467def encode_value(container, key, encode):
468 """ Run 'encode' on 'container[key]' value and update it. """
469
470 if key in container:
471 value = encode(container[key])
472 container.update({key: value})
473
474
475def chop(prefix, filename):
476 """ Create 'filename' from '/prefix/filename' """
477
478 return filename if not len(prefix) else os.path.relpath(filename, prefix)
479
480
481def escape(text):
482 """ Paranoid HTML escape method. (Python version independent) """
483
484 escape_table = {
485 '&': '&amp;',
486 '"': '&quot;',
487 "'": '&apos;',
488 '>': '&gt;',
489 '<': '&lt;'
490 }
491 return ''.join(escape_table.get(c, c) for c in text)
492
493
494def reindent(text, indent):
495 """ Utility function to format html output and keep indentation. """
496
497 result = ''
498 for line in text.splitlines():
499 if len(line.strip()):
500 result += ' ' * indent + line.split('|')[1] + os.linesep
501 return result
502
503
504def comment(name, opts=dict()):
505 """ Utility function to format meta information as comment. """
506
507 attributes = ''
508 for key, value in opts.items():
509 attributes += ' {0}="{1}"'.format(key, value)
510
511 return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
512
513
514def commonprefix_from(filename):
515 """ Create file prefix from a compilation database entries. """
516
517 with open(filename, 'r') as handle:
518 return commonprefix(item['file'] for item in json.load(handle))
519
520
521def commonprefix(files):
522 """ Fixed version of os.path.commonprefix. Return the longest path prefix
523 that is a prefix of all paths in filenames. """
524
525 result = None
526 for current in files:
527 if result is not None:
528 result = os.path.commonprefix([result, current])
529 else:
530 result = current
531
532 if result is None:
533 return ''
534 elif not os.path.isdir(result):
535 return os.path.dirname(result)
536 else:
537 return os.path.abspath(result)