Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 258ff25bbce734e8826a7d2275392a46bc7e3bcd / . / clang / tools / scan-build-py / libscanbuild / report.py
blob: 83b581b79fc17c237dfd438debc98ac0a76278f8 [file] [log] [blame]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]1# -*- coding: utf-8 -*-
2# The LLVM Compiler Infrastructure
3#
4# This file is distributed under the University of Illinois Open Source
5# License. See LICENSE.TXT for details.
6""" This module is responsible to generate 'index.html' for the report.
7
8The input for this step is the output directory, where individual reports
9could be found. It parses those reports and generates 'index.html'. """
10
11import re
12import os
13import os.path
14import sys
15import shutil
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]16import itertools
17import plistlib
18import glob
19import json
20import logging
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]21from libscanbuild import duplicate_check
22from libscanbuild.clang import get_version
23
Laszlo Nagy258ff252017-02-14 10:43:38 +0000[diff] [blame^]24__all__ = ['document']
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]25
26
27def document(args, output_dir, use_cdb):
28 """ Generates cover report and returns the number of bugs/crashes. """
29
30 html_reports_available = args.output_format in {'html', 'plist-html'}
31
32 logging.debug('count crashes and bugs')
33 crash_count = sum(1 for _ in read_crashes(output_dir))
34 bug_counter = create_counters()
35 for bug in read_bugs(output_dir, html_reports_available):
36 bug_counter(bug)
37 result = crash_count + bug_counter.total
38
39 if html_reports_available and result:
40 logging.debug('generate index.html file')
41 # common prefix for source files to have sort filenames
42 prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
43 # assemble the cover from multiple fragments
44 try:
45 fragments = []
46 if bug_counter.total:
47 fragments.append(bug_summary(output_dir, bug_counter))
48 fragments.append(bug_report(output_dir, prefix))
49 if crash_count:
50 fragments.append(crash_report(output_dir, prefix))
51 assemble_cover(output_dir, prefix, args, fragments)
52 # copy additinal files to the report
53 copy_resource_files(output_dir)
54 if use_cdb:
55 shutil.copy(args.cdb, output_dir)
56 finally:
57 for fragment in fragments:
58 os.remove(fragment)
59 return result
60
61
62def assemble_cover(output_dir, prefix, args, fragments):
63 """ Put together the fragments into a final report. """
64
65 import getpass
66 import socket
67 import datetime
68
69 if args.html_title is None:
70 args.html_title = os.path.basename(prefix) + ' - analyzer results'
71
72 with open(os.path.join(output_dir, 'index.html'), 'w') as handle:
73 indent = 0
74 handle.write(reindent("""
75 |<!DOCTYPE html>
76 |<html>
77 | <head>
78 | <title>{html_title}</title>
79 | <link type="text/css" rel="stylesheet" href="scanview.css"/>
80 | <script type='text/javascript' src="sorttable.js"></script>
81 | <script type='text/javascript' src='selectable.js'></script>
82 | </head>""", indent).format(html_title=args.html_title))
83 handle.write(comment('SUMMARYENDHEAD'))
84 handle.write(reindent("""
85 | <body>
86 | <h1>{html_title}</h1>
87 | <table>
88 | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr>
89 | <tr><th>Working Directory:</th><td>{current_dir}</td></tr>
90 | <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
91 | <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
92 | <tr><th>Date:</th><td>{date}</td></tr>
93 | </table>""", indent).format(html_title=args.html_title,
94 user_name=getpass.getuser(),
95 host_name=socket.gethostname(),
96 current_dir=prefix,
97 cmd_args=' '.join(sys.argv),
98 clang_version=get_version(args.clang),
99 date=datetime.datetime.today(
100 ).strftime('%c')))
101 for fragment in fragments:
102 # copy the content of fragments
103 with open(fragment, 'r') as input_handle:
104 shutil.copyfileobj(input_handle, handle)
105 handle.write(reindent("""
106 | </body>
107 |</html>""", indent))
108
109
110def bug_summary(output_dir, bug_counter):
111 """ Bug summary is a HTML table to give a better overview of the bugs. """
112
113 name = os.path.join(output_dir, 'summary.html.fragment')
114 with open(name, 'w') as handle:
115 indent = 4
116 handle.write(reindent("""
117 |<h2>Bug Summary</h2>
118 |<table>
119 | <thead>
120 | <tr>
121 | <td>Bug Type</td>
122 | <td>Quantity</td>
123 | <td class="sorttable_nosort">Display?</td>
124 | </tr>
125 | </thead>
126 | <tbody>""", indent))
127 handle.write(reindent("""
128 | <tr style="font-weight:bold">
129 | <td class="SUMM_DESC">All Bugs</td>
130 | <td class="Q">{0}</td>
131 | <td>
132 | <center>
133 | <input checked type="checkbox" id="AllBugsCheck"
134 | onClick="CopyCheckedStateToCheckButtons(this);"/>
135 | </center>
136 | </td>
137 | </tr>""", indent).format(bug_counter.total))
138 for category, types in bug_counter.categories.items():
139 handle.write(reindent("""
140 | <tr>
141 | <th>{0}</th><th colspan=2></th>
142 | </tr>""", indent).format(category))
143 for bug_type in types.values():
144 handle.write(reindent("""
145 | <tr>
146 | <td class="SUMM_DESC">{bug_type}</td>
147 | <td class="Q">{bug_count}</td>
148 | <td>
149 | <center>
150 | <input checked type="checkbox"
151 | onClick="ToggleDisplay(this,'{bug_type_class}');"/>
152 | </center>
153 | </td>
154 | </tr>""", indent).format(**bug_type))
155 handle.write(reindent("""
156 | </tbody>
157 |</table>""", indent))
158 handle.write(comment('SUMMARYBUGEND'))
159 return name
160
161
162def bug_report(output_dir, prefix):
163 """ Creates a fragment from the analyzer reports. """
164
165 pretty = prettify_bug(prefix, output_dir)
166 bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
167
168 name = os.path.join(output_dir, 'bugs.html.fragment')
169 with open(name, 'w') as handle:
170 indent = 4
171 handle.write(reindent("""
172 |<h2>Reports</h2>
173 |<table class="sortable" style="table-layout:automatic">
174 | <thead>
175 | <tr>
176 | <td>Bug Group</td>
177 | <td class="sorttable_sorted">
178 | Bug Type
179 | <span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span>
180 | </td>
181 | <td>File</td>
182 | <td>Function/Method</td>
183 | <td class="Q">Line</td>
184 | <td class="Q">Path Length</td>
185 | <td class="sorttable_nosort"></td>
186 | </tr>
187 | </thead>
188 | <tbody>""", indent))
189 handle.write(comment('REPORTBUGCOL'))
190 for current in bugs:
191 handle.write(reindent("""
192 | <tr class="{bug_type_class}">
193 | <td class="DESC">{bug_category}</td>
194 | <td class="DESC">{bug_type}</td>
195 | <td>{bug_file}</td>
196 | <td class="DESC">{bug_function}</td>
197 | <td class="Q">{bug_line}</td>
198 | <td class="Q">{bug_path_length}</td>
199 | <td><a href="{report_file}#EndPath">View Report</a></td>
200 | </tr>""", indent).format(**current))
201 handle.write(comment('REPORTBUG', {'id': current['report_file']}))
202 handle.write(reindent("""
203 | </tbody>
204 |</table>""", indent))
205 handle.write(comment('REPORTBUGEND'))
206 return name
207
208
209def crash_report(output_dir, prefix):
210 """ Creates a fragment from the compiler crashes. """
211
212 pretty = prettify_crash(prefix, output_dir)
213 crashes = (pretty(crash) for crash in read_crashes(output_dir))
214
215 name = os.path.join(output_dir, 'crashes.html.fragment')
216 with open(name, 'w') as handle:
217 indent = 4
218 handle.write(reindent("""
219 |<h2>Analyzer Failures</h2>
220 |<p>The analyzer had problems processing the following files:</p>
221 |<table>
222 | <thead>
223 | <tr>
224 | <td>Problem</td>
225 | <td>Source File</td>
226 | <td>Preprocessed File</td>
227 | <td>STDERR Output</td>
228 | </tr>
229 | </thead>
230 | <tbody>""", indent))
231 for current in crashes:
232 handle.write(reindent("""
233 | <tr>
234 | <td>{problem}</td>
235 | <td>{source}</td>
236 | <td><a href="{file}">preprocessor output</a></td>
237 | <td><a href="{stderr}">analyzer std err</a></td>
238 | </tr>""", indent).format(**current))
239 handle.write(comment('REPORTPROBLEM', current))
240 handle.write(reindent("""
241 | </tbody>
242 |</table>""", indent))
243 handle.write(comment('REPORTCRASHES'))
244 return name
245
246
247def read_crashes(output_dir):
248 """ Generate a unique sequence of crashes from given output directory. """
249
250 return (parse_crash(filename)
251 for filename in glob.iglob(os.path.join(output_dir, 'failures',
252 '*.info.txt')))
253
254
255def read_bugs(output_dir, html):
256 """ Generate a unique sequence of bugs from given output directory.
257
258 Duplicates can be in a project if the same module was compiled multiple
259 times with different compiler options. These would be better to show in
260 the final report (cover) only once. """
261
262 parser = parse_bug_html if html else parse_bug_plist
263 pattern = '*.html' if html else '*.plist'
264
265 duplicate = duplicate_check(
266 lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
267
268 bugs = itertools.chain.from_iterable(
269 # parser creates a bug generator not the bug itself
270 parser(filename)
271 for filename in glob.iglob(os.path.join(output_dir, pattern)))
272
273 return (bug for bug in bugs if not duplicate(bug))
274
275
276def parse_bug_plist(filename):
277 """ Returns the generator of bugs from a single .plist file. """
278
279 content = plistlib.readPlist(filename)
280 files = content.get('files')
281 for bug in content.get('diagnostics', []):
282 if len(files) <= int(bug['location']['file']):
283 logging.warning('Parsing bug from "%s" failed', filename)
284 continue
285
286 yield {
287 'result': filename,
288 'bug_type': bug['type'],
289 'bug_category': bug['category'],
290 'bug_line': int(bug['location']['line']),
291 'bug_path_length': int(bug['location']['col']),
292 'bug_file': files[int(bug['location']['file'])]
293 }
294
295
296def parse_bug_html(filename):
297 """ Parse out the bug information from HTML output. """
298
299 patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
300 re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
301 re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
302 re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
303 re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
304 re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
305 re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
306 endsign = re.compile(r'<!-- BUGMETAEND -->')
307
308 bug = {
309 'report_file': filename,
310 'bug_function': 'n/a', # compatibility with < clang-3.5
311 'bug_category': 'Other',
312 'bug_line': 0,
313 'bug_path_length': 1
314 }
315
316 with open(filename) as handler:
317 for line in handler.readlines():
318 # do not read the file further
319 if endsign.match(line):
320 break
321 # search for the right lines
322 for regex in patterns:
323 match = regex.match(line.strip())
324 if match:
325 bug.update(match.groupdict())
326 break
327
328 encode_value(bug, 'bug_line', int)
329 encode_value(bug, 'bug_path_length', int)
330
331 yield bug
332
333
334def parse_crash(filename):
335 """ Parse out the crash information from the report file. """
336
337 match = re.match(r'(.*)\.info\.txt', filename)
338 name = match.group(1) if match else None
339 with open(filename) as handler:
340 lines = handler.readlines()
341 return {
342 'source': lines[0].rstrip(),
343 'problem': lines[1].rstrip(),
344 'file': name,
345 'info': name + '.info.txt',
346 'stderr': name + '.stderr.txt'
347 }
348
349
350def category_type_name(bug):
351 """ Create a new bug attribute from bug by category and type.
352
353 The result will be used as CSS class selector in the final report. """
354
355 def smash(key):
356 """ Make value ready to be HTML attribute value. """
357
358 return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
359
360 return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
361
362
363def create_counters():
364 """ Create counters for bug statistics.
365
366 Two entries are maintained: 'total' is an integer, represents the
367 number of bugs. The 'categories' is a two level categorisation of bug
368 counters. The first level is 'bug category' the second is 'bug type'.
369 Each entry in this classification is a dictionary of 'count', 'type'
370 and 'label'. """
371
372 def predicate(bug):
373 bug_category = bug['bug_category']
374 bug_type = bug['bug_type']
375 current_category = predicate.categories.get(bug_category, dict())
376 current_type = current_category.get(bug_type, {
377 'bug_type': bug_type,
378 'bug_type_class': category_type_name(bug),
379 'bug_count': 0
380 })
381 current_type.update({'bug_count': current_type['bug_count'] + 1})
382 current_category.update({bug_type: current_type})
383 predicate.categories.update({bug_category: current_category})
384 predicate.total += 1
385
386 predicate.total = 0
387 predicate.categories = dict()
388 return predicate
389
390
391def prettify_bug(prefix, output_dir):
392 def predicate(bug):
393 """ Make safe this values to embed into HTML. """
394
395 bug['bug_type_class'] = category_type_name(bug)
396
397 encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
398 encode_value(bug, 'bug_category', escape)
399 encode_value(bug, 'bug_type', escape)
400 encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
401 return bug
402
403 return predicate
404
405
406def prettify_crash(prefix, output_dir):
407 def predicate(crash):
408 """ Make safe this values to embed into HTML. """
409
410 encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
411 encode_value(crash, 'problem', escape)
412 encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
413 encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
414 encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
415 return crash
416
417 return predicate
418
419
420def copy_resource_files(output_dir):
421 """ Copy the javascript and css files to the report directory. """
422
423 this_dir = os.path.dirname(os.path.realpath(__file__))
424 for resource in os.listdir(os.path.join(this_dir, 'resources')):
425 shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
426
427
428def encode_value(container, key, encode):
429 """ Run 'encode' on 'container[key]' value and update it. """
430
431 if key in container:
432 value = encode(container[key])
433 container.update({key: value})
434
435
436def chop(prefix, filename):
437 """ Create 'filename' from '/prefix/filename' """
438
439 return filename if not len(prefix) else os.path.relpath(filename, prefix)
440
441
442def escape(text):
443 """ Paranoid HTML escape method. (Python version independent) """
444
445 escape_table = {
446 '&': '&amp;',
447 '"': '&quot;',
448 "'": '&apos;',
449 '>': '&gt;',
450 '<': '&lt;'
451 }
452 return ''.join(escape_table.get(c, c) for c in text)
453
454
455def reindent(text, indent):
456 """ Utility function to format html output and keep indentation. """
457
458 result = ''
459 for line in text.splitlines():
460 if len(line.strip()):
461 result += ' ' * indent + line.split('|')[1] + os.linesep
462 return result
463
464
465def comment(name, opts=dict()):
466 """ Utility function to format meta information as comment. """
467
468 attributes = ''
469 for key, value in opts.items():
470 attributes += ' {0}="{1}"'.format(key, value)
471
472 return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
473
474
475def commonprefix_from(filename):
476 """ Create file prefix from a compilation database entries. """
477
478 with open(filename, 'r') as handle:
479 return commonprefix(item['file'] for item in json.load(handle))
480
481
482def commonprefix(files):
483 """ Fixed version of os.path.commonprefix. Return the longest path prefix
484 that is a prefix of all paths in filenames. """
485
486 result = None
487 for current in files:
488 if result is not None:
489 result = os.path.commonprefix([result, current])
490 else:
491 result = current
492
493 if result is None:
494 return ''
495 elif not os.path.isdir(result):
496 return os.path.dirname(result)
497 else:
498 return os.path.abspath(result)