Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 1ec03e46a7b9652f8164a71a2e579fb9142cf677 / . / clang / tools / scan-build-py / libscanbuild / report.py
blob: 08170093f7079745bea53cd5a8dc994d9773f021 [file] [log] [blame]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]1# -*- coding: utf-8 -*-
2# The LLVM Compiler Infrastructure
3#
4# This file is distributed under the University of Illinois Open Source
5# License. See LICENSE.TXT for details.
6""" This module is responsible to generate 'index.html' for the report.
7
8The input for this step is the output directory, where individual reports
9could be found. It parses those reports and generates 'index.html'. """
10
11import re
12import os
13import os.path
14import sys
15import shutil
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]16import itertools
17import plistlib
18import glob
19import json
20import logging
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]21from libscanbuild import duplicate_check
22from libscanbuild.clang import get_version
23
Laszlo Nagy258ff252017-02-14 10:43:38 +0000[diff] [blame]24__all__ = ['document']
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]25
26
27def document(args, output_dir, use_cdb):
28 """ Generates cover report and returns the number of bugs/crashes. """
29
30 html_reports_available = args.output_format in {'html', 'plist-html'}
31
32 logging.debug('count crashes and bugs')
33 crash_count = sum(1 for _ in read_crashes(output_dir))
34 bug_counter = create_counters()
35 for bug in read_bugs(output_dir, html_reports_available):
36 bug_counter(bug)
37 result = crash_count + bug_counter.total
38
39 if html_reports_available and result:
40 logging.debug('generate index.html file')
41 # common prefix for source files to have sort filenames
42 prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
43 # assemble the cover from multiple fragments
44 try:
45 fragments = []
46 if bug_counter.total:
47 fragments.append(bug_summary(output_dir, bug_counter))
48 fragments.append(bug_report(output_dir, prefix))
49 if crash_count:
50 fragments.append(crash_report(output_dir, prefix))
51 assemble_cover(output_dir, prefix, args, fragments)
52 # copy additinal files to the report
53 copy_resource_files(output_dir)
54 if use_cdb:
55 shutil.copy(args.cdb, output_dir)
56 finally:
57 for fragment in fragments:
58 os.remove(fragment)
59 return result
60
61
62def assemble_cover(output_dir, prefix, args, fragments):
63 """ Put together the fragments into a final report. """
64
65 import getpass
66 import socket
67 import datetime
68
69 if args.html_title is None:
70 args.html_title = os.path.basename(prefix) + ' - analyzer results'
71
72 with open(os.path.join(output_dir, 'index.html'), 'w') as handle:
73 indent = 0
74 handle.write(reindent("""
75 |<!DOCTYPE html>
76 |<html>
77 | <head>
78 | <title>{html_title}</title>
79 | <link type="text/css" rel="stylesheet" href="scanview.css"/>
80 | <script type='text/javascript' src="sorttable.js"></script>
81 | <script type='text/javascript' src='selectable.js'></script>
82 | </head>""", indent).format(html_title=args.html_title))
83 handle.write(comment('SUMMARYENDHEAD'))
84 handle.write(reindent("""
85 | <body>
86 | <h1>{html_title}</h1>
87 | <table>
88 | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr>
89 | <tr><th>Working Directory:</th><td>{current_dir}</td></tr>
90 | <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
91 | <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
92 | <tr><th>Date:</th><td>{date}</td></tr>
93 | </table>""", indent).format(html_title=args.html_title,
94 user_name=getpass.getuser(),
95 host_name=socket.gethostname(),
96 current_dir=prefix,
97 cmd_args=' '.join(sys.argv),
98 clang_version=get_version(args.clang),
99 date=datetime.datetime.today(
100 ).strftime('%c')))
101 for fragment in fragments:
102 # copy the content of fragments
103 with open(fragment, 'r') as input_handle:
104 shutil.copyfileobj(input_handle, handle)
105 handle.write(reindent("""
106 | </body>
107 |</html>""", indent))
108
109
110def bug_summary(output_dir, bug_counter):
111 """ Bug summary is a HTML table to give a better overview of the bugs. """
112
113 name = os.path.join(output_dir, 'summary.html.fragment')
114 with open(name, 'w') as handle:
115 indent = 4
116 handle.write(reindent("""
117 |<h2>Bug Summary</h2>
118 |<table>
119 | <thead>
120 | <tr>
121 | <td>Bug Type</td>
122 | <td>Quantity</td>
123 | <td class="sorttable_nosort">Display?</td>
124 | </tr>
125 | </thead>
126 | <tbody>""", indent))
127 handle.write(reindent("""
128 | <tr style="font-weight:bold">
129 | <td class="SUMM_DESC">All Bugs</td>
130 | <td class="Q">{0}</td>
131 | <td>
132 | <center>
133 | <input checked type="checkbox" id="AllBugsCheck"
134 | onClick="CopyCheckedStateToCheckButtons(this);"/>
135 | </center>
136 | </td>
137 | </tr>""", indent).format(bug_counter.total))
138 for category, types in bug_counter.categories.items():
139 handle.write(reindent("""
140 | <tr>
141 | <th>{0}</th><th colspan=2></th>
142 | </tr>""", indent).format(category))
143 for bug_type in types.values():
144 handle.write(reindent("""
145 | <tr>
146 | <td class="SUMM_DESC">{bug_type}</td>
147 | <td class="Q">{bug_count}</td>
148 | <td>
149 | <center>
150 | <input checked type="checkbox"
151 | onClick="ToggleDisplay(this,'{bug_type_class}');"/>
152 | </center>
153 | </td>
154 | </tr>""", indent).format(**bug_type))
155 handle.write(reindent("""
156 | </tbody>
157 |</table>""", indent))
158 handle.write(comment('SUMMARYBUGEND'))
159 return name
160
161
162def bug_report(output_dir, prefix):
163 """ Creates a fragment from the analyzer reports. """
164
165 pretty = prettify_bug(prefix, output_dir)
166 bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
167
168 name = os.path.join(output_dir, 'bugs.html.fragment')
169 with open(name, 'w') as handle:
170 indent = 4
171 handle.write(reindent("""
172 |<h2>Reports</h2>
173 |<table class="sortable" style="table-layout:automatic">
174 | <thead>
175 | <tr>
176 | <td>Bug Group</td>
177 | <td class="sorttable_sorted">
178 | Bug Type
179 | <span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span>
180 | </td>
181 | <td>File</td>
182 | <td>Function/Method</td>
183 | <td class="Q">Line</td>
184 | <td class="Q">Path Length</td>
185 | <td class="sorttable_nosort"></td>
186 | </tr>
187 | </thead>
188 | <tbody>""", indent))
189 handle.write(comment('REPORTBUGCOL'))
190 for current in bugs:
191 handle.write(reindent("""
192 | <tr class="{bug_type_class}">
193 | <td class="DESC">{bug_category}</td>
194 | <td class="DESC">{bug_type}</td>
195 | <td>{bug_file}</td>
196 | <td class="DESC">{bug_function}</td>
197 | <td class="Q">{bug_line}</td>
198 | <td class="Q">{bug_path_length}</td>
199 | <td><a href="{report_file}#EndPath">View Report</a></td>
200 | </tr>""", indent).format(**current))
201 handle.write(comment('REPORTBUG', {'id': current['report_file']}))
202 handle.write(reindent("""
203 | </tbody>
204 |</table>""", indent))
205 handle.write(comment('REPORTBUGEND'))
206 return name
207
208
209def crash_report(output_dir, prefix):
210 """ Creates a fragment from the compiler crashes. """
211
212 pretty = prettify_crash(prefix, output_dir)
213 crashes = (pretty(crash) for crash in read_crashes(output_dir))
214
215 name = os.path.join(output_dir, 'crashes.html.fragment')
216 with open(name, 'w') as handle:
217 indent = 4
218 handle.write(reindent("""
219 |<h2>Analyzer Failures</h2>
220 |<p>The analyzer had problems processing the following files:</p>
221 |<table>
222 | <thead>
223 | <tr>
224 | <td>Problem</td>
225 | <td>Source File</td>
226 | <td>Preprocessed File</td>
227 | <td>STDERR Output</td>
228 | </tr>
229 | </thead>
230 | <tbody>""", indent))
231 for current in crashes:
232 handle.write(reindent("""
233 | <tr>
234 | <td>{problem}</td>
235 | <td>{source}</td>
236 | <td><a href="{file}">preprocessor output</a></td>
237 | <td><a href="{stderr}">analyzer std err</a></td>
238 | </tr>""", indent).format(**current))
239 handle.write(comment('REPORTPROBLEM', current))
240 handle.write(reindent("""
241 | </tbody>
242 |</table>""", indent))
243 handle.write(comment('REPORTCRASHES'))
244 return name
245
246
247def read_crashes(output_dir):
248 """ Generate a unique sequence of crashes from given output directory. """
249
250 return (parse_crash(filename)
251 for filename in glob.iglob(os.path.join(output_dir, 'failures',
252 '*.info.txt')))
253
254
255def read_bugs(output_dir, html):
256 """ Generate a unique sequence of bugs from given output directory.
257
258 Duplicates can be in a project if the same module was compiled multiple
259 times with different compiler options. These would be better to show in
260 the final report (cover) only once. """
261
262 parser = parse_bug_html if html else parse_bug_plist
263 pattern = '*.html' if html else '*.plist'
264
265 duplicate = duplicate_check(
266 lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
267
268 bugs = itertools.chain.from_iterable(
269 # parser creates a bug generator not the bug itself
270 parser(filename)
271 for filename in glob.iglob(os.path.join(output_dir, pattern)))
272
273 return (bug for bug in bugs if not duplicate(bug))
274
275
276def parse_bug_plist(filename):
277 """ Returns the generator of bugs from a single .plist file. """
278
279 content = plistlib.readPlist(filename)
280 files = content.get('files')
281 for bug in content.get('diagnostics', []):
282 if len(files) <= int(bug['location']['file']):
283 logging.warning('Parsing bug from "%s" failed', filename)
284 continue
285
286 yield {
287 'result': filename,
288 'bug_type': bug['type'],
289 'bug_category': bug['category'],
290 'bug_line': int(bug['location']['line']),
291 'bug_path_length': int(bug['location']['col']),
292 'bug_file': files[int(bug['location']['file'])]
293 }
294
295
296def parse_bug_html(filename):
297 """ Parse out the bug information from HTML output. """
298
299 patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
300 re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
301 re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
302 re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
303 re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
304 re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
305 re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
306 endsign = re.compile(r'<!-- BUGMETAEND -->')
307
308 bug = {
309 'report_file': filename,
310 'bug_function': 'n/a', # compatibility with < clang-3.5
311 'bug_category': 'Other',
312 'bug_line': 0,
313 'bug_path_length': 1
314 }
315
316 with open(filename) as handler:
317 for line in handler.readlines():
318 # do not read the file further
319 if endsign.match(line):
320 break
321 # search for the right lines
322 for regex in patterns:
323 match = regex.match(line.strip())
324 if match:
325 bug.update(match.groupdict())
326 break
327
328 encode_value(bug, 'bug_line', int)
329 encode_value(bug, 'bug_path_length', int)
330
331 yield bug
332
333
334def parse_crash(filename):
335 """ Parse out the crash information from the report file. """
336
337 match = re.match(r'(.*)\.info\.txt', filename)
338 name = match.group(1) if match else None
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]339 with open(filename, mode='rb') as handler:
340 # this is a workaround to fix windows read '\r\n' as new lines.
341 lines = [line.decode().rstrip() for line in handler.readlines()]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]342 return {
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]343 'source': lines[0],
344 'problem': lines[1],
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]345 'file': name,
346 'info': name + '.info.txt',
347 'stderr': name + '.stderr.txt'
348 }
349
350
351def category_type_name(bug):
352 """ Create a new bug attribute from bug by category and type.
353
354 The result will be used as CSS class selector in the final report. """
355
356 def smash(key):
357 """ Make value ready to be HTML attribute value. """
358
359 return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
360
361 return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
362
363
364def create_counters():
365 """ Create counters for bug statistics.
366
367 Two entries are maintained: 'total' is an integer, represents the
368 number of bugs. The 'categories' is a two level categorisation of bug
369 counters. The first level is 'bug category' the second is 'bug type'.
370 Each entry in this classification is a dictionary of 'count', 'type'
371 and 'label'. """
372
373 def predicate(bug):
374 bug_category = bug['bug_category']
375 bug_type = bug['bug_type']
376 current_category = predicate.categories.get(bug_category, dict())
377 current_type = current_category.get(bug_type, {
378 'bug_type': bug_type,
379 'bug_type_class': category_type_name(bug),
380 'bug_count': 0
381 })
382 current_type.update({'bug_count': current_type['bug_count'] + 1})
383 current_category.update({bug_type: current_type})
384 predicate.categories.update({bug_category: current_category})
385 predicate.total += 1
386
387 predicate.total = 0
388 predicate.categories = dict()
389 return predicate
390
391
392def prettify_bug(prefix, output_dir):
393 def predicate(bug):
394 """ Make safe this values to embed into HTML. """
395
396 bug['bug_type_class'] = category_type_name(bug)
397
398 encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
399 encode_value(bug, 'bug_category', escape)
400 encode_value(bug, 'bug_type', escape)
401 encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
402 return bug
403
404 return predicate
405
406
407def prettify_crash(prefix, output_dir):
408 def predicate(crash):
409 """ Make safe this values to embed into HTML. """
410
411 encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
412 encode_value(crash, 'problem', escape)
413 encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
414 encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
415 encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
416 return crash
417
418 return predicate
419
420
421def copy_resource_files(output_dir):
422 """ Copy the javascript and css files to the report directory. """
423
424 this_dir = os.path.dirname(os.path.realpath(__file__))
425 for resource in os.listdir(os.path.join(this_dir, 'resources')):
426 shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
427
428
429def encode_value(container, key, encode):
430 """ Run 'encode' on 'container[key]' value and update it. """
431
432 if key in container:
433 value = encode(container[key])
434 container.update({key: value})
435
436
437def chop(prefix, filename):
438 """ Create 'filename' from '/prefix/filename' """
439
440 return filename if not len(prefix) else os.path.relpath(filename, prefix)
441
442
443def escape(text):
444 """ Paranoid HTML escape method. (Python version independent) """
445
446 escape_table = {
447 '&': '&amp;',
448 '"': '&quot;',
449 "'": '&apos;',
450 '>': '&gt;',
451 '<': '&lt;'
452 }
453 return ''.join(escape_table.get(c, c) for c in text)
454
455
456def reindent(text, indent):
457 """ Utility function to format html output and keep indentation. """
458
459 result = ''
460 for line in text.splitlines():
461 if len(line.strip()):
462 result += ' ' * indent + line.split('|')[1] + os.linesep
463 return result
464
465
466def comment(name, opts=dict()):
467 """ Utility function to format meta information as comment. """
468
469 attributes = ''
470 for key, value in opts.items():
471 attributes += ' {0}="{1}"'.format(key, value)
472
473 return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
474
475
476def commonprefix_from(filename):
477 """ Create file prefix from a compilation database entries. """
478
479 with open(filename, 'r') as handle:
480 return commonprefix(item['file'] for item in json.load(handle))
481
482
483def commonprefix(files):
484 """ Fixed version of os.path.commonprefix. Return the longest path prefix
485 that is a prefix of all paths in filenames. """
486
487 result = None
488 for current in files:
489 if result is not None:
490 result = os.path.commonprefix([result, current])
491 else:
492 result = current
493
494 if result is None:
495 return ''
496 elif not os.path.isdir(result):
497 return os.path.dirname(result)
498 else:
499 return os.path.abspath(result)