Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 8b9b3bd07c5046ae3010516d84d3217934ad10d5 / . / clang / tools / scan-build-py / libscanbuild / report.py
blob: b3753c1d9d4e9d4661674b169937ba901bf56669 [file] [log] [blame]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]1# -*- coding: utf-8 -*-
2# The LLVM Compiler Infrastructure
3#
4# This file is distributed under the University of Illinois Open Source
5# License. See LICENSE.TXT for details.
6""" This module is responsible to generate 'index.html' for the report.
7
8The input for this step is the output directory, where individual reports
9could be found. It parses those reports and generates 'index.html'. """
10
11import re
12import os
13import os.path
14import sys
15import shutil
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]16import plistlib
17import glob
18import json
19import logging
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]20import datetime
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]21from libscanbuild import duplicate_check
22from libscanbuild.clang import get_version
23
Laszlo Nagy258ff252017-02-14 10:43:38 +0000[diff] [blame]24__all__ = ['document']
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]25
26
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]27def document(args):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]28 """ Generates cover report and returns the number of bugs/crashes. """
29
30 html_reports_available = args.output_format in {'html', 'plist-html'}
31
32 logging.debug('count crashes and bugs')
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]33 crash_count = sum(1 for _ in read_crashes(args.output))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]34 bug_counter = create_counters()
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]35 for bug in read_bugs(args.output, html_reports_available):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]36 bug_counter(bug)
37 result = crash_count + bug_counter.total
38
39 if html_reports_available and result:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]40 use_cdb = os.path.exists(args.cdb)
41
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]42 logging.debug('generate index.html file')
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]43 # common prefix for source files to have sorter path
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]44 prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
45 # assemble the cover from multiple fragments
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]46 fragments = []
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]47 try:
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]48 if bug_counter.total:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]49 fragments.append(bug_summary(args.output, bug_counter))
50 fragments.append(bug_report(args.output, prefix))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]51 if crash_count:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]52 fragments.append(crash_report(args.output, prefix))
53 assemble_cover(args, prefix, fragments)
54 # copy additional files to the report
55 copy_resource_files(args.output)
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]56 if use_cdb:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]57 shutil.copy(args.cdb, args.output)
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]58 finally:
59 for fragment in fragments:
60 os.remove(fragment)
61 return result
62
63
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]64def assemble_cover(args, prefix, fragments):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]65 """ Put together the fragments into a final report. """
66
67 import getpass
68 import socket
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]69
70 if args.html_title is None:
71 args.html_title = os.path.basename(prefix) + ' - analyzer results'
72
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]73 with open(os.path.join(args.output, 'index.html'), 'w') as handle:
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]74 indent = 0
75 handle.write(reindent("""
76 |<!DOCTYPE html>
77 |<html>
78 | <head>
79 | <title>{html_title}</title>
80 | <link type="text/css" rel="stylesheet" href="scanview.css"/>
81 | <script type='text/javascript' src="sorttable.js"></script>
82 | <script type='text/javascript' src='selectable.js'></script>
83 | </head>""", indent).format(html_title=args.html_title))
84 handle.write(comment('SUMMARYENDHEAD'))
85 handle.write(reindent("""
86 | <body>
87 | <h1>{html_title}</h1>
88 | <table>
89 | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr>
90 | <tr><th>Working Directory:</th><td>{current_dir}</td></tr>
91 | <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
92 | <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
93 | <tr><th>Date:</th><td>{date}</td></tr>
94 | </table>""", indent).format(html_title=args.html_title,
95 user_name=getpass.getuser(),
96 host_name=socket.gethostname(),
97 current_dir=prefix,
98 cmd_args=' '.join(sys.argv),
99 clang_version=get_version(args.clang),
100 date=datetime.datetime.today(
101 ).strftime('%c')))
102 for fragment in fragments:
103 # copy the content of fragments
104 with open(fragment, 'r') as input_handle:
105 shutil.copyfileobj(input_handle, handle)
106 handle.write(reindent("""
107 | </body>
108 |</html>""", indent))
109
110
111def bug_summary(output_dir, bug_counter):
112 """ Bug summary is a HTML table to give a better overview of the bugs. """
113
114 name = os.path.join(output_dir, 'summary.html.fragment')
115 with open(name, 'w') as handle:
116 indent = 4
117 handle.write(reindent("""
118 |<h2>Bug Summary</h2>
119 |<table>
120 | <thead>
121 | <tr>
122 | <td>Bug Type</td>
123 | <td>Quantity</td>
124 | <td class="sorttable_nosort">Display?</td>
125 | </tr>
126 | </thead>
127 | <tbody>""", indent))
128 handle.write(reindent("""
129 | <tr style="font-weight:bold">
130 | <td class="SUMM_DESC">All Bugs</td>
131 | <td class="Q">{0}</td>
132 | <td>
133 | <center>
134 | <input checked type="checkbox" id="AllBugsCheck"
135 | onClick="CopyCheckedStateToCheckButtons(this);"/>
136 | </center>
137 | </td>
138 | </tr>""", indent).format(bug_counter.total))
139 for category, types in bug_counter.categories.items():
140 handle.write(reindent("""
141 | <tr>
142 | <th>{0}</th><th colspan=2></th>
143 | </tr>""", indent).format(category))
144 for bug_type in types.values():
145 handle.write(reindent("""
146 | <tr>
147 | <td class="SUMM_DESC">{bug_type}</td>
148 | <td class="Q">{bug_count}</td>
149 | <td>
150 | <center>
151 | <input checked type="checkbox"
152 | onClick="ToggleDisplay(this,'{bug_type_class}');"/>
153 | </center>
154 | </td>
155 | </tr>""", indent).format(**bug_type))
156 handle.write(reindent("""
157 | </tbody>
158 |</table>""", indent))
159 handle.write(comment('SUMMARYBUGEND'))
160 return name
161
162
163def bug_report(output_dir, prefix):
164 """ Creates a fragment from the analyzer reports. """
165
166 pretty = prettify_bug(prefix, output_dir)
167 bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
168
169 name = os.path.join(output_dir, 'bugs.html.fragment')
170 with open(name, 'w') as handle:
171 indent = 4
172 handle.write(reindent("""
173 |<h2>Reports</h2>
174 |<table class="sortable" style="table-layout:automatic">
175 | <thead>
176 | <tr>
177 | <td>Bug Group</td>
178 | <td class="sorttable_sorted">
179 | Bug Type
180 | <span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span>
181 | </td>
182 | <td>File</td>
183 | <td>Function/Method</td>
184 | <td class="Q">Line</td>
185 | <td class="Q">Path Length</td>
186 | <td class="sorttable_nosort"></td>
187 | </tr>
188 | </thead>
189 | <tbody>""", indent))
190 handle.write(comment('REPORTBUGCOL'))
191 for current in bugs:
192 handle.write(reindent("""
193 | <tr class="{bug_type_class}">
194 | <td class="DESC">{bug_category}</td>
195 | <td class="DESC">{bug_type}</td>
196 | <td>{bug_file}</td>
197 | <td class="DESC">{bug_function}</td>
198 | <td class="Q">{bug_line}</td>
199 | <td class="Q">{bug_path_length}</td>
200 | <td><a href="{report_file}#EndPath">View Report</a></td>
201 | </tr>""", indent).format(**current))
202 handle.write(comment('REPORTBUG', {'id': current['report_file']}))
203 handle.write(reindent("""
204 | </tbody>
205 |</table>""", indent))
206 handle.write(comment('REPORTBUGEND'))
207 return name
208
209
210def crash_report(output_dir, prefix):
211 """ Creates a fragment from the compiler crashes. """
212
213 pretty = prettify_crash(prefix, output_dir)
214 crashes = (pretty(crash) for crash in read_crashes(output_dir))
215
216 name = os.path.join(output_dir, 'crashes.html.fragment')
217 with open(name, 'w') as handle:
218 indent = 4
219 handle.write(reindent("""
220 |<h2>Analyzer Failures</h2>
221 |<p>The analyzer had problems processing the following files:</p>
222 |<table>
223 | <thead>
224 | <tr>
225 | <td>Problem</td>
226 | <td>Source File</td>
227 | <td>Preprocessed File</td>
228 | <td>STDERR Output</td>
229 | </tr>
230 | </thead>
231 | <tbody>""", indent))
232 for current in crashes:
233 handle.write(reindent("""
234 | <tr>
235 | <td>{problem}</td>
236 | <td>{source}</td>
237 | <td><a href="{file}">preprocessor output</a></td>
238 | <td><a href="{stderr}">analyzer std err</a></td>
239 | </tr>""", indent).format(**current))
240 handle.write(comment('REPORTPROBLEM', current))
241 handle.write(reindent("""
242 | </tbody>
243 |</table>""", indent))
244 handle.write(comment('REPORTCRASHES'))
245 return name
246
247
248def read_crashes(output_dir):
249 """ Generate a unique sequence of crashes from given output directory. """
250
251 return (parse_crash(filename)
252 for filename in glob.iglob(os.path.join(output_dir, 'failures',
253 '*.info.txt')))
254
255
256def read_bugs(output_dir, html):
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame^]257 # type: (str, bool) -> Generator[Dict[str, Any], None, None]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]258 """ Generate a unique sequence of bugs from given output directory.
259
260 Duplicates can be in a project if the same module was compiled multiple
261 times with different compiler options. These would be better to show in
262 the final report (cover) only once. """
263
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame^]264 def empty(file_name):
265 return os.stat(file_name).st_size == 0
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]266
267 duplicate = duplicate_check(
268 lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
269
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame^]270 # get the right parser for the job.
271 parser = parse_bug_html if html else parse_bug_plist
272 # get the input files, which are not empty.
273 pattern = os.path.join(output_dir, '*.html' if html else '*.plist')
274 bug_files = (file for file in glob.iglob(pattern) if not empty(file))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]275
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame^]276 for bug_file in bug_files:
277 for bug in parser(bug_file):
278 if not duplicate(bug):
279 yield bug
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]280
281
282def parse_bug_plist(filename):
283 """ Returns the generator of bugs from a single .plist file. """
284
285 content = plistlib.readPlist(filename)
286 files = content.get('files')
287 for bug in content.get('diagnostics', []):
288 if len(files) <= int(bug['location']['file']):
289 logging.warning('Parsing bug from "%s" failed', filename)
290 continue
291
292 yield {
293 'result': filename,
294 'bug_type': bug['type'],
295 'bug_category': bug['category'],
296 'bug_line': int(bug['location']['line']),
297 'bug_path_length': int(bug['location']['col']),
298 'bug_file': files[int(bug['location']['file'])]
299 }
300
301
302def parse_bug_html(filename):
303 """ Parse out the bug information from HTML output. """
304
305 patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
306 re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
307 re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
308 re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
309 re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
310 re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
311 re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
312 endsign = re.compile(r'<!-- BUGMETAEND -->')
313
314 bug = {
315 'report_file': filename,
316 'bug_function': 'n/a', # compatibility with < clang-3.5
317 'bug_category': 'Other',
318 'bug_line': 0,
319 'bug_path_length': 1
320 }
321
322 with open(filename) as handler:
323 for line in handler.readlines():
324 # do not read the file further
325 if endsign.match(line):
326 break
327 # search for the right lines
328 for regex in patterns:
329 match = regex.match(line.strip())
330 if match:
331 bug.update(match.groupdict())
332 break
333
334 encode_value(bug, 'bug_line', int)
335 encode_value(bug, 'bug_path_length', int)
336
337 yield bug
338
339
340def parse_crash(filename):
341 """ Parse out the crash information from the report file. """
342
343 match = re.match(r'(.*)\.info\.txt', filename)
344 name = match.group(1) if match else None
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]345 with open(filename, mode='rb') as handler:
346 # this is a workaround to fix windows read '\r\n' as new lines.
347 lines = [line.decode().rstrip() for line in handler.readlines()]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]348 return {
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]349 'source': lines[0],
350 'problem': lines[1],
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]351 'file': name,
352 'info': name + '.info.txt',
353 'stderr': name + '.stderr.txt'
354 }
355
356
357def category_type_name(bug):
358 """ Create a new bug attribute from bug by category and type.
359
360 The result will be used as CSS class selector in the final report. """
361
362 def smash(key):
363 """ Make value ready to be HTML attribute value. """
364
365 return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
366
367 return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
368
369
370def create_counters():
371 """ Create counters for bug statistics.
372
373 Two entries are maintained: 'total' is an integer, represents the
374 number of bugs. The 'categories' is a two level categorisation of bug
375 counters. The first level is 'bug category' the second is 'bug type'.
376 Each entry in this classification is a dictionary of 'count', 'type'
377 and 'label'. """
378
379 def predicate(bug):
380 bug_category = bug['bug_category']
381 bug_type = bug['bug_type']
382 current_category = predicate.categories.get(bug_category, dict())
383 current_type = current_category.get(bug_type, {
384 'bug_type': bug_type,
385 'bug_type_class': category_type_name(bug),
386 'bug_count': 0
387 })
388 current_type.update({'bug_count': current_type['bug_count'] + 1})
389 current_category.update({bug_type: current_type})
390 predicate.categories.update({bug_category: current_category})
391 predicate.total += 1
392
393 predicate.total = 0
394 predicate.categories = dict()
395 return predicate
396
397
398def prettify_bug(prefix, output_dir):
399 def predicate(bug):
400 """ Make safe this values to embed into HTML. """
401
402 bug['bug_type_class'] = category_type_name(bug)
403
404 encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
405 encode_value(bug, 'bug_category', escape)
406 encode_value(bug, 'bug_type', escape)
407 encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
408 return bug
409
410 return predicate
411
412
413def prettify_crash(prefix, output_dir):
414 def predicate(crash):
415 """ Make safe this values to embed into HTML. """
416
417 encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
418 encode_value(crash, 'problem', escape)
419 encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
420 encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
421 encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
422 return crash
423
424 return predicate
425
426
427def copy_resource_files(output_dir):
428 """ Copy the javascript and css files to the report directory. """
429
430 this_dir = os.path.dirname(os.path.realpath(__file__))
431 for resource in os.listdir(os.path.join(this_dir, 'resources')):
432 shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
433
434
435def encode_value(container, key, encode):
436 """ Run 'encode' on 'container[key]' value and update it. """
437
438 if key in container:
439 value = encode(container[key])
440 container.update({key: value})
441
442
443def chop(prefix, filename):
444 """ Create 'filename' from '/prefix/filename' """
445
446 return filename if not len(prefix) else os.path.relpath(filename, prefix)
447
448
449def escape(text):
450 """ Paranoid HTML escape method. (Python version independent) """
451
452 escape_table = {
453 '&': '&amp;',
454 '"': '&quot;',
455 "'": '&apos;',
456 '>': '&gt;',
457 '<': '&lt;'
458 }
459 return ''.join(escape_table.get(c, c) for c in text)
460
461
462def reindent(text, indent):
463 """ Utility function to format html output and keep indentation. """
464
465 result = ''
466 for line in text.splitlines():
467 if len(line.strip()):
468 result += ' ' * indent + line.split('|')[1] + os.linesep
469 return result
470
471
472def comment(name, opts=dict()):
473 """ Utility function to format meta information as comment. """
474
475 attributes = ''
476 for key, value in opts.items():
477 attributes += ' {0}="{1}"'.format(key, value)
478
479 return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
480
481
482def commonprefix_from(filename):
483 """ Create file prefix from a compilation database entries. """
484
485 with open(filename, 'r') as handle:
486 return commonprefix(item['file'] for item in json.load(handle))
487
488
489def commonprefix(files):
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]490 """ Fixed version of os.path.commonprefix.
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]491
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]492 :param files: list of file names.
493 :return: the longest path prefix that is a prefix of all files. """
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]494 result = None
495 for current in files:
496 if result is not None:
497 result = os.path.commonprefix([result, current])
498 else:
499 result = current
500
501 if result is None:
502 return ''
503 elif not os.path.isdir(result):
504 return os.path.dirname(result)
505 else:
506 return os.path.abspath(result)