Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 2fc5a786beaba1fbac1de65d78889d72b667f2d3 / . / clang / tools / scan-build-py / libscanbuild / report.py
blob: 8bd6385fce692feaaa645101e658ef56ab05bdb4 [file] [log] [blame]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]1# -*- coding: utf-8 -*-
Chandler Carruth2946cd72019-01-19 08:50:56 +0000[diff] [blame]2# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3# See https://llvm.org/LICENSE.txt for license information.
4# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]5""" This module is responsible to generate 'index.html' for the report.
6
7The input for this step is the output directory, where individual reports
8could be found. It parses those reports and generates 'index.html'. """
9
10import re
11import os
12import os.path
13import sys
14import shutil
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]15import plistlib
16import glob
17import json
18import logging
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]19import datetime
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]20from libscanbuild import duplicate_check
21from libscanbuild.clang import get_version
22
Laszlo Nagy258ff252017-02-14 10:43:38 +0000[diff] [blame]23__all__ = ['document']
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]24
25
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]26def document(args):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]27 """ Generates cover report and returns the number of bugs/crashes. """
28
29 html_reports_available = args.output_format in {'html', 'plist-html'}
30
31 logging.debug('count crashes and bugs')
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]32 crash_count = sum(1 for _ in read_crashes(args.output))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]33 bug_counter = create_counters()
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]34 for bug in read_bugs(args.output, html_reports_available):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]35 bug_counter(bug)
36 result = crash_count + bug_counter.total
37
38 if html_reports_available and result:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]39 use_cdb = os.path.exists(args.cdb)
40
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]41 logging.debug('generate index.html file')
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]42 # common prefix for source files to have sorter path
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]43 prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
44 # assemble the cover from multiple fragments
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]45 fragments = []
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]46 try:
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]47 if bug_counter.total:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]48 fragments.append(bug_summary(args.output, bug_counter))
49 fragments.append(bug_report(args.output, prefix))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]50 if crash_count:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]51 fragments.append(crash_report(args.output, prefix))
52 assemble_cover(args, prefix, fragments)
53 # copy additional files to the report
54 copy_resource_files(args.output)
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]55 if use_cdb:
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]56 shutil.copy(args.cdb, args.output)
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]57 finally:
58 for fragment in fragments:
59 os.remove(fragment)
60 return result
61
62
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]63def assemble_cover(args, prefix, fragments):
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]64 """ Put together the fragments into a final report. """
65
66 import getpass
67 import socket
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]68
69 if args.html_title is None:
70 args.html_title = os.path.basename(prefix) + ' - analyzer results'
71
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]72 with open(os.path.join(args.output, 'index.html'), 'w') as handle:
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]73 indent = 0
74 handle.write(reindent("""
75 |<!DOCTYPE html>
76 |<html>
77 | <head>
78 | <title>{html_title}</title>
79 | <link type="text/css" rel="stylesheet" href="scanview.css"/>
80 | <script type='text/javascript' src="sorttable.js"></script>
81 | <script type='text/javascript' src='selectable.js'></script>
82 | </head>""", indent).format(html_title=args.html_title))
83 handle.write(comment('SUMMARYENDHEAD'))
84 handle.write(reindent("""
85 | <body>
86 | <h1>{html_title}</h1>
87 | <table>
88 | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr>
89 | <tr><th>Working Directory:</th><td>{current_dir}</td></tr>
90 | <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
91 | <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
92 | <tr><th>Date:</th><td>{date}</td></tr>
93 | </table>""", indent).format(html_title=args.html_title,
94 user_name=getpass.getuser(),
95 host_name=socket.gethostname(),
96 current_dir=prefix,
97 cmd_args=' '.join(sys.argv),
98 clang_version=get_version(args.clang),
99 date=datetime.datetime.today(
100 ).strftime('%c')))
101 for fragment in fragments:
102 # copy the content of fragments
103 with open(fragment, 'r') as input_handle:
104 shutil.copyfileobj(input_handle, handle)
105 handle.write(reindent("""
106 | </body>
107 |</html>""", indent))
108
109
110def bug_summary(output_dir, bug_counter):
111 """ Bug summary is a HTML table to give a better overview of the bugs. """
112
113 name = os.path.join(output_dir, 'summary.html.fragment')
114 with open(name, 'w') as handle:
115 indent = 4
116 handle.write(reindent("""
117 |<h2>Bug Summary</h2>
118 |<table>
119 | <thead>
120 | <tr>
121 | <td>Bug Type</td>
122 | <td>Quantity</td>
123 | <td class="sorttable_nosort">Display?</td>
124 | </tr>
125 | </thead>
126 | <tbody>""", indent))
127 handle.write(reindent("""
128 | <tr style="font-weight:bold">
129 | <td class="SUMM_DESC">All Bugs</td>
130 | <td class="Q">{0}</td>
131 | <td>
132 | <center>
133 | <input checked type="checkbox" id="AllBugsCheck"
134 | onClick="CopyCheckedStateToCheckButtons(this);"/>
135 | </center>
136 | </td>
137 | </tr>""", indent).format(bug_counter.total))
138 for category, types in bug_counter.categories.items():
139 handle.write(reindent("""
140 | <tr>
141 | <th>{0}</th><th colspan=2></th>
142 | </tr>""", indent).format(category))
143 for bug_type in types.values():
144 handle.write(reindent("""
145 | <tr>
146 | <td class="SUMM_DESC">{bug_type}</td>
147 | <td class="Q">{bug_count}</td>
148 | <td>
149 | <center>
150 | <input checked type="checkbox"
151 | onClick="ToggleDisplay(this,'{bug_type_class}');"/>
152 | </center>
153 | </td>
154 | </tr>""", indent).format(**bug_type))
155 handle.write(reindent("""
156 | </tbody>
157 |</table>""", indent))
158 handle.write(comment('SUMMARYBUGEND'))
159 return name
160
161
162def bug_report(output_dir, prefix):
163 """ Creates a fragment from the analyzer reports. """
164
165 pretty = prettify_bug(prefix, output_dir)
166 bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
167
168 name = os.path.join(output_dir, 'bugs.html.fragment')
169 with open(name, 'w') as handle:
170 indent = 4
171 handle.write(reindent("""
172 |<h2>Reports</h2>
173 |<table class="sortable" style="table-layout:automatic">
174 | <thead>
175 | <tr>
176 | <td>Bug Group</td>
177 | <td class="sorttable_sorted">
178 | Bug Type
179 | <span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span>
180 | </td>
181 | <td>File</td>
182 | <td>Function/Method</td>
183 | <td class="Q">Line</td>
184 | <td class="Q">Path Length</td>
185 | <td class="sorttable_nosort"></td>
186 | </tr>
187 | </thead>
188 | <tbody>""", indent))
189 handle.write(comment('REPORTBUGCOL'))
190 for current in bugs:
191 handle.write(reindent("""
192 | <tr class="{bug_type_class}">
193 | <td class="DESC">{bug_category}</td>
194 | <td class="DESC">{bug_type}</td>
195 | <td>{bug_file}</td>
196 | <td class="DESC">{bug_function}</td>
197 | <td class="Q">{bug_line}</td>
198 | <td class="Q">{bug_path_length}</td>
199 | <td><a href="{report_file}#EndPath">View Report</a></td>
200 | </tr>""", indent).format(**current))
201 handle.write(comment('REPORTBUG', {'id': current['report_file']}))
202 handle.write(reindent("""
203 | </tbody>
204 |</table>""", indent))
205 handle.write(comment('REPORTBUGEND'))
206 return name
207
208
209def crash_report(output_dir, prefix):
210 """ Creates a fragment from the compiler crashes. """
211
212 pretty = prettify_crash(prefix, output_dir)
213 crashes = (pretty(crash) for crash in read_crashes(output_dir))
214
215 name = os.path.join(output_dir, 'crashes.html.fragment')
216 with open(name, 'w') as handle:
217 indent = 4
218 handle.write(reindent("""
219 |<h2>Analyzer Failures</h2>
220 |<p>The analyzer had problems processing the following files:</p>
221 |<table>
222 | <thead>
223 | <tr>
224 | <td>Problem</td>
225 | <td>Source File</td>
226 | <td>Preprocessed File</td>
227 | <td>STDERR Output</td>
228 | </tr>
229 | </thead>
230 | <tbody>""", indent))
231 for current in crashes:
232 handle.write(reindent("""
233 | <tr>
234 | <td>{problem}</td>
235 | <td>{source}</td>
236 | <td><a href="{file}">preprocessor output</a></td>
237 | <td><a href="{stderr}">analyzer std err</a></td>
238 | </tr>""", indent).format(**current))
239 handle.write(comment('REPORTPROBLEM', current))
240 handle.write(reindent("""
241 | </tbody>
242 |</table>""", indent))
243 handle.write(comment('REPORTCRASHES'))
244 return name
245
246
247def read_crashes(output_dir):
248 """ Generate a unique sequence of crashes from given output directory. """
249
250 return (parse_crash(filename)
251 for filename in glob.iglob(os.path.join(output_dir, 'failures',
252 '*.info.txt')))
253
254
255def read_bugs(output_dir, html):
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame]256 # type: (str, bool) -> Generator[Dict[str, Any], None, None]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]257 """ Generate a unique sequence of bugs from given output directory.
258
259 Duplicates can be in a project if the same module was compiled multiple
260 times with different compiler options. These would be better to show in
261 the final report (cover) only once. """
262
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame]263 def empty(file_name):
264 return os.stat(file_name).st_size == 0
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]265
266 duplicate = duplicate_check(
267 lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
268
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame]269 # get the right parser for the job.
270 parser = parse_bug_html if html else parse_bug_plist
271 # get the input files, which are not empty.
272 pattern = os.path.join(output_dir, '*.html' if html else '*.plist')
273 bug_files = (file for file in glob.iglob(pattern) if not empty(file))
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]274
Ilya Biryukov8b9b3bd2018-03-01 14:54:16 +0000[diff] [blame]275 for bug_file in bug_files:
276 for bug in parser(bug_file):
277 if not duplicate(bug):
278 yield bug
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]279
280
281def parse_bug_plist(filename):
282 """ Returns the generator of bugs from a single .plist file. """
283
284 content = plistlib.readPlist(filename)
285 files = content.get('files')
286 for bug in content.get('diagnostics', []):
287 if len(files) <= int(bug['location']['file']):
288 logging.warning('Parsing bug from "%s" failed', filename)
289 continue
290
291 yield {
292 'result': filename,
293 'bug_type': bug['type'],
294 'bug_category': bug['category'],
295 'bug_line': int(bug['location']['line']),
296 'bug_path_length': int(bug['location']['col']),
297 'bug_file': files[int(bug['location']['file'])]
298 }
299
300
301def parse_bug_html(filename):
302 """ Parse out the bug information from HTML output. """
303
304 patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
305 re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
306 re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
307 re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
308 re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
309 re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
310 re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
311 endsign = re.compile(r'<!-- BUGMETAEND -->')
312
313 bug = {
314 'report_file': filename,
315 'bug_function': 'n/a', # compatibility with < clang-3.5
316 'bug_category': 'Other',
317 'bug_line': 0,
318 'bug_path_length': 1
319 }
320
321 with open(filename) as handler:
322 for line in handler.readlines():
323 # do not read the file further
324 if endsign.match(line):
325 break
326 # search for the right lines
327 for regex in patterns:
328 match = regex.match(line.strip())
329 if match:
330 bug.update(match.groupdict())
331 break
332
333 encode_value(bug, 'bug_line', int)
334 encode_value(bug, 'bug_path_length', int)
335
336 yield bug
337
338
339def parse_crash(filename):
340 """ Parse out the crash information from the report file. """
341
342 match = re.match(r'(.*)\.info\.txt', filename)
343 name = match.group(1) if match else None
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]344 with open(filename, mode='rb') as handler:
345 # this is a workaround to fix windows read '\r\n' as new lines.
346 lines = [line.decode().rstrip() for line in handler.readlines()]
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]347 return {
Laszlo Nagyed739d92017-03-08 09:27:53 +0000[diff] [blame]348 'source': lines[0],
349 'problem': lines[1],
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]350 'file': name,
351 'info': name + '.info.txt',
352 'stderr': name + '.stderr.txt'
353 }
354
355
356def category_type_name(bug):
357 """ Create a new bug attribute from bug by category and type.
358
359 The result will be used as CSS class selector in the final report. """
360
361 def smash(key):
362 """ Make value ready to be HTML attribute value. """
363
364 return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
365
366 return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
367
368
369def create_counters():
370 """ Create counters for bug statistics.
371
372 Two entries are maintained: 'total' is an integer, represents the
373 number of bugs. The 'categories' is a two level categorisation of bug
374 counters. The first level is 'bug category' the second is 'bug type'.
375 Each entry in this classification is a dictionary of 'count', 'type'
376 and 'label'. """
377
378 def predicate(bug):
379 bug_category = bug['bug_category']
380 bug_type = bug['bug_type']
381 current_category = predicate.categories.get(bug_category, dict())
382 current_type = current_category.get(bug_type, {
383 'bug_type': bug_type,
384 'bug_type_class': category_type_name(bug),
385 'bug_count': 0
386 })
387 current_type.update({'bug_count': current_type['bug_count'] + 1})
388 current_category.update({bug_type: current_type})
389 predicate.categories.update({bug_category: current_category})
390 predicate.total += 1
391
392 predicate.total = 0
393 predicate.categories = dict()
394 return predicate
395
396
397def prettify_bug(prefix, output_dir):
398 def predicate(bug):
399 """ Make safe this values to embed into HTML. """
400
401 bug['bug_type_class'] = category_type_name(bug)
402
403 encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
404 encode_value(bug, 'bug_category', escape)
405 encode_value(bug, 'bug_type', escape)
406 encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
407 return bug
408
409 return predicate
410
411
412def prettify_crash(prefix, output_dir):
413 def predicate(crash):
414 """ Make safe this values to embed into HTML. """
415
416 encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
417 encode_value(crash, 'problem', escape)
418 encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
419 encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
420 encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
421 return crash
422
423 return predicate
424
425
426def copy_resource_files(output_dir):
427 """ Copy the javascript and css files to the report directory. """
428
429 this_dir = os.path.dirname(os.path.realpath(__file__))
430 for resource in os.listdir(os.path.join(this_dir, 'resources')):
431 shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
432
433
434def encode_value(container, key, encode):
435 """ Run 'encode' on 'container[key]' value and update it. """
436
437 if key in container:
438 value = encode(container[key])
439 container.update({key: value})
440
441
442def chop(prefix, filename):
443 """ Create 'filename' from '/prefix/filename' """
444
445 return filename if not len(prefix) else os.path.relpath(filename, prefix)
446
447
448def escape(text):
449 """ Paranoid HTML escape method. (Python version independent) """
450
451 escape_table = {
452 '&': '&amp;',
453 '"': '&quot;',
454 "'": '&apos;',
455 '>': '&gt;',
456 '<': '&lt;'
457 }
458 return ''.join(escape_table.get(c, c) for c in text)
459
460
461def reindent(text, indent):
462 """ Utility function to format html output and keep indentation. """
463
464 result = ''
465 for line in text.splitlines():
466 if len(line.strip()):
467 result += ' ' * indent + line.split('|')[1] + os.linesep
468 return result
469
470
471def comment(name, opts=dict()):
472 """ Utility function to format meta information as comment. """
473
474 attributes = ''
475 for key, value in opts.items():
476 attributes += ' {0}="{1}"'.format(key, value)
477
478 return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
479
480
481def commonprefix_from(filename):
482 """ Create file prefix from a compilation database entries. """
483
484 with open(filename, 'r') as handle:
485 return commonprefix(item['file'] for item in json.load(handle))
486
487
488def commonprefix(files):
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]489 """ Fixed version of os.path.commonprefix.
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]490
Laszlo Nagy57db7c62017-03-21 10:15:18 +0000[diff] [blame]491 :param files: list of file names.
492 :return: the longest path prefix that is a prefix of all files. """
Laszlo Nagybc687582016-01-12 22:38:41 +0000[diff] [blame]493 result = None
494 for current in files:
495 if result is not None:
496 result = os.path.commonprefix([result, current])
497 else:
498 result = current
499
500 if result is None:
501 return ''
502 elif not os.path.isdir(result):
503 return os.path.dirname(result)
504 else:
505 return os.path.abspath(result)