Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # |
| 4 | # Copyright 2015, The Android Open Source Project |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | # |
| 18 | |
| 19 | """Script that is used by developers to run style checks on Java files.""" |
| 20 | |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 21 | from __future__ import print_function |
| 22 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 23 | import argparse |
| 24 | import errno |
| 25 | import os |
| 26 | import shutil |
| 27 | import subprocess |
| 28 | import sys |
| 29 | import tempfile |
| 30 | import xml.dom.minidom |
| 31 | import gitlint.git as git |
| 32 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 33 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 34 | def _FindFoldersContaining(root, wanted): |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 35 | """Recursively finds directories that have a file with the given name. |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 36 | |
| 37 | Args: |
| 38 | root: Root folder to start the search from. |
| 39 | wanted: The filename that we are looking for. |
| 40 | |
| 41 | Returns: |
| 42 | List of folders that has a file with the given name |
| 43 | """ |
| 44 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 45 | if not root: |
| 46 | return [] |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 47 | if os.path.islink(root): |
| 48 | return [] |
| 49 | result = [] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 50 | for file_name in os.listdir(root): |
| 51 | file_path = os.path.join(root, file_name) |
| 52 | if os.path.isdir(file_path): |
| 53 | sub_result = _FindFoldersContaining(file_path, wanted) |
| 54 | result.extend(sub_result) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 55 | else: |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 56 | if file_name == wanted: |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 57 | result.append(root) |
| 58 | return result |
| 59 | |
| 60 | MAIN_DIRECTORY = os.path.normpath(os.path.dirname(__file__)) |
| 61 | CHECKSTYLE_JAR = os.path.join(MAIN_DIRECTORY, 'checkstyle.jar') |
| 62 | CHECKSTYLE_STYLE = os.path.join(MAIN_DIRECTORY, 'android-style.xml') |
| 63 | FORCED_RULES = ['com.puppycrawl.tools.checkstyle.checks.imports.ImportOrderCheck', |
| 64 | 'com.puppycrawl.tools.checkstyle.checks.imports.UnusedImportsCheck'] |
| 65 | SKIPPED_RULES_FOR_TEST_FILES = ['com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTypeCheck', |
| 66 | 'com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocMethodCheck'] |
Song Pan | 1b95dd7 | 2019-12-18 18:56:12 +0000 | [diff] [blame] | 67 | SUBPATH_FOR_TEST_FILES = ['/tests/', '/test/', '/androidTest/', '/perftests/', '/gts-tests/'] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 68 | SUBPATH_FOR_TEST_DATA_FILES = _FindFoldersContaining(git.repository_root(), |
| 69 | 'IGNORE_CHECKSTYLE') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 70 | ERROR_UNCOMMITTED = 'You need to commit all modified files before running Checkstyle\n' |
| 71 | ERROR_UNTRACKED = 'You have untracked java files that are not being checked:\n' |
| 72 | |
| 73 | |
| 74 | def RunCheckstyleOnFiles(java_files, classpath=CHECKSTYLE_JAR, config_xml=CHECKSTYLE_STYLE): |
| 75 | """Runs Checkstyle checks on a given set of java_files. |
| 76 | |
| 77 | Args: |
| 78 | java_files: A list of files to check. |
| 79 | classpath: The colon-delimited list of JARs in the classpath. |
| 80 | config_xml: Path of the checkstyle XML configuration file. |
| 81 | |
| 82 | Returns: |
| 83 | A tuple of errors and warnings. |
| 84 | """ |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 85 | print('Running Checkstyle on inputted files') |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 86 | java_files = list(map(os.path.abspath, java_files)) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 87 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 88 | (errors, warnings) = _ParseAndFilterOutput(stdout) |
| 89 | _PrintErrorsAndWarnings(errors, warnings) |
| 90 | return errors, warnings |
| 91 | |
| 92 | |
| 93 | def RunCheckstyleOnACommit(commit, |
| 94 | classpath=CHECKSTYLE_JAR, |
| 95 | config_xml=CHECKSTYLE_STYLE, |
| 96 | file_whitelist=None): |
| 97 | """Runs Checkstyle checks on a given commit. |
| 98 | |
| 99 | It will run Checkstyle on the changed Java files in a specified commit SHA-1 |
| 100 | and if that is None it will fallback to check the latest commit of the |
| 101 | currently checked out branch. |
| 102 | |
| 103 | Args: |
| 104 | commit: A full 40 character SHA-1 of a commit to check. |
| 105 | classpath: The colon-delimited list of JARs in the classpath. |
| 106 | config_xml: Path of the checkstyle XML configuration file. |
| 107 | file_whitelist: A list of whitelisted file paths that should be checked. |
| 108 | |
| 109 | Returns: |
| 110 | A tuple of errors and warnings. |
| 111 | """ |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 112 | if not git.repository_root(): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 113 | print('FAILURE: not inside a git repository') |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 114 | sys.exit(1) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 115 | explicit_commit = commit is not None |
| 116 | if not explicit_commit: |
| 117 | _WarnIfUntrackedFiles() |
| 118 | commit = git.last_commit() |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 119 | print('Running Checkstyle on %s commit' % commit) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 120 | commit_modified_files = _GetModifiedFiles(commit, explicit_commit) |
| 121 | commit_modified_files = _FilterFiles(commit_modified_files, file_whitelist) |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 122 | if not list(commit_modified_files.keys()): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 123 | print('No Java files to check') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 124 | return [], [] |
| 125 | |
| 126 | (tmp_dir, tmp_file_map) = _GetTempFilesForCommit( |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 127 | list(commit_modified_files.keys()), commit) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 128 | |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 129 | java_files = list(tmp_file_map.keys()) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 130 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 131 | |
| 132 | # Remove all the temporary files. |
| 133 | shutil.rmtree(tmp_dir) |
| 134 | |
| 135 | (errors, warnings) = _ParseAndFilterOutput(stdout, |
| 136 | commit, |
| 137 | commit_modified_files, |
| 138 | tmp_file_map) |
| 139 | _PrintErrorsAndWarnings(errors, warnings) |
| 140 | return errors, warnings |
| 141 | |
| 142 | |
| 143 | def _WarnIfUntrackedFiles(out=sys.stdout): |
| 144 | """Prints a warning and a list of untracked files if needed.""" |
| 145 | root = git.repository_root() |
| 146 | untracked_files = git.modified_files(root, False) |
| 147 | untracked_files = {f for f in untracked_files if f.endswith('.java')} |
| 148 | if untracked_files: |
| 149 | out.write(ERROR_UNTRACKED) |
| 150 | for untracked_file in untracked_files: |
| 151 | out.write(untracked_file + '\n') |
| 152 | out.write('\n') |
| 153 | |
| 154 | |
| 155 | def _PrintErrorsAndWarnings(errors, warnings): |
| 156 | """Prints given errors and warnings.""" |
| 157 | if errors: |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 158 | print('ERRORS:\n' + '\n'.join(errors)) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 159 | if warnings: |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 160 | print('WARNINGS:\n' + '\n'.join(warnings)) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 161 | |
| 162 | |
| 163 | def _ExecuteCheckstyle(java_files, classpath, config_xml): |
| 164 | """Runs Checkstyle to check give Java files for style errors. |
| 165 | |
| 166 | Args: |
| 167 | java_files: A list of Java files that needs to be checked. |
| 168 | classpath: The colon-delimited list of JARs in the classpath. |
| 169 | config_xml: Path of the checkstyle XML configuration file. |
| 170 | |
| 171 | Returns: |
| 172 | Checkstyle output in XML format. |
| 173 | """ |
| 174 | # Run checkstyle |
| 175 | checkstyle_env = os.environ.copy() |
| 176 | checkstyle_env['JAVA_CMD'] = 'java' |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 177 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 178 | try: |
| 179 | check = subprocess.Popen(['java', '-cp', classpath, |
| 180 | 'com.puppycrawl.tools.checkstyle.Main', '-c', |
| 181 | config_xml, '-f', 'xml'] + java_files, |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 182 | stdout=subprocess.PIPE, env=checkstyle_env, |
| 183 | universal_newlines=True) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 184 | stdout, _ = check.communicate() |
Sam Saccone | e1510db | 2020-07-09 23:19:46 +0000 | [diff] [blame] | 185 | stdout_lines = stdout.splitlines() |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 186 | # A work-around for Checkstyle printing error count to stdio. |
Sam Saccone | e1510db | 2020-07-09 23:19:46 +0000 | [diff] [blame] | 187 | if len(stdout_lines) < 2: |
| 188 | stdout = stdout_lines[0] |
| 189 | elif len(stdout_lines) >= 2 and '</checkstyle>' in stdout_lines[-2]: |
| 190 | stdout = '\n'.join(stdout_lines[:-1]) |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 191 | return stdout |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 192 | except OSError as e: |
| 193 | if e.errno == errno.ENOENT: |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 194 | print('Error running Checkstyle!') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 195 | sys.exit(1) |
| 196 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 197 | |
| 198 | def _ParseAndFilterOutput(stdout, |
| 199 | sha=None, |
| 200 | commit_modified_files=None, |
| 201 | tmp_file_map=None): |
| 202 | result_errors = [] |
| 203 | result_warnings = [] |
| 204 | root = xml.dom.minidom.parseString(stdout) |
| 205 | for file_element in root.getElementsByTagName('file'): |
| 206 | file_name = file_element.attributes['name'].value |
| 207 | if tmp_file_map: |
| 208 | file_name = tmp_file_map[file_name] |
| 209 | modified_lines = None |
| 210 | if commit_modified_files: |
| 211 | modified_lines = git.modified_lines(file_name, |
| 212 | commit_modified_files[file_name], |
| 213 | sha) |
| 214 | test_class = any(substring in file_name for substring |
| 215 | in SUBPATH_FOR_TEST_FILES) |
| 216 | test_data_class = any(substring in file_name for substring |
| 217 | in SUBPATH_FOR_TEST_DATA_FILES) |
| 218 | file_name = os.path.relpath(file_name) |
| 219 | errors = file_element.getElementsByTagName('error') |
| 220 | for error in errors: |
| 221 | line = int(error.attributes['line'].value) |
| 222 | rule = error.attributes['source'].value |
| 223 | if _ShouldSkip(commit_modified_files, modified_lines, line, rule, |
| 224 | test_class, test_data_class): |
| 225 | continue |
| 226 | |
| 227 | column = '' |
| 228 | if error.hasAttribute('column'): |
| 229 | column = '%s:' % error.attributes['column'].value |
| 230 | message = error.attributes['message'].value |
| 231 | project = '' |
| 232 | if os.environ.get('REPO_PROJECT'): |
| 233 | project = '[' + os.environ.get('REPO_PROJECT') + '] ' |
| 234 | |
| 235 | result = ' %s%s:%s:%s %s' % (project, file_name, line, column, message) |
| 236 | |
| 237 | severity = error.attributes['severity'].value |
| 238 | if severity == 'error': |
| 239 | result_errors.append(result) |
| 240 | elif severity == 'warning': |
| 241 | result_warnings.append(result) |
| 242 | return result_errors, result_warnings |
| 243 | |
| 244 | |
| 245 | def _ShouldSkip(commit_check, modified_lines, line, rule, test_class=False, |
| 246 | test_data_class=False): |
| 247 | """Returns whether an error on a given line should be skipped. |
| 248 | |
| 249 | Args: |
| 250 | commit_check: Whether Checkstyle is being run on a specific commit. |
| 251 | modified_lines: A list of lines that has been modified. |
| 252 | line: The line that has a rule violation. |
| 253 | rule: The type of rule that a given line is violating. |
| 254 | test_class: Whether the file being checked is a test class. |
| 255 | test_data_class: Whether the file being check is a class used as test data. |
| 256 | |
| 257 | Returns: |
| 258 | A boolean whether a given line should be skipped in the reporting. |
| 259 | """ |
| 260 | # None modified_lines means checked file is new and nothing should be skipped. |
| 261 | if test_data_class: |
| 262 | return True |
| 263 | if test_class and rule in SKIPPED_RULES_FOR_TEST_FILES: |
| 264 | return True |
| 265 | if not commit_check: |
| 266 | return False |
| 267 | if modified_lines is None: |
| 268 | return False |
| 269 | return line not in modified_lines and rule not in FORCED_RULES |
| 270 | |
| 271 | |
| 272 | def _GetModifiedFiles(commit, explicit_commit=False, out=sys.stdout): |
| 273 | root = git.repository_root() |
| 274 | pending_files = git.modified_files(root, True) |
| 275 | if pending_files and not explicit_commit: |
| 276 | out.write(ERROR_UNCOMMITTED) |
| 277 | sys.exit(1) |
| 278 | |
| 279 | modified_files = git.modified_files(root, True, commit) |
| 280 | modified_files = {f: modified_files[f] for f |
| 281 | in modified_files if f.endswith('.java')} |
| 282 | return modified_files |
| 283 | |
| 284 | |
| 285 | def _FilterFiles(files, file_whitelist): |
| 286 | if not file_whitelist: |
| 287 | return files |
| 288 | return {f: files[f] for f in files |
| 289 | for whitelist in file_whitelist if whitelist in f} |
| 290 | |
| 291 | |
| 292 | def _GetTempFilesForCommit(file_names, commit): |
| 293 | """Creates a temporary snapshot of the files in at a commit. |
| 294 | |
| 295 | Retrieves the state of every file in file_names at a given commit and writes |
| 296 | them all out to a temporary directory. |
| 297 | |
| 298 | Args: |
| 299 | file_names: A list of files that need to be retrieved. |
| 300 | commit: A full 40 character SHA-1 of a commit. |
| 301 | |
| 302 | Returns: |
| 303 | A tuple of temprorary directory name and a directionary of |
| 304 | temp_file_name: filename. For example: |
| 305 | |
| 306 | ('/tmp/random/', {'/tmp/random/blarg.java': 'real/path/to/file.java' } |
| 307 | """ |
| 308 | tmp_dir_name = tempfile.mkdtemp() |
| 309 | tmp_file_names = {} |
| 310 | for file_name in file_names: |
| 311 | rel_path = os.path.relpath(file_name) |
| 312 | content = subprocess.check_output( |
| 313 | ['git', 'show', commit + ':' + rel_path]) |
| 314 | |
| 315 | tmp_file_name = os.path.join(tmp_dir_name, rel_path) |
| 316 | # create directory for the file if it doesn't exist |
| 317 | if not os.path.exists(os.path.dirname(tmp_file_name)): |
| 318 | os.makedirs(os.path.dirname(tmp_file_name)) |
| 319 | |
Luca Stefani | 4ebbfec | 2020-02-09 14:08:24 +0100 | [diff] [blame] | 320 | tmp_file = open(tmp_file_name, 'wb') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 321 | tmp_file.write(content) |
| 322 | tmp_file.close() |
| 323 | tmp_file_names[tmp_file_name] = file_name |
| 324 | return tmp_dir_name, tmp_file_names |
| 325 | |
| 326 | |
| 327 | def main(args=None): |
| 328 | """Runs Checkstyle checks on a given set of java files or a commit. |
| 329 | |
| 330 | It will run Checkstyle on the list of java files first, if unspecified, |
| 331 | then the check will be run on a specified commit SHA-1 and if that |
| 332 | is None it will fallback to check the latest commit of the currently checked |
| 333 | out branch. |
| 334 | """ |
| 335 | parser = argparse.ArgumentParser() |
| 336 | parser.add_argument('--file', '-f', nargs='+') |
| 337 | parser.add_argument('--sha', '-s') |
| 338 | parser.add_argument('--config_xml', '-c') |
| 339 | parser.add_argument('--file_whitelist', '-fw', nargs='+') |
| 340 | parser.add_argument('--add_classpath', '-p') |
| 341 | args = parser.parse_args() |
| 342 | |
| 343 | config_xml = args.config_xml or CHECKSTYLE_STYLE |
| 344 | |
| 345 | if not os.path.exists(config_xml): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 346 | print('Java checkstyle configuration file is missing') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 347 | sys.exit(1) |
| 348 | |
| 349 | classpath = CHECKSTYLE_JAR |
| 350 | |
| 351 | if args.add_classpath: |
| 352 | classpath = args.add_classpath + ':' + classpath |
| 353 | |
| 354 | if args.file: |
| 355 | # Files to check were specified via command line. |
| 356 | (errors, warnings) = RunCheckstyleOnFiles(args.file, classpath, config_xml) |
| 357 | else: |
| 358 | (errors, warnings) = RunCheckstyleOnACommit(args.sha, classpath, config_xml, |
| 359 | args.file_whitelist) |
| 360 | |
| 361 | if errors or warnings: |
| 362 | sys.exit(1) |
| 363 | |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 364 | print('SUCCESS! NO ISSUES FOUND') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 365 | sys.exit(0) |
| 366 | |
| 367 | |
| 368 | if __name__ == '__main__': |
| 369 | main() |