Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # |
| 4 | # Copyright 2015, The Android Open Source Project |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | # |
| 18 | |
| 19 | """Script that is used by developers to run style checks on Java files.""" |
| 20 | |
| 21 | import argparse |
| 22 | import errno |
| 23 | import os |
| 24 | import shutil |
| 25 | import subprocess |
| 26 | import sys |
| 27 | import tempfile |
| 28 | import xml.dom.minidom |
| 29 | import gitlint.git as git |
| 30 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 31 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 32 | def _FindFoldersContaining(root, wanted): |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 33 | """Recursively finds directories that have a file with the given name. |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 34 | |
| 35 | Args: |
| 36 | root: Root folder to start the search from. |
| 37 | wanted: The filename that we are looking for. |
| 38 | |
| 39 | Returns: |
| 40 | List of folders that has a file with the given name |
| 41 | """ |
| 42 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 43 | if not root: |
| 44 | return [] |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 45 | if os.path.islink(root): |
| 46 | return [] |
| 47 | result = [] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 48 | for file_name in os.listdir(root): |
| 49 | file_path = os.path.join(root, file_name) |
| 50 | if os.path.isdir(file_path): |
| 51 | sub_result = _FindFoldersContaining(file_path, wanted) |
| 52 | result.extend(sub_result) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 53 | else: |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 54 | if file_name == wanted: |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 55 | result.append(root) |
| 56 | return result |
| 57 | |
| 58 | MAIN_DIRECTORY = os.path.normpath(os.path.dirname(__file__)) |
| 59 | CHECKSTYLE_JAR = os.path.join(MAIN_DIRECTORY, 'checkstyle.jar') |
| 60 | CHECKSTYLE_STYLE = os.path.join(MAIN_DIRECTORY, 'android-style.xml') |
| 61 | FORCED_RULES = ['com.puppycrawl.tools.checkstyle.checks.imports.ImportOrderCheck', |
| 62 | 'com.puppycrawl.tools.checkstyle.checks.imports.UnusedImportsCheck'] |
| 63 | SKIPPED_RULES_FOR_TEST_FILES = ['com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTypeCheck', |
| 64 | 'com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocMethodCheck'] |
Aurimas Liutikas | 5fd38ee | 2018-08-03 14:39:17 -0700 | [diff] [blame] | 65 | SUBPATH_FOR_TEST_FILES = ['/tests/', '/test/', '/androidTest/', '/perftests/'] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 66 | SUBPATH_FOR_TEST_DATA_FILES = _FindFoldersContaining(git.repository_root(), |
| 67 | 'IGNORE_CHECKSTYLE') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 68 | ERROR_UNCOMMITTED = 'You need to commit all modified files before running Checkstyle\n' |
| 69 | ERROR_UNTRACKED = 'You have untracked java files that are not being checked:\n' |
| 70 | |
| 71 | |
| 72 | def RunCheckstyleOnFiles(java_files, classpath=CHECKSTYLE_JAR, config_xml=CHECKSTYLE_STYLE): |
| 73 | """Runs Checkstyle checks on a given set of java_files. |
| 74 | |
| 75 | Args: |
| 76 | java_files: A list of files to check. |
| 77 | classpath: The colon-delimited list of JARs in the classpath. |
| 78 | config_xml: Path of the checkstyle XML configuration file. |
| 79 | |
| 80 | Returns: |
| 81 | A tuple of errors and warnings. |
| 82 | """ |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 83 | print('Running Checkstyle on inputted files') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 84 | java_files = map(os.path.abspath, java_files) |
| 85 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 86 | (errors, warnings) = _ParseAndFilterOutput(stdout) |
| 87 | _PrintErrorsAndWarnings(errors, warnings) |
| 88 | return errors, warnings |
| 89 | |
| 90 | |
| 91 | def RunCheckstyleOnACommit(commit, |
| 92 | classpath=CHECKSTYLE_JAR, |
| 93 | config_xml=CHECKSTYLE_STYLE, |
| 94 | file_whitelist=None): |
| 95 | """Runs Checkstyle checks on a given commit. |
| 96 | |
| 97 | It will run Checkstyle on the changed Java files in a specified commit SHA-1 |
| 98 | and if that is None it will fallback to check the latest commit of the |
| 99 | currently checked out branch. |
| 100 | |
| 101 | Args: |
| 102 | commit: A full 40 character SHA-1 of a commit to check. |
| 103 | classpath: The colon-delimited list of JARs in the classpath. |
| 104 | config_xml: Path of the checkstyle XML configuration file. |
| 105 | file_whitelist: A list of whitelisted file paths that should be checked. |
| 106 | |
| 107 | Returns: |
| 108 | A tuple of errors and warnings. |
| 109 | """ |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 110 | if not git.repository_root(): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 111 | print('FAILURE: not inside a git repository') |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 112 | sys.exit(1) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 113 | explicit_commit = commit is not None |
| 114 | if not explicit_commit: |
| 115 | _WarnIfUntrackedFiles() |
| 116 | commit = git.last_commit() |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 117 | print('Running Checkstyle on %s commit' % commit) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 118 | commit_modified_files = _GetModifiedFiles(commit, explicit_commit) |
| 119 | commit_modified_files = _FilterFiles(commit_modified_files, file_whitelist) |
| 120 | if not commit_modified_files.keys(): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 121 | print('No Java files to check') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 122 | return [], [] |
| 123 | |
| 124 | (tmp_dir, tmp_file_map) = _GetTempFilesForCommit( |
| 125 | commit_modified_files.keys(), commit) |
| 126 | |
| 127 | java_files = tmp_file_map.keys() |
| 128 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 129 | |
| 130 | # Remove all the temporary files. |
| 131 | shutil.rmtree(tmp_dir) |
| 132 | |
| 133 | (errors, warnings) = _ParseAndFilterOutput(stdout, |
| 134 | commit, |
| 135 | commit_modified_files, |
| 136 | tmp_file_map) |
| 137 | _PrintErrorsAndWarnings(errors, warnings) |
| 138 | return errors, warnings |
| 139 | |
| 140 | |
| 141 | def _WarnIfUntrackedFiles(out=sys.stdout): |
| 142 | """Prints a warning and a list of untracked files if needed.""" |
| 143 | root = git.repository_root() |
| 144 | untracked_files = git.modified_files(root, False) |
| 145 | untracked_files = {f for f in untracked_files if f.endswith('.java')} |
| 146 | if untracked_files: |
| 147 | out.write(ERROR_UNTRACKED) |
| 148 | for untracked_file in untracked_files: |
| 149 | out.write(untracked_file + '\n') |
| 150 | out.write('\n') |
| 151 | |
| 152 | |
| 153 | def _PrintErrorsAndWarnings(errors, warnings): |
| 154 | """Prints given errors and warnings.""" |
Aurimas Liutikas | 7de5d40 | 2019-01-07 11:08:42 -0800 | [diff] [blame] | 155 | system_encoding = sys.getdefaultencoding() |
| 156 | if (system_encoding == 'ascii'): |
| 157 | system_encoding = 'UTF-8' |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 158 | if errors: |
Aurimas Liutikas | 7de5d40 | 2019-01-07 11:08:42 -0800 | [diff] [blame] | 159 | print('ERRORS:\n' + '\n'.join(map(lambda x: x.encode(system_encoding), errors))) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 160 | if warnings: |
Aurimas Liutikas | 7de5d40 | 2019-01-07 11:08:42 -0800 | [diff] [blame] | 161 | print('WARNINGS:\n' + '\n'.join(map(lambda x: x.encode(system_encoding), warnings))) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 162 | |
| 163 | |
| 164 | def _ExecuteCheckstyle(java_files, classpath, config_xml): |
| 165 | """Runs Checkstyle to check give Java files for style errors. |
| 166 | |
| 167 | Args: |
| 168 | java_files: A list of Java files that needs to be checked. |
| 169 | classpath: The colon-delimited list of JARs in the classpath. |
| 170 | config_xml: Path of the checkstyle XML configuration file. |
| 171 | |
| 172 | Returns: |
| 173 | Checkstyle output in XML format. |
| 174 | """ |
| 175 | # Run checkstyle |
| 176 | checkstyle_env = os.environ.copy() |
| 177 | checkstyle_env['JAVA_CMD'] = 'java' |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 178 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 179 | try: |
| 180 | check = subprocess.Popen(['java', '-cp', classpath, |
| 181 | 'com.puppycrawl.tools.checkstyle.Main', '-c', |
| 182 | config_xml, '-f', 'xml'] + java_files, |
| 183 | stdout=subprocess.PIPE, env=checkstyle_env) |
| 184 | stdout, _ = check.communicate() |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 185 | # A work-around for Checkstyle printing error count to stdio. |
Aurimas Liutikas | 7de5d40 | 2019-01-07 11:08:42 -0800 | [diff] [blame] | 186 | if '</checkstyle>' in stdout.splitlines()[-2]: |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 187 | stdout = '\n'.join(stdout.splitlines()[:-1]) |
| 188 | return stdout |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 189 | except OSError as e: |
| 190 | if e.errno == errno.ENOENT: |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 191 | print('Error running Checkstyle!') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 192 | sys.exit(1) |
| 193 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 194 | |
| 195 | def _ParseAndFilterOutput(stdout, |
| 196 | sha=None, |
| 197 | commit_modified_files=None, |
| 198 | tmp_file_map=None): |
| 199 | result_errors = [] |
| 200 | result_warnings = [] |
| 201 | root = xml.dom.minidom.parseString(stdout) |
| 202 | for file_element in root.getElementsByTagName('file'): |
| 203 | file_name = file_element.attributes['name'].value |
| 204 | if tmp_file_map: |
| 205 | file_name = tmp_file_map[file_name] |
| 206 | modified_lines = None |
| 207 | if commit_modified_files: |
| 208 | modified_lines = git.modified_lines(file_name, |
| 209 | commit_modified_files[file_name], |
| 210 | sha) |
| 211 | test_class = any(substring in file_name for substring |
| 212 | in SUBPATH_FOR_TEST_FILES) |
| 213 | test_data_class = any(substring in file_name for substring |
| 214 | in SUBPATH_FOR_TEST_DATA_FILES) |
| 215 | file_name = os.path.relpath(file_name) |
| 216 | errors = file_element.getElementsByTagName('error') |
| 217 | for error in errors: |
| 218 | line = int(error.attributes['line'].value) |
| 219 | rule = error.attributes['source'].value |
| 220 | if _ShouldSkip(commit_modified_files, modified_lines, line, rule, |
| 221 | test_class, test_data_class): |
| 222 | continue |
| 223 | |
| 224 | column = '' |
| 225 | if error.hasAttribute('column'): |
| 226 | column = '%s:' % error.attributes['column'].value |
| 227 | message = error.attributes['message'].value |
| 228 | project = '' |
| 229 | if os.environ.get('REPO_PROJECT'): |
| 230 | project = '[' + os.environ.get('REPO_PROJECT') + '] ' |
| 231 | |
| 232 | result = ' %s%s:%s:%s %s' % (project, file_name, line, column, message) |
| 233 | |
| 234 | severity = error.attributes['severity'].value |
| 235 | if severity == 'error': |
| 236 | result_errors.append(result) |
| 237 | elif severity == 'warning': |
| 238 | result_warnings.append(result) |
| 239 | return result_errors, result_warnings |
| 240 | |
| 241 | |
| 242 | def _ShouldSkip(commit_check, modified_lines, line, rule, test_class=False, |
| 243 | test_data_class=False): |
| 244 | """Returns whether an error on a given line should be skipped. |
| 245 | |
| 246 | Args: |
| 247 | commit_check: Whether Checkstyle is being run on a specific commit. |
| 248 | modified_lines: A list of lines that has been modified. |
| 249 | line: The line that has a rule violation. |
| 250 | rule: The type of rule that a given line is violating. |
| 251 | test_class: Whether the file being checked is a test class. |
| 252 | test_data_class: Whether the file being check is a class used as test data. |
| 253 | |
| 254 | Returns: |
| 255 | A boolean whether a given line should be skipped in the reporting. |
| 256 | """ |
| 257 | # None modified_lines means checked file is new and nothing should be skipped. |
| 258 | if test_data_class: |
| 259 | return True |
| 260 | if test_class and rule in SKIPPED_RULES_FOR_TEST_FILES: |
| 261 | return True |
| 262 | if not commit_check: |
| 263 | return False |
| 264 | if modified_lines is None: |
| 265 | return False |
| 266 | return line not in modified_lines and rule not in FORCED_RULES |
| 267 | |
| 268 | |
| 269 | def _GetModifiedFiles(commit, explicit_commit=False, out=sys.stdout): |
| 270 | root = git.repository_root() |
| 271 | pending_files = git.modified_files(root, True) |
| 272 | if pending_files and not explicit_commit: |
| 273 | out.write(ERROR_UNCOMMITTED) |
| 274 | sys.exit(1) |
| 275 | |
| 276 | modified_files = git.modified_files(root, True, commit) |
| 277 | modified_files = {f: modified_files[f] for f |
| 278 | in modified_files if f.endswith('.java')} |
| 279 | return modified_files |
| 280 | |
| 281 | |
| 282 | def _FilterFiles(files, file_whitelist): |
| 283 | if not file_whitelist: |
| 284 | return files |
| 285 | return {f: files[f] for f in files |
| 286 | for whitelist in file_whitelist if whitelist in f} |
| 287 | |
| 288 | |
| 289 | def _GetTempFilesForCommit(file_names, commit): |
| 290 | """Creates a temporary snapshot of the files in at a commit. |
| 291 | |
| 292 | Retrieves the state of every file in file_names at a given commit and writes |
| 293 | them all out to a temporary directory. |
| 294 | |
| 295 | Args: |
| 296 | file_names: A list of files that need to be retrieved. |
| 297 | commit: A full 40 character SHA-1 of a commit. |
| 298 | |
| 299 | Returns: |
| 300 | A tuple of temprorary directory name and a directionary of |
| 301 | temp_file_name: filename. For example: |
| 302 | |
| 303 | ('/tmp/random/', {'/tmp/random/blarg.java': 'real/path/to/file.java' } |
| 304 | """ |
| 305 | tmp_dir_name = tempfile.mkdtemp() |
| 306 | tmp_file_names = {} |
| 307 | for file_name in file_names: |
| 308 | rel_path = os.path.relpath(file_name) |
| 309 | content = subprocess.check_output( |
| 310 | ['git', 'show', commit + ':' + rel_path]) |
| 311 | |
| 312 | tmp_file_name = os.path.join(tmp_dir_name, rel_path) |
| 313 | # create directory for the file if it doesn't exist |
| 314 | if not os.path.exists(os.path.dirname(tmp_file_name)): |
| 315 | os.makedirs(os.path.dirname(tmp_file_name)) |
| 316 | |
| 317 | tmp_file = open(tmp_file_name, 'w') |
| 318 | tmp_file.write(content) |
| 319 | tmp_file.close() |
| 320 | tmp_file_names[tmp_file_name] = file_name |
| 321 | return tmp_dir_name, tmp_file_names |
| 322 | |
| 323 | |
| 324 | def main(args=None): |
| 325 | """Runs Checkstyle checks on a given set of java files or a commit. |
| 326 | |
| 327 | It will run Checkstyle on the list of java files first, if unspecified, |
| 328 | then the check will be run on a specified commit SHA-1 and if that |
| 329 | is None it will fallback to check the latest commit of the currently checked |
| 330 | out branch. |
| 331 | """ |
| 332 | parser = argparse.ArgumentParser() |
| 333 | parser.add_argument('--file', '-f', nargs='+') |
| 334 | parser.add_argument('--sha', '-s') |
| 335 | parser.add_argument('--config_xml', '-c') |
| 336 | parser.add_argument('--file_whitelist', '-fw', nargs='+') |
| 337 | parser.add_argument('--add_classpath', '-p') |
| 338 | args = parser.parse_args() |
| 339 | |
| 340 | config_xml = args.config_xml or CHECKSTYLE_STYLE |
| 341 | |
| 342 | if not os.path.exists(config_xml): |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 343 | print('Java checkstyle configuration file is missing') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 344 | sys.exit(1) |
| 345 | |
| 346 | classpath = CHECKSTYLE_JAR |
| 347 | |
| 348 | if args.add_classpath: |
| 349 | classpath = args.add_classpath + ':' + classpath |
| 350 | |
| 351 | if args.file: |
| 352 | # Files to check were specified via command line. |
| 353 | (errors, warnings) = RunCheckstyleOnFiles(args.file, classpath, config_xml) |
| 354 | else: |
| 355 | (errors, warnings) = RunCheckstyleOnACommit(args.sha, classpath, config_xml, |
| 356 | args.file_whitelist) |
| 357 | |
| 358 | if errors or warnings: |
| 359 | sys.exit(1) |
| 360 | |
Daniel Kutik | c52010a | 2018-04-10 22:09:13 +0800 | [diff] [blame] | 361 | print('SUCCESS! NO ISSUES FOUND') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 362 | sys.exit(0) |
| 363 | |
| 364 | |
| 365 | if __name__ == '__main__': |
| 366 | main() |