Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # |
| 4 | # Copyright 2015, The Android Open Source Project |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | # |
| 18 | |
| 19 | """Script that is used by developers to run style checks on Java files.""" |
| 20 | |
| 21 | import argparse |
| 22 | import errno |
| 23 | import os |
| 24 | import shutil |
| 25 | import subprocess |
| 26 | import sys |
| 27 | import tempfile |
| 28 | import xml.dom.minidom |
| 29 | import gitlint.git as git |
| 30 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 31 | |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 32 | def _FindFoldersContaining(root, wanted): |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 33 | """Recursively finds directories that have a file with the given name. |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 34 | |
| 35 | Args: |
| 36 | root: Root folder to start the search from. |
| 37 | wanted: The filename that we are looking for. |
| 38 | |
| 39 | Returns: |
| 40 | List of folders that has a file with the given name |
| 41 | """ |
| 42 | |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 43 | if not root: |
| 44 | return [] |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 45 | if os.path.islink(root): |
| 46 | return [] |
| 47 | result = [] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 48 | for file_name in os.listdir(root): |
| 49 | file_path = os.path.join(root, file_name) |
| 50 | if os.path.isdir(file_path): |
| 51 | sub_result = _FindFoldersContaining(file_path, wanted) |
| 52 | result.extend(sub_result) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 53 | else: |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 54 | if file_name == wanted: |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 55 | result.append(root) |
| 56 | return result |
| 57 | |
| 58 | MAIN_DIRECTORY = os.path.normpath(os.path.dirname(__file__)) |
| 59 | CHECKSTYLE_JAR = os.path.join(MAIN_DIRECTORY, 'checkstyle.jar') |
| 60 | CHECKSTYLE_STYLE = os.path.join(MAIN_DIRECTORY, 'android-style.xml') |
| 61 | FORCED_RULES = ['com.puppycrawl.tools.checkstyle.checks.imports.ImportOrderCheck', |
| 62 | 'com.puppycrawl.tools.checkstyle.checks.imports.UnusedImportsCheck'] |
| 63 | SKIPPED_RULES_FOR_TEST_FILES = ['com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTypeCheck', |
| 64 | 'com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocMethodCheck'] |
| 65 | SUBPATH_FOR_TEST_FILES = ['/tests/', '/test/', '/androidTest/'] |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 66 | SUBPATH_FOR_TEST_DATA_FILES = _FindFoldersContaining(git.repository_root(), |
| 67 | 'IGNORE_CHECKSTYLE') |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 68 | ERROR_UNCOMMITTED = 'You need to commit all modified files before running Checkstyle\n' |
| 69 | ERROR_UNTRACKED = 'You have untracked java files that are not being checked:\n' |
| 70 | |
| 71 | |
| 72 | def RunCheckstyleOnFiles(java_files, classpath=CHECKSTYLE_JAR, config_xml=CHECKSTYLE_STYLE): |
| 73 | """Runs Checkstyle checks on a given set of java_files. |
| 74 | |
| 75 | Args: |
| 76 | java_files: A list of files to check. |
| 77 | classpath: The colon-delimited list of JARs in the classpath. |
| 78 | config_xml: Path of the checkstyle XML configuration file. |
| 79 | |
| 80 | Returns: |
| 81 | A tuple of errors and warnings. |
| 82 | """ |
| 83 | print 'Running Checkstyle on inputted files' |
| 84 | java_files = map(os.path.abspath, java_files) |
| 85 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 86 | (errors, warnings) = _ParseAndFilterOutput(stdout) |
| 87 | _PrintErrorsAndWarnings(errors, warnings) |
| 88 | return errors, warnings |
| 89 | |
| 90 | |
| 91 | def RunCheckstyleOnACommit(commit, |
| 92 | classpath=CHECKSTYLE_JAR, |
| 93 | config_xml=CHECKSTYLE_STYLE, |
| 94 | file_whitelist=None): |
| 95 | """Runs Checkstyle checks on a given commit. |
| 96 | |
| 97 | It will run Checkstyle on the changed Java files in a specified commit SHA-1 |
| 98 | and if that is None it will fallback to check the latest commit of the |
| 99 | currently checked out branch. |
| 100 | |
| 101 | Args: |
| 102 | commit: A full 40 character SHA-1 of a commit to check. |
| 103 | classpath: The colon-delimited list of JARs in the classpath. |
| 104 | config_xml: Path of the checkstyle XML configuration file. |
| 105 | file_whitelist: A list of whitelisted file paths that should be checked. |
| 106 | |
| 107 | Returns: |
| 108 | A tuple of errors and warnings. |
| 109 | """ |
Aurimas Liutikas | 4a7cc7e | 2017-01-05 17:41:25 -0800 | [diff] [blame] | 110 | if not git.repository_root(): |
| 111 | print 'FAILURE: not inside a git repository' |
| 112 | sys.exit(1) |
Aurimas Liutikas | 0383fc2 | 2016-12-14 19:09:31 -0800 | [diff] [blame] | 113 | explicit_commit = commit is not None |
| 114 | if not explicit_commit: |
| 115 | _WarnIfUntrackedFiles() |
| 116 | commit = git.last_commit() |
| 117 | print 'Running Checkstyle on %s commit' % commit |
| 118 | commit_modified_files = _GetModifiedFiles(commit, explicit_commit) |
| 119 | commit_modified_files = _FilterFiles(commit_modified_files, file_whitelist) |
| 120 | if not commit_modified_files.keys(): |
| 121 | print 'No Java files to check' |
| 122 | return [], [] |
| 123 | |
| 124 | (tmp_dir, tmp_file_map) = _GetTempFilesForCommit( |
| 125 | commit_modified_files.keys(), commit) |
| 126 | |
| 127 | java_files = tmp_file_map.keys() |
| 128 | stdout = _ExecuteCheckstyle(java_files, classpath, config_xml) |
| 129 | |
| 130 | # Remove all the temporary files. |
| 131 | shutil.rmtree(tmp_dir) |
| 132 | |
| 133 | (errors, warnings) = _ParseAndFilterOutput(stdout, |
| 134 | commit, |
| 135 | commit_modified_files, |
| 136 | tmp_file_map) |
| 137 | _PrintErrorsAndWarnings(errors, warnings) |
| 138 | return errors, warnings |
| 139 | |
| 140 | |
| 141 | def _WarnIfUntrackedFiles(out=sys.stdout): |
| 142 | """Prints a warning and a list of untracked files if needed.""" |
| 143 | root = git.repository_root() |
| 144 | untracked_files = git.modified_files(root, False) |
| 145 | untracked_files = {f for f in untracked_files if f.endswith('.java')} |
| 146 | if untracked_files: |
| 147 | out.write(ERROR_UNTRACKED) |
| 148 | for untracked_file in untracked_files: |
| 149 | out.write(untracked_file + '\n') |
| 150 | out.write('\n') |
| 151 | |
| 152 | |
| 153 | def _PrintErrorsAndWarnings(errors, warnings): |
| 154 | """Prints given errors and warnings.""" |
| 155 | if errors: |
| 156 | print 'ERRORS:' |
| 157 | print '\n'.join(errors) |
| 158 | if warnings: |
| 159 | print 'WARNINGS:' |
| 160 | print '\n'.join(warnings) |
| 161 | |
| 162 | |
| 163 | def _ExecuteCheckstyle(java_files, classpath, config_xml): |
| 164 | """Runs Checkstyle to check give Java files for style errors. |
| 165 | |
| 166 | Args: |
| 167 | java_files: A list of Java files that needs to be checked. |
| 168 | classpath: The colon-delimited list of JARs in the classpath. |
| 169 | config_xml: Path of the checkstyle XML configuration file. |
| 170 | |
| 171 | Returns: |
| 172 | Checkstyle output in XML format. |
| 173 | """ |
| 174 | # Run checkstyle |
| 175 | checkstyle_env = os.environ.copy() |
| 176 | checkstyle_env['JAVA_CMD'] = 'java' |
| 177 | try: |
| 178 | check = subprocess.Popen(['java', '-cp', classpath, |
| 179 | 'com.puppycrawl.tools.checkstyle.Main', '-c', |
| 180 | config_xml, '-f', 'xml'] + java_files, |
| 181 | stdout=subprocess.PIPE, env=checkstyle_env) |
| 182 | stdout, _ = check.communicate() |
| 183 | except OSError as e: |
| 184 | if e.errno == errno.ENOENT: |
| 185 | print 'Error running Checkstyle!' |
| 186 | sys.exit(1) |
| 187 | |
| 188 | # A work-around for Checkstyle printing error count to stdio. |
| 189 | if 'Checkstyle ends with' in stdout.splitlines()[-1]: |
| 190 | stdout = '\n'.join(stdout.splitlines()[:-1]) |
| 191 | return stdout |
| 192 | |
| 193 | |
| 194 | def _ParseAndFilterOutput(stdout, |
| 195 | sha=None, |
| 196 | commit_modified_files=None, |
| 197 | tmp_file_map=None): |
| 198 | result_errors = [] |
| 199 | result_warnings = [] |
| 200 | root = xml.dom.minidom.parseString(stdout) |
| 201 | for file_element in root.getElementsByTagName('file'): |
| 202 | file_name = file_element.attributes['name'].value |
| 203 | if tmp_file_map: |
| 204 | file_name = tmp_file_map[file_name] |
| 205 | modified_lines = None |
| 206 | if commit_modified_files: |
| 207 | modified_lines = git.modified_lines(file_name, |
| 208 | commit_modified_files[file_name], |
| 209 | sha) |
| 210 | test_class = any(substring in file_name for substring |
| 211 | in SUBPATH_FOR_TEST_FILES) |
| 212 | test_data_class = any(substring in file_name for substring |
| 213 | in SUBPATH_FOR_TEST_DATA_FILES) |
| 214 | file_name = os.path.relpath(file_name) |
| 215 | errors = file_element.getElementsByTagName('error') |
| 216 | for error in errors: |
| 217 | line = int(error.attributes['line'].value) |
| 218 | rule = error.attributes['source'].value |
| 219 | if _ShouldSkip(commit_modified_files, modified_lines, line, rule, |
| 220 | test_class, test_data_class): |
| 221 | continue |
| 222 | |
| 223 | column = '' |
| 224 | if error.hasAttribute('column'): |
| 225 | column = '%s:' % error.attributes['column'].value |
| 226 | message = error.attributes['message'].value |
| 227 | project = '' |
| 228 | if os.environ.get('REPO_PROJECT'): |
| 229 | project = '[' + os.environ.get('REPO_PROJECT') + '] ' |
| 230 | |
| 231 | result = ' %s%s:%s:%s %s' % (project, file_name, line, column, message) |
| 232 | |
| 233 | severity = error.attributes['severity'].value |
| 234 | if severity == 'error': |
| 235 | result_errors.append(result) |
| 236 | elif severity == 'warning': |
| 237 | result_warnings.append(result) |
| 238 | return result_errors, result_warnings |
| 239 | |
| 240 | |
| 241 | def _ShouldSkip(commit_check, modified_lines, line, rule, test_class=False, |
| 242 | test_data_class=False): |
| 243 | """Returns whether an error on a given line should be skipped. |
| 244 | |
| 245 | Args: |
| 246 | commit_check: Whether Checkstyle is being run on a specific commit. |
| 247 | modified_lines: A list of lines that has been modified. |
| 248 | line: The line that has a rule violation. |
| 249 | rule: The type of rule that a given line is violating. |
| 250 | test_class: Whether the file being checked is a test class. |
| 251 | test_data_class: Whether the file being check is a class used as test data. |
| 252 | |
| 253 | Returns: |
| 254 | A boolean whether a given line should be skipped in the reporting. |
| 255 | """ |
| 256 | # None modified_lines means checked file is new and nothing should be skipped. |
| 257 | if test_data_class: |
| 258 | return True |
| 259 | if test_class and rule in SKIPPED_RULES_FOR_TEST_FILES: |
| 260 | return True |
| 261 | if not commit_check: |
| 262 | return False |
| 263 | if modified_lines is None: |
| 264 | return False |
| 265 | return line not in modified_lines and rule not in FORCED_RULES |
| 266 | |
| 267 | |
| 268 | def _GetModifiedFiles(commit, explicit_commit=False, out=sys.stdout): |
| 269 | root = git.repository_root() |
| 270 | pending_files = git.modified_files(root, True) |
| 271 | if pending_files and not explicit_commit: |
| 272 | out.write(ERROR_UNCOMMITTED) |
| 273 | sys.exit(1) |
| 274 | |
| 275 | modified_files = git.modified_files(root, True, commit) |
| 276 | modified_files = {f: modified_files[f] for f |
| 277 | in modified_files if f.endswith('.java')} |
| 278 | return modified_files |
| 279 | |
| 280 | |
| 281 | def _FilterFiles(files, file_whitelist): |
| 282 | if not file_whitelist: |
| 283 | return files |
| 284 | return {f: files[f] for f in files |
| 285 | for whitelist in file_whitelist if whitelist in f} |
| 286 | |
| 287 | |
| 288 | def _GetTempFilesForCommit(file_names, commit): |
| 289 | """Creates a temporary snapshot of the files in at a commit. |
| 290 | |
| 291 | Retrieves the state of every file in file_names at a given commit and writes |
| 292 | them all out to a temporary directory. |
| 293 | |
| 294 | Args: |
| 295 | file_names: A list of files that need to be retrieved. |
| 296 | commit: A full 40 character SHA-1 of a commit. |
| 297 | |
| 298 | Returns: |
| 299 | A tuple of temprorary directory name and a directionary of |
| 300 | temp_file_name: filename. For example: |
| 301 | |
| 302 | ('/tmp/random/', {'/tmp/random/blarg.java': 'real/path/to/file.java' } |
| 303 | """ |
| 304 | tmp_dir_name = tempfile.mkdtemp() |
| 305 | tmp_file_names = {} |
| 306 | for file_name in file_names: |
| 307 | rel_path = os.path.relpath(file_name) |
| 308 | content = subprocess.check_output( |
| 309 | ['git', 'show', commit + ':' + rel_path]) |
| 310 | |
| 311 | tmp_file_name = os.path.join(tmp_dir_name, rel_path) |
| 312 | # create directory for the file if it doesn't exist |
| 313 | if not os.path.exists(os.path.dirname(tmp_file_name)): |
| 314 | os.makedirs(os.path.dirname(tmp_file_name)) |
| 315 | |
| 316 | tmp_file = open(tmp_file_name, 'w') |
| 317 | tmp_file.write(content) |
| 318 | tmp_file.close() |
| 319 | tmp_file_names[tmp_file_name] = file_name |
| 320 | return tmp_dir_name, tmp_file_names |
| 321 | |
| 322 | |
| 323 | def main(args=None): |
| 324 | """Runs Checkstyle checks on a given set of java files or a commit. |
| 325 | |
| 326 | It will run Checkstyle on the list of java files first, if unspecified, |
| 327 | then the check will be run on a specified commit SHA-1 and if that |
| 328 | is None it will fallback to check the latest commit of the currently checked |
| 329 | out branch. |
| 330 | """ |
| 331 | parser = argparse.ArgumentParser() |
| 332 | parser.add_argument('--file', '-f', nargs='+') |
| 333 | parser.add_argument('--sha', '-s') |
| 334 | parser.add_argument('--config_xml', '-c') |
| 335 | parser.add_argument('--file_whitelist', '-fw', nargs='+') |
| 336 | parser.add_argument('--add_classpath', '-p') |
| 337 | args = parser.parse_args() |
| 338 | |
| 339 | config_xml = args.config_xml or CHECKSTYLE_STYLE |
| 340 | |
| 341 | if not os.path.exists(config_xml): |
| 342 | print 'Java checkstyle configuration file is missing' |
| 343 | sys.exit(1) |
| 344 | |
| 345 | classpath = CHECKSTYLE_JAR |
| 346 | |
| 347 | if args.add_classpath: |
| 348 | classpath = args.add_classpath + ':' + classpath |
| 349 | |
| 350 | if args.file: |
| 351 | # Files to check were specified via command line. |
| 352 | (errors, warnings) = RunCheckstyleOnFiles(args.file, classpath, config_xml) |
| 353 | else: |
| 354 | (errors, warnings) = RunCheckstyleOnACommit(args.sha, classpath, config_xml, |
| 355 | args.file_whitelist) |
| 356 | |
| 357 | if errors or warnings: |
| 358 | sys.exit(1) |
| 359 | |
| 360 | print 'SUCCESS! NO ISSUES FOUND' |
| 361 | sys.exit(0) |
| 362 | |
| 363 | |
| 364 | if __name__ == '__main__': |
| 365 | main() |