Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
| 2 | # Copyright 2015, Google Inc. |
| 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 31 | """Filter out tests based on file differences compared to merge target branch""" |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 32 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 33 | import re |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 34 | from subprocess import call, check_output |
| 35 | |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 36 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 37 | class TestSuite: |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 38 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 39 | Contains tag to identify job as belonging to this test suite and |
| 40 | triggers to identify if changed files are relevant |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 41 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 42 | def __init__(self, tags): |
| 43 | """ |
| 44 | Build TestSuite to group tests by their tags |
| 45 | :param tag: string used to identify if a job belongs to this TestSuite |
| 46 | todo(mattkwong): Change the use of tag because do not want to depend on |
| 47 | job.shortname to identify what suite a test belongs to |
| 48 | """ |
| 49 | self.triggers = [] |
| 50 | self.tags = tags |
| 51 | |
| 52 | def add_trigger(self, trigger): |
| 53 | """ |
| 54 | Add a regex to list of triggers that determine if a changed file should run tests |
| 55 | :param trigger: regex matching file relevant to tests |
| 56 | """ |
| 57 | self.triggers.append(trigger) |
| 58 | |
| 59 | # Create test suites |
| 60 | _core_test_suite = TestSuite(['_c_']) |
| 61 | _cpp_test_suite = TestSuite(['_c++_']) |
| 62 | _csharp_test_suite = TestSuite(['_csharp_']) |
| 63 | _node_test_suite = TestSuite(['_node_']) |
| 64 | _objc_test_suite = TestSuite(['_objc_']) |
| 65 | _php_test_suite = TestSuite(['_php_', '_php7_']) |
| 66 | _python_test_suite = TestSuite(['_python_']) |
| 67 | _ruby_test_suite = TestSuite(['_ruby']) |
| 68 | _all_test_suites = [_core_test_suite, _cpp_test_suite, _csharp_test_suite, |
| 69 | _node_test_suite, _objc_test_suite, _php_test_suite, |
| 70 | _python_test_suite, _ruby_test_suite] |
| 71 | |
| 72 | # Dictionary of whitelistable files where the key is a regex matching changed files |
| 73 | # and the value is a list of tests that should be run. An empty list means that |
| 74 | # the changed files should not trigger any tests. Any changed file that does not |
| 75 | # match any of these regexes will trigger all tests |
| 76 | _WHITELIST_DICT = { |
| 77 | '^templates/.*': [], |
| 78 | '^doc/.*': [], |
| 79 | '^examples/.*': [], |
| 80 | '^summerofcode/.*': [], |
| 81 | '.*README.md$': [], |
| 82 | '.*LICENSE$': [], |
| 83 | '^src/cpp.*': [_cpp_test_suite], |
| 84 | '^src/csharp.*': [_csharp_test_suite], |
| 85 | '^src/node.*': [_node_test_suite], |
| 86 | '^src/objective-c.*': [_objc_test_suite], |
| 87 | '^src/php.*': [_php_test_suite], |
| 88 | '^src/python.*': [_python_test_suite], |
| 89 | '^src/ruby.*': [_ruby_test_suite], |
| 90 | '^test/core.*': [_core_test_suite], |
| 91 | '^test/cpp.*': [_cpp_test_suite], |
| 92 | '^test/distrib/cpp.*': [_cpp_test_suite], |
| 93 | '^test/distrib/csharp.*': [_csharp_test_suite], |
| 94 | '^test/distrib/node.*': [_node_test_suite], |
| 95 | '^test/distrib/php.*': [_php_test_suite], |
| 96 | '^test/distrib/python.*': [_python_test_suite], |
| 97 | '^test/distrib/ruby.*': [_ruby_test_suite] |
| 98 | } |
| 99 | # Add all triggers to their respective test suites |
| 100 | for trigger, test_suites in _WHITELIST_DICT.iteritems(): |
| 101 | for test_suite in test_suites: |
| 102 | test_suite.add_trigger(trigger) |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 103 | |
| 104 | |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 105 | def _get_changed_files(base_branch): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 106 | """ |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 107 | Get list of changed files between current branch and base of target merge branch |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 108 | """ |
| 109 | # git fetch might need to be called on Jenkins slave |
| 110 | # todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this |
| 111 | # call(['git', 'fetch']) |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 112 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 113 | # Get file changes between branch and merge-base of specified branch |
| 114 | # Not combined to be Windows friendly |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 115 | base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() |
| 116 | return check_output(["git", "diff", base_commit, "--name-only"]).splitlines() |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 117 | |
| 118 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 119 | def _can_skip_tests(file_names, triggers): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 120 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 121 | Determines if tests are skippable based on if all files do not match list of regexes |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 122 | :param file_names: list of changed files generated by _get_changed_files() |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 123 | :param triggers: list of regexes matching file name that indicates tests should be run |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 124 | :return: safe to skip tests |
| 125 | """ |
| 126 | for file_name in file_names: |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 127 | if any(re.match(trigger, file_name) for trigger in triggers): |
| 128 | return False |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 129 | return True |
| 130 | |
| 131 | |
| 132 | def _remove_irrelevant_tests(tests, tag): |
| 133 | """ |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 134 | Filters out tests by config or language - will not remove sanitizer tests |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 135 | :param tests: list of all tests generated by run_tests_matrix.py |
| 136 | :param tag: string representing language or config to filter - "_(language)_" or "_(config)" |
| 137 | :return: list of relevant tests |
| 138 | """ |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 139 | # todo(mattkwong): find a more reliable way to filter tests - don't use shortname |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 140 | return [test for test in tests if tag not in test.shortname or |
| 141 | any(san_tag in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 142 | |
| 143 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 144 | def _remove_sanitizer_tests(tests): |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 145 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 146 | Filters out sanitizer tests |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 147 | :param tests: list of all tests generated by run_tests_matrix.py |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 148 | :return: list of relevant tests |
| 149 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 150 | # todo(mattkwong): find a more reliable way to filter tests - don't use shortname |
| 151 | return [test for test in tests if |
| 152 | all(san_tag not in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] |
| 153 | |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 154 | |
| 155 | def filter_tests(tests, base_branch): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 156 | """ |
| 157 | Filters out tests that are safe to ignore |
| 158 | :param tests: list of all tests generated by run_tests_matrix.py |
| 159 | :return: list of relevant tests |
| 160 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 161 | print("Finding file differences between %s repo and current branch...\n" % base_branch) |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 162 | changed_files = _get_changed_files(base_branch) |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 163 | for changed_file in changed_files: |
| 164 | print(changed_file) |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 165 | print |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 166 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 167 | # Regex that combines all keys in _WHITELIST_DICT |
| 168 | all_triggers = "(" + ")|(".join(_WHITELIST_DICT.keys()) + ")" |
| 169 | # Check if all tests have to be run |
| 170 | for changed_file in changed_files: |
| 171 | if not re.match(all_triggers, changed_file): |
| 172 | return(tests) |
| 173 | # Filter out tests by language |
| 174 | for test_suite in _all_test_suites: |
| 175 | if _can_skip_tests(changed_files, test_suite.triggers): |
| 176 | for tag in test_suite.tags: |
| 177 | print(" Filtering %s tests" % tag) |
| 178 | tests = _remove_irrelevant_tests(tests, tag) |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 179 | # Sanitizer tests skipped if core and c++ are skipped |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 180 | if _can_skip_tests(changed_files, _cpp_test_suite.triggers + _core_test_suite.triggers): |
| 181 | print(" Filtering Sanitizer tests") |
| 182 | tests = _remove_sanitizer_tests(tests) |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 183 | |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 184 | return tests |