Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 2 | # Copyright 2015 gRPC authors. |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 3 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 7 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 9 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 15 | |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 16 | """Filter out tests based on file differences compared to merge target branch""" |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 17 | |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 18 | from __future__ import print_function |
| 19 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 20 | import re |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 21 | import six |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 22 | from subprocess import check_output |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 23 | |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 24 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 25 | class TestSuite: |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 26 | """ |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 27 | Contains label to identify job as belonging to this test suite and |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 28 | triggers to identify if changed files are relevant |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 29 | """ |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 30 | def __init__(self, labels): |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 31 | """ |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 32 | Build TestSuite to group tests based on labeling |
| 33 | :param label: strings that should match a jobs's platform, config, language, or test group |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 34 | """ |
| 35 | self.triggers = [] |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 36 | self.labels = labels |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 37 | |
| 38 | def add_trigger(self, trigger): |
| 39 | """ |
| 40 | Add a regex to list of triggers that determine if a changed file should run tests |
| 41 | :param trigger: regex matching file relevant to tests |
| 42 | """ |
| 43 | self.triggers.append(trigger) |
| 44 | |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 45 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 46 | # Create test suites |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 47 | _CORE_TEST_SUITE = TestSuite(['c']) |
| 48 | _CPP_TEST_SUITE = TestSuite(['c++']) |
| 49 | _CSHARP_TEST_SUITE = TestSuite(['csharp']) |
murgatroid99 | 3800775 | 2017-10-18 11:24:17 -0700 | [diff] [blame] | 50 | _NODE_TEST_SUITE = TestSuite(['grpc-node']) |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 51 | _OBJC_TEST_SUITE = TestSuite(['objc']) |
| 52 | _PHP_TEST_SUITE = TestSuite(['php', 'php7']) |
| 53 | _PYTHON_TEST_SUITE = TestSuite(['python']) |
| 54 | _RUBY_TEST_SUITE = TestSuite(['ruby']) |
| 55 | _LINUX_TEST_SUITE = TestSuite(['linux']) |
| 56 | _WINDOWS_TEST_SUITE = TestSuite(['windows']) |
| 57 | _MACOS_TEST_SUITE = TestSuite(['macos']) |
Matt Kwong | fe1bcd9 | 2016-11-07 14:07:06 -0800 | [diff] [blame] | 58 | _ALL_TEST_SUITES = [_CORE_TEST_SUITE, _CPP_TEST_SUITE, _CSHARP_TEST_SUITE, |
| 59 | _NODE_TEST_SUITE, _OBJC_TEST_SUITE, _PHP_TEST_SUITE, |
| 60 | _PYTHON_TEST_SUITE, _RUBY_TEST_SUITE, _LINUX_TEST_SUITE, |
| 61 | _WINDOWS_TEST_SUITE, _MACOS_TEST_SUITE] |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 62 | |
| 63 | # Dictionary of whitelistable files where the key is a regex matching changed files |
| 64 | # and the value is a list of tests that should be run. An empty list means that |
| 65 | # the changed files should not trigger any tests. Any changed file that does not |
| 66 | # match any of these regexes will trigger all tests |
Matt Kwong | 5c691c6 | 2016-10-20 17:11:18 -0700 | [diff] [blame] | 67 | # DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING (be careful even if you do) |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 68 | _WHITELIST_DICT = { |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 69 | '^doc/': [], |
| 70 | '^examples/': [], |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 71 | '^include/grpc\+\+/': [_CPP_TEST_SUITE], |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 72 | '^summerofcode/': [], |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 73 | '^src/cpp/': [_CPP_TEST_SUITE], |
| 74 | '^src/csharp/': [_CSHARP_TEST_SUITE], |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 75 | '^src/objective\-c/': [_OBJC_TEST_SUITE], |
| 76 | '^src/php/': [_PHP_TEST_SUITE], |
| 77 | '^src/python/': [_PYTHON_TEST_SUITE], |
| 78 | '^src/ruby/': [_RUBY_TEST_SUITE], |
Matt Kwong | fe1bcd9 | 2016-11-07 14:07:06 -0800 | [diff] [blame] | 79 | '^templates/': [], |
Craig Tiller | 520e59f | 2017-10-07 17:34:41 -0700 | [diff] [blame] | 80 | '^test/core/': [_CORE_TEST_SUITE, _CPP_TEST_SUITE], |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 81 | '^test/cpp/': [_CPP_TEST_SUITE], |
| 82 | '^test/distrib/cpp/': [_CPP_TEST_SUITE], |
| 83 | '^test/distrib/csharp/': [_CSHARP_TEST_SUITE], |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 84 | '^test/distrib/php/': [_PHP_TEST_SUITE], |
| 85 | '^test/distrib/python/': [_PYTHON_TEST_SUITE], |
| 86 | '^test/distrib/ruby/': [_RUBY_TEST_SUITE], |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 87 | '^vsprojects/': [_WINDOWS_TEST_SUITE], |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 88 | 'composer\.json$': [_PHP_TEST_SUITE], |
| 89 | 'config\.m4$': [_PHP_TEST_SUITE], |
| 90 | 'CONTRIBUTING\.md$': [], |
| 91 | 'Gemfile$': [_RUBY_TEST_SUITE], |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 92 | 'grpc\.def$': [_WINDOWS_TEST_SUITE], |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 93 | 'grpc\.gemspec$': [_RUBY_TEST_SUITE], |
| 94 | 'gRPC\.podspec$': [_OBJC_TEST_SUITE], |
| 95 | 'gRPC\-Core\.podspec$': [_OBJC_TEST_SUITE], |
| 96 | 'gRPC\-ProtoRPC\.podspec$': [_OBJC_TEST_SUITE], |
| 97 | 'gRPC\-RxLibrary\.podspec$': [_OBJC_TEST_SUITE], |
| 98 | 'INSTALL\.md$': [], |
| 99 | 'LICENSE$': [], |
| 100 | 'MANIFEST\.md$': [], |
| 101 | 'package\.json$': [_PHP_TEST_SUITE], |
| 102 | 'package\.xml$': [_PHP_TEST_SUITE], |
| 103 | 'PATENTS$': [], |
| 104 | 'PYTHON\-MANIFEST\.in$': [_PYTHON_TEST_SUITE], |
| 105 | 'README\.md$': [], |
| 106 | 'requirements\.txt$': [_PYTHON_TEST_SUITE], |
| 107 | 'setup\.cfg$': [_PYTHON_TEST_SUITE], |
| 108 | 'setup\.py$': [_PYTHON_TEST_SUITE] |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 109 | } |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 110 | |
Matt Kwong | 5eecba4 | 2017-04-05 15:09:21 -0700 | [diff] [blame] | 111 | # Regex that combines all keys in _WHITELIST_DICT |
| 112 | _ALL_TRIGGERS = "(" + ")|(".join(_WHITELIST_DICT.keys()) + ")" |
| 113 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 114 | # Add all triggers to their respective test suites |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 115 | for trigger, test_suites in six.iteritems(_WHITELIST_DICT): |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 116 | for test_suite in test_suites: |
| 117 | test_suite.add_trigger(trigger) |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 118 | |
| 119 | |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 120 | def _get_changed_files(base_branch): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 121 | """ |
Matt Kwong | 037704d | 2016-10-06 18:18:11 -0700 | [diff] [blame] | 122 | Get list of changed files between current branch and base of target merge branch |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 123 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 124 | # Get file changes between branch and merge-base of specified branch |
| 125 | # Not combined to be Windows friendly |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 126 | base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() |
Matt Kwong | 6d11e90 | 2017-08-23 11:43:40 -0700 | [diff] [blame] | 127 | return check_output(["git", "diff", base_commit, "--name-only", "HEAD"]).splitlines() |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 128 | |
| 129 | |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 130 | def _can_skip_tests(file_names, triggers): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 131 | """ |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 132 | Determines if tests are skippable based on if all files do not match list of regexes |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 133 | :param file_names: list of changed files generated by _get_changed_files() |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 134 | :param triggers: list of regexes matching file name that indicates tests should be run |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 135 | :return: safe to skip tests |
| 136 | """ |
| 137 | for file_name in file_names: |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 138 | if any(re.match(trigger, file_name) for trigger in triggers): |
| 139 | return False |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 140 | return True |
| 141 | |
| 142 | |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 143 | def _remove_irrelevant_tests(tests, skippable_labels): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 144 | """ |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 145 | Filters out tests by config or language - will not remove sanitizer tests |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 146 | :param tests: list of all tests generated by run_tests_matrix.py |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 147 | :param skippable_labels: list of languages and platforms with skippable tests |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 148 | :return: list of relevant tests |
| 149 | """ |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 150 | # test.labels[0] is platform and test.labels[2] is language |
| 151 | # We skip a test if both are considered safe to skip |
| 152 | return [test for test in tests if test.labels[0] not in skippable_labels or \ |
| 153 | test.labels[2] not in skippable_labels] |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 154 | |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 155 | |
Matt Kwong | 5eecba4 | 2017-04-05 15:09:21 -0700 | [diff] [blame] | 156 | def affects_c_cpp(base_branch): |
| 157 | """ |
| 158 | Determines if a pull request's changes affect C/C++. This function exists because |
| 159 | there are pull request tests that only test C/C++ code |
| 160 | :param base_branch: branch that a pull request is requesting to merge into |
| 161 | :return: boolean indicating whether C/C++ changes are made in pull request |
| 162 | """ |
| 163 | changed_files = _get_changed_files(base_branch) |
| 164 | # Run all tests if any changed file is not in the whitelist dictionary |
| 165 | for changed_file in changed_files: |
| 166 | if not re.match(_ALL_TRIGGERS, changed_file): |
| 167 | return True |
| 168 | return not _can_skip_tests(changed_files, _CPP_TEST_SUITE.triggers + _CORE_TEST_SUITE.triggers) |
| 169 | |
| 170 | |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 171 | def filter_tests(tests, base_branch): |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 172 | """ |
| 173 | Filters out tests that are safe to ignore |
| 174 | :param tests: list of all tests generated by run_tests_matrix.py |
| 175 | :return: list of relevant tests |
| 176 | """ |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 177 | print('Finding file differences between gRPC %s branch and pull request...\n' % base_branch) |
Matt Kwong | f3f2872 | 2016-10-07 15:25:56 -0700 | [diff] [blame] | 178 | changed_files = _get_changed_files(base_branch) |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 179 | for changed_file in changed_files: |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 180 | print(' %s' % changed_file) |
| 181 | print('') |
Matt Kwong | 5c691c6 | 2016-10-20 17:11:18 -0700 | [diff] [blame] | 182 | |
Matt Kwong | 5eecba4 | 2017-04-05 15:09:21 -0700 | [diff] [blame] | 183 | # Run all tests if any changed file is not in the whitelist dictionary |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 184 | for changed_file in changed_files: |
Matt Kwong | 5eecba4 | 2017-04-05 15:09:21 -0700 | [diff] [blame] | 185 | if not re.match(_ALL_TRIGGERS, changed_file): |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 186 | return(tests) |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 187 | # Figure out which language and platform tests to run |
| 188 | skippable_labels = [] |
Matt Kwong | 1fb0e47 | 2016-10-17 12:16:14 -0700 | [diff] [blame] | 189 | for test_suite in _ALL_TEST_SUITES: |
Matt Kwong | f01122c | 2016-10-12 18:24:53 -0700 | [diff] [blame] | 190 | if _can_skip_tests(changed_files, test_suite.triggers): |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 191 | for label in test_suite.labels: |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 192 | print(' %s tests safe to skip' % label) |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 193 | skippable_labels.append(label) |
Matt Kwong | 2c4453d | 2016-10-19 16:35:50 -0700 | [diff] [blame] | 194 | tests = _remove_irrelevant_tests(tests, skippable_labels) |
Matt Kwong | e9163f0 | 2016-10-05 11:42:55 -0700 | [diff] [blame] | 195 | return tests |