Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 2 | |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | # Copyright 2016 gRPC authors. |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 4 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 8 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 10 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 16 | |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 17 | import os |
| 18 | import sys |
| 19 | import unittest |
| 20 | import re |
| 21 | |
| 22 | # hack import paths to pick up extra code |
| 23 | sys.path.insert(0, os.path.abspath('tools/run_tests/')) |
| 24 | from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs |
Jan Tattermusch | 5c79a31 | 2016-12-20 11:02:50 +0100 | [diff] [blame] | 25 | import python_utils.filter_pull_request_tests as filter_pull_request_tests |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 26 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 27 | _LIST_OF_LANGUAGE_LABELS = [ |
| 28 | 'c', 'c++', 'csharp', 'grpc-node', 'objc', 'php', 'php7', 'python', 'ruby' |
| 29 | ] |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 30 | _LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows'] |
| 31 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 32 | |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 33 | class TestFilteringTest(unittest.TestCase): |
| 34 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 35 | def generate_all_tests(self): |
| 36 | all_jobs = _create_test_jobs() + _create_portability_test_jobs() |
| 37 | self.assertIsNotNone(all_jobs) |
| 38 | return all_jobs |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 39 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 40 | def test_filtering(self, changed_files=[], labels=_LIST_OF_LANGUAGE_LABELS): |
| 41 | """ |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 42 | Default args should filter no tests because changed_files is empty and |
| 43 | default labels should be able to match all jobs |
| 44 | :param changed_files: mock list of changed_files from pull request |
| 45 | :param labels: list of job labels that should be skipped |
| 46 | """ |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 47 | all_jobs = self.generate_all_tests() |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 48 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 49 | # Replacing _get_changed_files function to allow specifying changed files in filter_tests function |
| 50 | def _get_changed_files(foo): |
| 51 | return changed_files |
Matt Kwong | fe1bcd9 | 2016-11-07 14:07:06 -0800 | [diff] [blame] | 52 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 53 | filter_pull_request_tests._get_changed_files = _get_changed_files |
| 54 | print() |
| 55 | filtered_jobs = filter_pull_request_tests.filter_tests(all_jobs, "test") |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 56 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 57 | # Make sure sanity tests aren't being filtered out |
| 58 | sanity_tests_in_all_jobs = 0 |
| 59 | sanity_tests_in_filtered_jobs = 0 |
| 60 | for job in all_jobs: |
| 61 | if "sanity" in job.labels: |
| 62 | sanity_tests_in_all_jobs += 1 |
| 63 | all_jobs = [job for job in all_jobs if "sanity" not in job.labels] |
| 64 | for job in filtered_jobs: |
| 65 | if "sanity" in job.labels: |
| 66 | sanity_tests_in_filtered_jobs += 1 |
| 67 | filtered_jobs = [ |
| 68 | job for job in filtered_jobs if "sanity" not in job.labels |
| 69 | ] |
| 70 | self.assertEquals(sanity_tests_in_all_jobs, |
| 71 | sanity_tests_in_filtered_jobs) |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 72 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 73 | for label in labels: |
| 74 | for job in filtered_jobs: |
| 75 | self.assertNotIn(label, job.labels) |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 76 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 77 | jobs_matching_labels = 0 |
| 78 | for label in labels: |
| 79 | for job in all_jobs: |
| 80 | if (label in job.labels): |
| 81 | jobs_matching_labels += 1 |
| 82 | self.assertEquals( |
| 83 | len(filtered_jobs), len(all_jobs) - jobs_matching_labels) |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 84 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 85 | def test_individual_language_filters(self): |
| 86 | # Changing unlisted file should trigger all languages |
| 87 | self.test_filtering(['ffffoo/bar.baz'], [_LIST_OF_LANGUAGE_LABELS]) |
| 88 | # Changing core should trigger all tests |
| 89 | self.test_filtering(['src/core/foo.bar'], [_LIST_OF_LANGUAGE_LABELS]) |
| 90 | # Testing individual languages |
| 91 | self.test_filtering(['test/core/foo.bar'], [ |
| 92 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 93 | if label not in filter_pull_request_tests._CORE_TEST_SUITE.labels + |
| 94 | filter_pull_request_tests._CPP_TEST_SUITE.labels |
| 95 | ]) |
| 96 | self.test_filtering(['src/cpp/foo.bar'], [ |
| 97 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 98 | if label not in filter_pull_request_tests._CPP_TEST_SUITE.labels |
| 99 | ]) |
| 100 | self.test_filtering(['src/csharp/foo.bar'], [ |
| 101 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 102 | if label not in filter_pull_request_tests._CSHARP_TEST_SUITE.labels |
| 103 | ]) |
| 104 | self.test_filtering(['src/objective-c/foo.bar'], [ |
| 105 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 106 | if label not in filter_pull_request_tests._OBJC_TEST_SUITE.labels |
| 107 | ]) |
| 108 | self.test_filtering(['src/php/foo.bar'], [ |
| 109 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 110 | if label not in filter_pull_request_tests._PHP_TEST_SUITE.labels |
| 111 | ]) |
| 112 | self.test_filtering(['src/python/foo.bar'], [ |
| 113 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 114 | if label not in filter_pull_request_tests._PYTHON_TEST_SUITE.labels |
| 115 | ]) |
| 116 | self.test_filtering(['src/ruby/foo.bar'], [ |
| 117 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 118 | if label not in filter_pull_request_tests._RUBY_TEST_SUITE.labels |
| 119 | ]) |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 120 | |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 121 | def test_combined_language_filters(self): |
| 122 | self.test_filtering(['src/cpp/foo.bar', 'test/core/foo.bar'], [ |
| 123 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 124 | if label not in filter_pull_request_tests._CPP_TEST_SUITE.labels and |
| 125 | label not in filter_pull_request_tests._CORE_TEST_SUITE.labels |
| 126 | ]) |
| 127 | self.test_filtering(['src/cpp/foo.bar', "src/csharp/foo.bar"], [ |
| 128 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 129 | if label not in filter_pull_request_tests._CPP_TEST_SUITE.labels and |
| 130 | label not in filter_pull_request_tests._CSHARP_TEST_SUITE.labels |
| 131 | ]) |
| 132 | self.test_filtering([ |
| 133 | 'src/objective-c/foo.bar', 'src/php/foo.bar', "src/python/foo.bar", |
| 134 | "src/ruby/foo.bar" |
| 135 | ], [ |
| 136 | label for label in _LIST_OF_LANGUAGE_LABELS |
| 137 | if label not in filter_pull_request_tests._OBJC_TEST_SUITE.labels |
| 138 | and label not in filter_pull_request_tests._PHP_TEST_SUITE.labels |
| 139 | and label not in filter_pull_request_tests._PYTHON_TEST_SUITE.labels |
| 140 | and label not in filter_pull_request_tests._RUBY_TEST_SUITE.labels |
| 141 | ]) |
| 142 | |
| 143 | def test_platform_filter(self): |
| 144 | self.test_filtering(['vsprojects/foo.bar'], [ |
| 145 | label for label in _LIST_OF_PLATFORM_LABELS |
| 146 | if label not in filter_pull_request_tests._WINDOWS_TEST_SUITE.labels |
| 147 | ]) |
| 148 | |
| 149 | def test_whitelist(self): |
| 150 | whitelist = filter_pull_request_tests._WHITELIST_DICT |
| 151 | files_that_should_trigger_all_tests = [ |
| 152 | 'src/core/foo.bar', 'some_file_not_on_the_white_list', 'BUILD', |
| 153 | 'etc/roots.pem', 'Makefile', 'tools/foo' |
| 154 | ] |
| 155 | for key in whitelist.keys(): |
| 156 | for file_name in files_that_should_trigger_all_tests: |
| 157 | self.assertFalse(re.match(key, file_name)) |
| 158 | |
Matt Kwong | 7e9bd6c | 2016-10-24 17:30:25 -0700 | [diff] [blame] | 159 | |
| 160 | if __name__ == '__main__': |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 161 | unittest.main(verbosity=2) |