Phil Dubach | ec19a57 | 2009-08-21 15:20:13 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 2 | |
| 3 | # Copyright (C) 2009 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """Module for generating CTS test descriptions and test plans.""" |
| 18 | |
| 19 | import glob |
| 20 | import os |
| 21 | import re |
Unsuk Jung | 2a692d1 | 2013-09-29 20:57:32 -0700 | [diff] [blame] | 22 | import shutil |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 23 | import subprocess |
| 24 | import sys |
| 25 | import xml.dom.minidom as dom |
| 26 | from cts import tools |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 27 | from multiprocessing import Pool |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 28 | |
| 29 | def GetSubDirectories(root): |
| 30 | """Return all directories under the given root directory.""" |
| 31 | return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))] |
| 32 | |
Jarkko Pöyry | 1ab85bf | 2015-04-24 14:43:42 -0700 | [diff] [blame] | 33 | def ReadFileLines(filePath): |
| 34 | """Reads a file and returns its contents as a line list.""" |
| 35 | f = open(filePath, 'r'); |
| 36 | lines = [line.strip() for line in f.readlines()] |
| 37 | f.close() |
| 38 | return lines |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 39 | |
| 40 | def GetMakeFileVars(makefile_path): |
| 41 | """Extracts variable definitions from the given make file. |
| 42 | |
| 43 | Args: |
| 44 | makefile_path: Path to the make file. |
| 45 | |
| 46 | Returns: |
| 47 | A dictionary mapping variable names to their assigned value. |
| 48 | """ |
| 49 | result = {} |
| 50 | pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL) |
| 51 | stream = open(makefile_path, 'r') |
| 52 | content = stream.read() |
| 53 | for match in pattern.finditer(content): |
| 54 | result[match.group(1)] = match.group(2) |
| 55 | stream.close() |
| 56 | return result |
| 57 | |
| 58 | |
| 59 | class CtsBuilder(object): |
| 60 | """Main class for generating test descriptions and test plans.""" |
| 61 | |
| 62 | def __init__(self, argv): |
| 63 | """Initialize the CtsBuilder from command line arguments.""" |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 64 | if len(argv) != 6: |
| 65 | print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0] |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 66 | print '' |
| 67 | print 'testRoot: Directory under which to search for CTS tests.' |
| 68 | print 'ctsOutputDir: Directory in which the CTS repository should be created.' |
| 69 | print 'tempDir: Directory to use for storing temporary files.' |
| 70 | print 'androidRootDir: Root directory of the Android source tree.' |
| 71 | print 'docletPath: Class path where the DescriptionGenerator doclet can be found.' |
| 72 | sys.exit(1) |
| 73 | self.test_root = sys.argv[1] |
| 74 | self.out_dir = sys.argv[2] |
| 75 | self.temp_dir = sys.argv[3] |
| 76 | self.android_root = sys.argv[4] |
| 77 | self.doclet_path = sys.argv[5] |
| 78 | |
| 79 | self.test_repository = os.path.join(self.out_dir, 'repository/testcases') |
| 80 | self.plan_repository = os.path.join(self.out_dir, 'repository/plans') |
Unsuk Jung | 08d97f9 | 2013-09-29 22:40:22 -0700 | [diff] [blame] | 81 | self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans') |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 82 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 83 | def GenerateTestDescriptions(self): |
| 84 | """Generate test descriptions for all packages.""" |
Brian Muramatsu | 5df641c | 2011-12-28 15:46:57 -0800 | [diff] [blame] | 85 | pool = Pool(processes=2) |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 86 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 87 | # generate test descriptions for android tests |
Brian Muramatsu | bdcfc7c | 2011-12-01 14:32:02 -0800 | [diff] [blame] | 88 | results = [] |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 89 | pool.close() |
| 90 | pool.join() |
Brian Muramatsu | bdcfc7c | 2011-12-01 14:32:02 -0800 | [diff] [blame] | 91 | return sum(map(lambda result: result.get(), results)) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 92 | |
| 93 | def __WritePlan(self, plan, plan_name): |
| 94 | print 'Generating test plan %s' % plan_name |
| 95 | plan.Write(os.path.join(self.plan_repository, plan_name + '.xml')) |
| 96 | |
| 97 | def GenerateTestPlans(self): |
| 98 | """Generate default test plans.""" |
| 99 | # TODO: Instead of hard-coding the plans here, use a configuration file, |
| 100 | # such as test_defs.xml |
| 101 | packages = [] |
Phil Dubach | 8dcedfe | 2009-08-20 16:10:39 -0700 | [diff] [blame] | 102 | descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml'))) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 103 | for description in descriptions: |
| 104 | doc = tools.XmlFile(description) |
| 105 | packages.append(doc.GetAttr('TestPackage', 'appPackageName')) |
Keun young Park | 965d191 | 2013-02-08 11:37:16 -0800 | [diff] [blame] | 106 | # sort the list to give the same sequence based on name |
| 107 | packages.sort() |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 108 | |
| 109 | plan = tools.TestPlan(packages) |
Tsu Chiang Chuang | d4cce3b | 2011-05-12 12:19:54 -0700 | [diff] [blame] | 110 | plan.Exclude('android\.performance.*') |
Brian Muramatsu | 87ff4df | 2011-12-16 11:41:17 -0800 | [diff] [blame] | 111 | self.__WritePlan(plan, 'CTS') |
Brian Muramatsu | 89ae08c | 2012-01-03 10:55:02 -0800 | [diff] [blame] | 112 | self.__WritePlan(plan, 'CTS-TF') |
Tsu Chiang Chuang | d4cce3b | 2011-05-12 12:19:54 -0700 | [diff] [blame] | 113 | |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 114 | plan = tools.TestPlan(packages) |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 115 | plan.Exclude('android\.performance.*') |
Guru Nagarajan | 55f10db | 2013-06-03 13:50:37 -0700 | [diff] [blame] | 116 | plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*') |
| 117 | # Test plan to not include media streaming tests |
| 118 | self.__WritePlan(plan, 'CTS-No-Media-Stream') |
| 119 | |
| 120 | plan = tools.TestPlan(packages) |
| 121 | plan.Exclude('android\.performance.*') |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 122 | self.__WritePlan(plan, 'SDK') |
| 123 | |
Stuart Scott | f67280f | 2014-09-30 14:07:30 -0700 | [diff] [blame] | 124 | plan.Exclude(r'android\.signature') |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 125 | plan.Exclude(r'android\.core.*') |
| 126 | self.__WritePlan(plan, 'Android') |
| 127 | |
| 128 | plan = tools.TestPlan(packages) |
| 129 | plan.Include(r'android\.core\.tests.*') |
Tsu Chiang Chuang | cf7ceab | 2012-10-24 16:48:03 -0700 | [diff] [blame] | 130 | plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*') |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 131 | self.__WritePlan(plan, 'Java') |
| 132 | |
Tsu Chiang Chuang | cf7ceab | 2012-10-24 16:48:03 -0700 | [diff] [blame] | 133 | # TODO: remove this once the tests are fixed and merged into Java plan above. |
| 134 | plan = tools.TestPlan(packages) |
| 135 | plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*') |
| 136 | self.__WritePlan(plan, 'Harmony') |
| 137 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 138 | plan = tools.TestPlan(packages) |
Tsu Chiang Chuang | 9a223d7 | 2011-04-27 17:19:46 -0700 | [diff] [blame] | 139 | plan.Include(r'android\.core\.vm-tests-tf') |
| 140 | self.__WritePlan(plan, 'VM-TF') |
| 141 | |
| 142 | plan = tools.TestPlan(packages) |
Phil Dubach | 6068048 | 2009-08-19 10:13:23 -0700 | [diff] [blame] | 143 | plan.Include(r'android\.tests\.appsecurity') |
| 144 | self.__WritePlan(plan, 'AppSecurity') |
| 145 | |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 146 | # hard-coded white list for PDK plan |
| 147 | plan.Exclude('.*') |
keunyoung | c767cba | 2013-04-03 14:50:58 -0700 | [diff] [blame] | 148 | plan.Include('android\.aadb') |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 149 | plan.Include('android\.bluetooth') |
| 150 | plan.Include('android\.graphics.*') |
| 151 | plan.Include('android\.hardware') |
keunyoung | de51946 | 2013-03-21 12:06:42 -0700 | [diff] [blame] | 152 | plan.Include('android\.media') |
| 153 | plan.Exclude('android\.mediastress') |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 154 | plan.Include('android\.net') |
| 155 | plan.Include('android\.opengl.*') |
| 156 | plan.Include('android\.renderscript') |
| 157 | plan.Include('android\.telephony') |
| 158 | plan.Include('android\.nativemedia.*') |
Stuart Scott | a132af6 | 2013-11-07 10:30:32 -0800 | [diff] [blame] | 159 | plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these? |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 160 | self.__WritePlan(plan, 'PDK') |
| 161 | |
Nicholas Sauer | 72632f9 | 2015-05-04 14:45:50 -0700 | [diff] [blame] | 162 | temporarily_known_failure_tests = BuildCtsTemporarilyKnownFailureList(); |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 163 | flaky_tests = BuildCtsFlakyTestList() |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 164 | releasekey_tests = BuildListForReleaseBuildTest() |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 165 | |
| 166 | # CTS Stable plan |
| 167 | plan = tools.TestPlan(packages) |
Unsuk Jung | 7c7832b | 2014-09-06 20:31:08 -0700 | [diff] [blame] | 168 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 169 | for package, test_list in flaky_tests.iteritems(): |
| 170 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 171 | for package, test_list in releasekey_tests.iteritems(): |
| 172 | plan.ExcludeTests(package, test_list) |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 173 | self.__WritePlan(plan, 'CTS-stable') |
| 174 | |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 175 | # CTS Flaky plan - list of tests known to be flaky in lab environment |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 176 | plan = tools.TestPlan(packages) |
| 177 | plan.Exclude('.*') |
Unsuk Jung | 7c7832b | 2014-09-06 20:31:08 -0700 | [diff] [blame] | 178 | plan.Include(r'com\.android\.cts\.browserbench') |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 179 | for package, test_list in flaky_tests.iteritems(): |
Unsuk Jung | b5153c4 | 2014-09-19 02:12:50 -0700 | [diff] [blame] | 180 | plan.Include(package+'$') |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 181 | plan.IncludeTests(package, test_list) |
| 182 | self.__WritePlan(plan, 'CTS-flaky') |
| 183 | |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 184 | small_tests = BuildAospSmallSizeTestList() |
| 185 | medium_tests = BuildAospMediumSizeTestList() |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 186 | new_test_packages = BuildCtsVettedNewPackagesList() |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 187 | |
| 188 | # CTS - sub plan for public, small size tests |
| 189 | plan = tools.TestPlan(packages) |
| 190 | plan.Exclude('.*') |
| 191 | for package, test_list in small_tests.iteritems(): |
| 192 | plan.Include(package+'$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 193 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 194 | for package, test_list in flaky_tests.iteritems(): |
| 195 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 196 | for package, test_list in releasekey_tests.iteritems(): |
| 197 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 198 | self.__WritePlan(plan, 'CTS-kitkat-small') |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 199 | self.__WritePlan(plan, 'CTS-public-small') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 200 | |
| 201 | # CTS - sub plan for public, medium size tests |
| 202 | plan = tools.TestPlan(packages) |
| 203 | plan.Exclude('.*') |
| 204 | for package, test_list in medium_tests.iteritems(): |
| 205 | plan.Include(package+'$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 206 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 207 | for package, test_list in flaky_tests.iteritems(): |
| 208 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 209 | for package, test_list in releasekey_tests.iteritems(): |
| 210 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 211 | self.__WritePlan(plan, 'CTS-kitkat-medium') |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 212 | self.__WritePlan(plan, 'CTS-public-medium') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 213 | |
| 214 | # CTS - sub plan for hardware tests which is public, large |
| 215 | plan = tools.TestPlan(packages) |
| 216 | plan.Exclude('.*') |
| 217 | plan.Include(r'android\.hardware$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 218 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 219 | for package, test_list in flaky_tests.iteritems(): |
| 220 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 221 | for package, test_list in releasekey_tests.iteritems(): |
| 222 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 223 | self.__WritePlan(plan, 'CTS-hardware') |
| 224 | |
| 225 | # CTS - sub plan for media tests which is public, large |
| 226 | plan = tools.TestPlan(packages) |
| 227 | plan.Exclude('.*') |
| 228 | plan.Include(r'android\.media$') |
Unsuk Jung | fd4955a | 2014-10-21 18:11:48 -0700 | [diff] [blame] | 229 | plan.Include(r'android\.view$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 230 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 231 | for package, test_list in flaky_tests.iteritems(): |
| 232 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 233 | for package, test_list in releasekey_tests.iteritems(): |
| 234 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 235 | self.__WritePlan(plan, 'CTS-media') |
| 236 | |
| 237 | # CTS - sub plan for mediastress tests which is public, large |
| 238 | plan = tools.TestPlan(packages) |
| 239 | plan.Exclude('.*') |
| 240 | plan.Include(r'android\.mediastress$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 241 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 242 | for package, test_list in flaky_tests.iteritems(): |
| 243 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 244 | for package, test_list in releasekey_tests.iteritems(): |
| 245 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 246 | self.__WritePlan(plan, 'CTS-mediastress') |
| 247 | |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 248 | # CTS - sub plan for new tests that is vetted for L launch |
| 249 | plan = tools.TestPlan(packages) |
| 250 | plan.Exclude('.*') |
| 251 | for package, test_list in new_test_packages.iteritems(): |
| 252 | plan.Include(package+'$') |
| 253 | plan.Exclude(r'com\.android\.cts\.browserbench') |
| 254 | for package, test_list in flaky_tests.iteritems(): |
| 255 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 256 | for package, test_list in releasekey_tests.iteritems(): |
| 257 | plan.ExcludeTests(package, test_list) |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 258 | self.__WritePlan(plan, 'CTS-l-tests') |
| 259 | |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 260 | # CTS - sub plan for tests in drawelement packages |
| 261 | plan = tools.TestPlan(packages) |
| 262 | plan.Exclude('.*') |
| 263 | plan.Include(r'com\.drawelements\.') |
Pyry Haulos | e793db1 | 2015-07-27 11:09:55 -0700 | [diff] [blame] | 264 | self.__WritePlan(plan, 'CTS-DEQP') |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 265 | |
| 266 | # CTS - sub plan for new test packages added for staging |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 267 | plan = tools.TestPlan(packages) |
| 268 | for package, test_list in small_tests.iteritems(): |
| 269 | plan.Exclude(package+'$') |
| 270 | for package, test_list in medium_tests.iteritems(): |
| 271 | plan.Exclude(package+'$') |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 272 | for package, tests_list in new_test_packages.iteritems(): |
| 273 | plan.Exclude(package+'$') |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 274 | plan.Exclude(r'com\.drawelements\.') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 275 | plan.Exclude(r'android\.hardware$') |
| 276 | plan.Exclude(r'android\.media$') |
Unsuk Jung | fd4955a | 2014-10-21 18:11:48 -0700 | [diff] [blame] | 277 | plan.Exclude(r'android\.view$') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 278 | plan.Exclude(r'android\.mediastress$') |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 279 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 280 | for package, test_list in flaky_tests.iteritems(): |
| 281 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 282 | for package, test_list in releasekey_tests.iteritems(): |
| 283 | plan.ExcludeTests(package, test_list) |
Unsuk Jung | 0a99c97 | 2015-09-02 15:28:15 -0700 | [diff] [blame] | 284 | self.__WritePlan(plan, 'CTS-m-tests') |
| 285 | |
| 286 | |
| 287 | # CTS - sub plan for new test packages added for staging |
| 288 | plan = tools.TestPlan(packages) |
| 289 | plan.Exclude('.*') |
| 290 | for package, test_list in temporarily_known_failure_tests.iteritems(): |
| 291 | plan.Include(package+'$') |
| 292 | plan.IncludeTests(package, test_list) |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 293 | self.__WritePlan(plan, 'CTS-staging') |
| 294 | |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 295 | plan = tools.TestPlan(packages) |
| 296 | plan.Exclude('.*') |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 297 | self.__WritePlan(plan, 'CTS-webview') |
| 298 | |
| 299 | |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 300 | def BuildAospMediumSizeTestList(): |
| 301 | """ Construct a defaultdic that lists package names of medium tests |
| 302 | already published to aosp. """ |
| 303 | return { |
| 304 | 'android.app' : [], |
| 305 | 'android.core.tests.libcore.package.libcore' : [], |
| 306 | 'android.core.tests.libcore.package.org' : [], |
| 307 | 'android.core.vm-tests-tf' : [], |
| 308 | 'android.dpi' : [], |
| 309 | 'android.host.security' : [], |
| 310 | 'android.net' : [], |
| 311 | 'android.os' : [], |
Unsuk Jung | 2e4e929 | 2014-10-09 02:20:44 -0700 | [diff] [blame] | 312 | 'android.permission2' : [], |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 313 | 'android.security' : [], |
| 314 | 'android.telephony' : [], |
| 315 | 'android.webkit' : [], |
| 316 | 'android.widget' : [], |
| 317 | 'com.android.cts.browserbench' : []} |
| 318 | |
| 319 | def BuildAospSmallSizeTestList(): |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 320 | """ Construct a default dict that lists packages names of small tests |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 321 | already published to aosp. """ |
| 322 | return { |
| 323 | 'android.aadb' : [], |
| 324 | 'android.acceleration' : [], |
| 325 | 'android.accessibility' : [], |
| 326 | 'android.accessibilityservice' : [], |
| 327 | 'android.accounts' : [], |
| 328 | 'android.admin' : [], |
| 329 | 'android.animation' : [], |
| 330 | 'android.bionic' : [], |
| 331 | 'android.bluetooth' : [], |
| 332 | 'android.calendarcommon' : [], |
| 333 | 'android.content' : [], |
| 334 | 'android.core.tests.libcore.package.com' : [], |
| 335 | 'android.core.tests.libcore.package.conscrypt' : [], |
| 336 | 'android.core.tests.libcore.package.dalvik' : [], |
| 337 | 'android.core.tests.libcore.package.sun' : [], |
| 338 | 'android.core.tests.libcore.package.tests' : [], |
| 339 | 'android.database' : [], |
| 340 | 'android.dreams' : [], |
| 341 | 'android.drm' : [], |
| 342 | 'android.effect' : [], |
| 343 | 'android.gesture' : [], |
| 344 | 'android.graphics' : [], |
| 345 | 'android.graphics2' : [], |
| 346 | 'android.jni' : [], |
| 347 | 'android.keystore' : [], |
| 348 | 'android.location' : [], |
| 349 | 'android.nativemedia.sl' : [], |
| 350 | 'android.nativemedia.xa' : [], |
| 351 | 'android.nativeopengl' : [], |
| 352 | 'android.ndef' : [], |
| 353 | 'android.opengl' : [], |
| 354 | 'android.openglperf' : [], |
| 355 | 'android.permission' : [], |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 356 | 'android.preference' : [], |
| 357 | 'android.preference2' : [], |
| 358 | 'android.provider' : [], |
| 359 | 'android.renderscript' : [], |
| 360 | 'android.rscpp' : [], |
| 361 | 'android.rsg' : [], |
| 362 | 'android.sax' : [], |
Stuart Scott | f67280f | 2014-09-30 14:07:30 -0700 | [diff] [blame] | 363 | 'android.signature' : [], |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 364 | 'android.speech' : [], |
| 365 | 'android.tests.appsecurity' : [], |
| 366 | 'android.text' : [], |
| 367 | 'android.textureview' : [], |
| 368 | 'android.theme' : [], |
| 369 | 'android.usb' : [], |
| 370 | 'android.util' : [], |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 371 | 'com.android.cts.dram' : [], |
| 372 | 'com.android.cts.filesystemperf' : [], |
| 373 | 'com.android.cts.jank' : [], |
Guang Zhu | cf5c254 | 2015-03-09 23:13:27 -0700 | [diff] [blame] | 374 | 'com.android.cts.jank2' : [], |
Unsuk Jung | 8398ab8 | 2014-09-19 03:26:39 -0700 | [diff] [blame] | 375 | 'com.android.cts.opengl' : [], |
| 376 | 'com.android.cts.simplecpu' : [], |
| 377 | 'com.android.cts.ui' : [], |
| 378 | 'com.android.cts.uihost' : [], |
| 379 | 'com.android.cts.videoperf' : [], |
| 380 | 'zzz.android.monkey' : []} |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 381 | |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 382 | def BuildCtsVettedNewPackagesList(): |
| 383 | """ Construct a defaultdict that maps package names that is vetted for L. """ |
| 384 | return { |
| 385 | 'android.JobScheduler' : [], |
| 386 | 'android.core.tests.libcore.package.harmony_annotation' : [], |
| 387 | 'android.core.tests.libcore.package.harmony_beans' : [], |
| 388 | 'android.core.tests.libcore.package.harmony_java_io' : [], |
| 389 | 'android.core.tests.libcore.package.harmony_java_lang' : [], |
| 390 | 'android.core.tests.libcore.package.harmony_java_math' : [], |
| 391 | 'android.core.tests.libcore.package.harmony_java_net' : [], |
| 392 | 'android.core.tests.libcore.package.harmony_java_nio' : [], |
| 393 | 'android.core.tests.libcore.package.harmony_java_util' : [], |
| 394 | 'android.core.tests.libcore.package.harmony_java_text' : [], |
| 395 | 'android.core.tests.libcore.package.harmony_javax_security' : [], |
| 396 | 'android.core.tests.libcore.package.harmony_logging' : [], |
| 397 | 'android.core.tests.libcore.package.harmony_prefs' : [], |
| 398 | 'android.core.tests.libcore.package.harmony_sql' : [], |
| 399 | 'android.core.tests.libcore.package.jsr166' : [], |
| 400 | 'android.core.tests.libcore.package.okhttp' : [], |
| 401 | 'android.display' : [], |
| 402 | 'android.host.theme' : [], |
| 403 | 'android.jdwp' : [], |
| 404 | 'android.location2' : [], |
| 405 | 'android.print' : [], |
| 406 | 'android.renderscriptlegacy' : [], |
| 407 | 'android.signature' : [], |
| 408 | 'android.tv' : [], |
| 409 | 'android.uiautomation' : [], |
Bo Liu | c30bdbc | 2015-05-18 14:15:15 -0700 | [diff] [blame] | 410 | 'android.uirendering' : []} |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 411 | |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 412 | def BuildListForReleaseBuildTest(): |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 413 | """ Construct a defaultdict that maps package name to a list of tests |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 414 | that are expected to pass only when running against a user/release-key build. """ |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 415 | return { |
| 416 | 'android.app' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 417 | 'android.app.cts.ActivityManagerTest#testIsRunningInTestHarness',], |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 418 | 'android.dpi' : [ |
| 419 | 'android.dpi.cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',], |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 420 | 'android.host.security' : [ |
| 421 | 'android.cts.security.SELinuxHostTest#testAllEnforcing', |
| 422 | 'android.cts.security.SELinuxHostTest#testSuDomain',], |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 423 | 'android.os' : [ |
| 424 | 'android.os.cts.BuildVersionTest#testReleaseVersion', |
| 425 | 'android.os.cts.BuildTest#testIsSecureUserBuild',], |
| 426 | 'android.security' : [ |
| 427 | 'android.security.cts.BannedFilesTest#testNoSu', |
| 428 | 'android.security.cts.BannedFilesTest#testNoSuInPath', |
| 429 | 'android.security.cts.PackageSignatureTest#testPackageSignatures', |
| 430 | 'android.security.cts.SELinuxDomainTest#testSuDomain',], |
| 431 | '' : []} |
| 432 | |
| 433 | def BuildCtsFlakyTestList(): |
| 434 | """ Construct a defaultdict that maps package name to a list of tests |
| 435 | that flaky during dev cycle and cause other subsequent tests to fail. """ |
| 436 | return { |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 437 | 'android.hardware' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 438 | 'android.hardware.cts.CameraTest#testVideoSnapshot', |
| 439 | 'android.hardware.cts.CameraGLTest#testCameraToSurfaceTextureMetadata', |
| 440 | 'android.hardware.cts.CameraGLTest#testSetPreviewTextureBothCallbacks', |
| 441 | 'android.hardware.cts.CameraGLTest#testSetPreviewTexturePreviewCallback',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 442 | 'android.media' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 443 | 'android.media.cts.DecoderTest#testCodecResetsH264WithSurface', |
| 444 | 'android.media.cts.StreamingMediaPlayerTest#testHLS',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 445 | 'android.net' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 446 | 'android.net.cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri', |
| 447 | 'android.net.cts.DnsTest#testDnsWorks', |
| 448 | 'android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket', |
| 449 | 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind', |
| 450 | 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple', |
| 451 | 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping', |
| 452 | 'android.net.cts.TrafficStatsTest#testTrafficStatsForLocalhost', |
| 453 | 'android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 454 | 'android.security' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 455 | 'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports', |
Unsuk Jung | 1345d3d | 2015-04-24 12:14:59 -0700 | [diff] [blame] | 456 | 'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 457 | 'android.webkit' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 458 | 'android.webkit.cts.WebViewClientTest#testOnUnhandledKeyEvent',], |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 459 | 'com.android.cts.filesystemperf' : [ |
Nicholas Sauer | 42b1b61 | 2015-04-24 15:58:47 -0700 | [diff] [blame] | 460 | 'com.android.cts.filesystemperf.RandomRWTest#testRandomRead', |
| 461 | 'com.android.cts.filesystemperf.RandomRWTest#testRandomUpdate',], |
Guang Zhu | 80f667f | 2015-01-15 16:40:46 -0800 | [diff] [blame] | 462 | '' : []} |
Unsuk Jung | 2a692d1 | 2013-09-29 20:57:32 -0700 | [diff] [blame] | 463 | |
Nicholas Sauer | 72632f9 | 2015-05-04 14:45:50 -0700 | [diff] [blame] | 464 | def BuildCtsTemporarilyKnownFailureList(): |
| 465 | """ Construct a defaultdict that maps package name to a list of tests |
| 466 | that are known failures during dev cycle but expected to be fixed before launch """ |
| 467 | return { |
Unsuk Jung | 0a99c97 | 2015-09-02 15:28:15 -0700 | [diff] [blame] | 468 | 'android.alarmclock' : [ |
| 469 | 'android.alarmclock.cts.DismissAlarmTest#testAll', |
| 470 | 'android.alarmclock.cts.SetAlarmTest#testAll', |
| 471 | 'android.alarmclock.cts.SnoozeAlarmTest#testAll', |
| 472 | ], |
Alice Lee | eb39d69 | 2015-08-25 14:29:16 -0700 | [diff] [blame] | 473 | 'android.assist' : [ |
Cameron Neale | 51d0f6a | 2015-09-04 10:09:27 -0700 | [diff] [blame] | 474 | 'android.assist.cts.AssistantContentViewTest', |
Alice Lee | e4839c3 | 2015-09-04 15:11:35 -0700 | [diff] [blame] | 475 | 'android.assist.cts.ExtraAssistDataTest', |
Cameron Neale | 38de8fd | 2015-09-14 13:22:38 -0700 | [diff] [blame] | 476 | 'android.assist.cts.FocusChangeTest', |
Alice Lee | e4839c3 | 2015-09-04 15:11:35 -0700 | [diff] [blame] | 477 | 'android.assist.cts.LargeViewHierarchyTest', |
| 478 | 'android.assist.cts.ScreenshotTest', |
| 479 | 'android.assist.cts.TextViewTest', |
| 480 | 'android.assist.cts.WebViewTest', |
Alice Lee | eb39d69 | 2015-08-25 14:29:16 -0700 | [diff] [blame] | 481 | ], |
Unsuk Jung | 0a99c97 | 2015-09-02 15:28:15 -0700 | [diff] [blame] | 482 | 'android.calllog' : [ |
| 483 | 'android.calllog.cts.CallLogBackupTest#testSingleCallBackup', |
| 484 | ], |
| 485 | 'android.dumpsys' : [ |
| 486 | 'android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput', |
| 487 | 'android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats', |
| 488 | ], |
| 489 | 'android.telecom' : [ |
| 490 | 'android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect', |
| 491 | 'android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections', |
| 492 | ], |
| 493 | 'android.transition' : [ |
| 494 | 'android.transition.cts.ChangeScrollTest#testChangeScroll', |
| 495 | ], |
| 496 | 'android.voicesettings' : [ |
| 497 | 'android.voicesettings.cts.ZenModeTest#testAll', |
| 498 | ], |
Adrian Roos | d052b58 | 2015-08-27 19:06:39 -0700 | [diff] [blame] | 499 | 'com.android.cts.systemui' : [ |
| 500 | 'com.android.cts.systemui.LightStatusBarTests#testLightStatusBarIcons', |
| 501 | ], |
Unsuk Jung | 0a99c97 | 2015-09-02 15:28:15 -0700 | [diff] [blame] | 502 | 'com.android.cts.app.os' : [ |
| 503 | 'com.android.cts.app.os.OsHostTests#testNonExportedActivities', |
| 504 | ], |
Alex Chau | 8b5df14 | 2015-08-10 18:11:02 +0100 | [diff] [blame] | 505 | 'com.android.cts.devicepolicy' : [ |
| 506 | 'com.android.cts.devicepolicy.MixedDeviceOwnerTest#testPackageInstallUserRestrictions', |
| 507 | 'com.android.cts.devicepolicy.MixedProfileOwnerTest#testPackageInstallUserRestrictions', |
| 508 | ], |
Nicholas Sauer | 72632f9 | 2015-05-04 14:45:50 -0700 | [diff] [blame] | 509 | '' : []} |
| 510 | |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 511 | def LogGenerateDescription(name): |
| 512 | print 'Generating test description for package %s' % name |
| 513 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 514 | if __name__ == '__main__': |
| 515 | builder = CtsBuilder(sys.argv) |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 516 | result = builder.GenerateTestDescriptions() |
| 517 | if result != 0: |
| 518 | sys.exit(result) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 519 | builder.GenerateTestPlans() |
Keun young Park | 3769d33 | 2012-08-29 10:16:37 -0700 | [diff] [blame] | 520 | |