Phil Dubach | ec19a57 | 2009-08-21 15:20:13 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 2 | |
| 3 | # Copyright (C) 2009 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """Module for generating CTS test descriptions and test plans.""" |
| 18 | |
| 19 | import glob |
| 20 | import os |
| 21 | import re |
Unsuk Jung | 2a692d1 | 2013-09-29 20:57:32 -0700 | [diff] [blame] | 22 | import shutil |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 23 | import subprocess |
| 24 | import sys |
| 25 | import xml.dom.minidom as dom |
| 26 | from cts import tools |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 27 | from multiprocessing import Pool |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 28 | |
| 29 | def GetSubDirectories(root): |
| 30 | """Return all directories under the given root directory.""" |
| 31 | return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))] |
| 32 | |
| 33 | |
| 34 | def GetMakeFileVars(makefile_path): |
| 35 | """Extracts variable definitions from the given make file. |
| 36 | |
| 37 | Args: |
| 38 | makefile_path: Path to the make file. |
| 39 | |
| 40 | Returns: |
| 41 | A dictionary mapping variable names to their assigned value. |
| 42 | """ |
| 43 | result = {} |
| 44 | pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL) |
| 45 | stream = open(makefile_path, 'r') |
| 46 | content = stream.read() |
| 47 | for match in pattern.finditer(content): |
| 48 | result[match.group(1)] = match.group(2) |
| 49 | stream.close() |
| 50 | return result |
| 51 | |
| 52 | |
| 53 | class CtsBuilder(object): |
| 54 | """Main class for generating test descriptions and test plans.""" |
| 55 | |
| 56 | def __init__(self, argv): |
| 57 | """Initialize the CtsBuilder from command line arguments.""" |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 58 | if len(argv) != 6: |
| 59 | print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0] |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 60 | print '' |
| 61 | print 'testRoot: Directory under which to search for CTS tests.' |
| 62 | print 'ctsOutputDir: Directory in which the CTS repository should be created.' |
| 63 | print 'tempDir: Directory to use for storing temporary files.' |
| 64 | print 'androidRootDir: Root directory of the Android source tree.' |
| 65 | print 'docletPath: Class path where the DescriptionGenerator doclet can be found.' |
| 66 | sys.exit(1) |
| 67 | self.test_root = sys.argv[1] |
| 68 | self.out_dir = sys.argv[2] |
| 69 | self.temp_dir = sys.argv[3] |
| 70 | self.android_root = sys.argv[4] |
| 71 | self.doclet_path = sys.argv[5] |
| 72 | |
| 73 | self.test_repository = os.path.join(self.out_dir, 'repository/testcases') |
| 74 | self.plan_repository = os.path.join(self.out_dir, 'repository/plans') |
Unsuk Jung | 08d97f9 | 2013-09-29 22:40:22 -0700 | [diff] [blame] | 75 | self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans') |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 76 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 77 | def GenerateTestDescriptions(self): |
| 78 | """Generate test descriptions for all packages.""" |
Brian Muramatsu | 5df641c | 2011-12-28 15:46:57 -0800 | [diff] [blame] | 79 | pool = Pool(processes=2) |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 80 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 81 | # individually generate descriptions not following conventions |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 82 | pool.apply_async(GenerateSignatureCheckDescription, [self.test_repository]) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 83 | |
| 84 | # generate test descriptions for android tests |
Brian Muramatsu | bdcfc7c | 2011-12-01 14:32:02 -0800 | [diff] [blame] | 85 | results = [] |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 86 | pool.close() |
| 87 | pool.join() |
Brian Muramatsu | bdcfc7c | 2011-12-01 14:32:02 -0800 | [diff] [blame] | 88 | return sum(map(lambda result: result.get(), results)) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 89 | |
| 90 | def __WritePlan(self, plan, plan_name): |
| 91 | print 'Generating test plan %s' % plan_name |
| 92 | plan.Write(os.path.join(self.plan_repository, plan_name + '.xml')) |
| 93 | |
| 94 | def GenerateTestPlans(self): |
| 95 | """Generate default test plans.""" |
| 96 | # TODO: Instead of hard-coding the plans here, use a configuration file, |
| 97 | # such as test_defs.xml |
| 98 | packages = [] |
Phil Dubach | 8dcedfe | 2009-08-20 16:10:39 -0700 | [diff] [blame] | 99 | descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml'))) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 100 | for description in descriptions: |
| 101 | doc = tools.XmlFile(description) |
| 102 | packages.append(doc.GetAttr('TestPackage', 'appPackageName')) |
Keun young Park | 965d191 | 2013-02-08 11:37:16 -0800 | [diff] [blame] | 103 | # sort the list to give the same sequence based on name |
| 104 | packages.sort() |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 105 | |
| 106 | plan = tools.TestPlan(packages) |
Tsu Chiang Chuang | d4cce3b | 2011-05-12 12:19:54 -0700 | [diff] [blame] | 107 | plan.Exclude('android\.performance.*') |
Brian Muramatsu | 87ff4df | 2011-12-16 11:41:17 -0800 | [diff] [blame] | 108 | self.__WritePlan(plan, 'CTS') |
Brian Muramatsu | 89ae08c | 2012-01-03 10:55:02 -0800 | [diff] [blame] | 109 | self.__WritePlan(plan, 'CTS-TF') |
Tsu Chiang Chuang | d4cce3b | 2011-05-12 12:19:54 -0700 | [diff] [blame] | 110 | |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 111 | plan = tools.TestPlan(packages) |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 112 | plan.Exclude('android\.performance.*') |
Guru Nagarajan | 55f10db | 2013-06-03 13:50:37 -0700 | [diff] [blame] | 113 | plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*') |
| 114 | # Test plan to not include media streaming tests |
| 115 | self.__WritePlan(plan, 'CTS-No-Media-Stream') |
| 116 | |
| 117 | plan = tools.TestPlan(packages) |
| 118 | plan.Exclude('android\.performance.*') |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 119 | self.__WritePlan(plan, 'SDK') |
| 120 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 121 | plan.Exclude(r'android\.tests\.sigtest') |
| 122 | plan.Exclude(r'android\.core.*') |
| 123 | self.__WritePlan(plan, 'Android') |
| 124 | |
| 125 | plan = tools.TestPlan(packages) |
| 126 | plan.Include(r'android\.core\.tests.*') |
Tsu Chiang Chuang | cf7ceab | 2012-10-24 16:48:03 -0700 | [diff] [blame] | 127 | plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*') |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 128 | self.__WritePlan(plan, 'Java') |
| 129 | |
Tsu Chiang Chuang | cf7ceab | 2012-10-24 16:48:03 -0700 | [diff] [blame] | 130 | # TODO: remove this once the tests are fixed and merged into Java plan above. |
| 131 | plan = tools.TestPlan(packages) |
| 132 | plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*') |
| 133 | self.__WritePlan(plan, 'Harmony') |
| 134 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 135 | plan = tools.TestPlan(packages) |
Tsu Chiang Chuang | 9a223d7 | 2011-04-27 17:19:46 -0700 | [diff] [blame] | 136 | plan.Include(r'android\.core\.vm-tests-tf') |
| 137 | self.__WritePlan(plan, 'VM-TF') |
| 138 | |
| 139 | plan = tools.TestPlan(packages) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 140 | plan.Include(r'android\.tests\.sigtest') |
| 141 | self.__WritePlan(plan, 'Signature') |
| 142 | |
| 143 | plan = tools.TestPlan(packages) |
Phil Dubach | 6068048 | 2009-08-19 10:13:23 -0700 | [diff] [blame] | 144 | plan.Include(r'android\.tests\.appsecurity') |
| 145 | self.__WritePlan(plan, 'AppSecurity') |
| 146 | |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 147 | # hard-coded white list for PDK plan |
| 148 | plan.Exclude('.*') |
keunyoung | c767cba | 2013-04-03 14:50:58 -0700 | [diff] [blame] | 149 | plan.Include('android\.aadb') |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 150 | plan.Include('android\.bluetooth') |
| 151 | plan.Include('android\.graphics.*') |
| 152 | plan.Include('android\.hardware') |
keunyoung | de51946 | 2013-03-21 12:06:42 -0700 | [diff] [blame] | 153 | plan.Include('android\.media') |
| 154 | plan.Exclude('android\.mediastress') |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 155 | plan.Include('android\.net') |
| 156 | plan.Include('android\.opengl.*') |
| 157 | plan.Include('android\.renderscript') |
| 158 | plan.Include('android\.telephony') |
| 159 | plan.Include('android\.nativemedia.*') |
Stuart Scott | a132af6 | 2013-11-07 10:30:32 -0800 | [diff] [blame] | 160 | plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these? |
Keun young Park | ff19417 | 2012-08-14 17:56:11 -0700 | [diff] [blame] | 161 | self.__WritePlan(plan, 'PDK') |
| 162 | |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 163 | flaky_tests = BuildCtsFlakyTestList() |
| 164 | |
| 165 | # CTS Stable plan |
| 166 | plan = tools.TestPlan(packages) |
Unsuk Jung | 7c7832b | 2014-09-06 20:31:08 -0700 | [diff] [blame] | 167 | plan.Exclude(r'com\.android\.cts\.browserbench') |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 168 | for package, test_list in flaky_tests.iteritems(): |
| 169 | plan.ExcludeTests(package, test_list) |
| 170 | self.__WritePlan(plan, 'CTS-stable') |
| 171 | |
| 172 | # CTS Flaky plan - inversion of CTS Stable |
| 173 | plan = tools.TestPlan(packages) |
| 174 | plan.Exclude('.*') |
Unsuk Jung | 7c7832b | 2014-09-06 20:31:08 -0700 | [diff] [blame] | 175 | plan.Include(r'com\.android\.cts\.browserbench') |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 176 | for package, test_list in flaky_tests.iteritems(): |
| 177 | plan.Include(package) |
| 178 | plan.IncludeTests(package, test_list) |
| 179 | self.__WritePlan(plan, 'CTS-flaky') |
| 180 | |
| 181 | |
| 182 | def BuildCtsFlakyTestList(): |
| 183 | """ Construct a defaultdict that maps package name to a list of tests |
| 184 | that are known to be flaky. """ |
| 185 | return { |
| 186 | 'android.app' : [ |
| 187 | 'cts.ActivityManagerTest#testIsRunningInTestHarness', |
| 188 | 'cts.AlertDialogTest#testAlertDialogCancelable', |
| 189 | 'cts.ExpandableListActivityTest#testCallback',], |
Unsuk Jung | 1d72924 | 2014-09-08 09:23:03 -0700 | [diff] [blame^] | 190 | 'android.dpi' : [ |
| 191 | 'cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 192 | 'android.hardware' : [ |
| 193 | 'camera2.cts.CameraDeviceTest#testCameraDeviceRepeatingRequest', |
| 194 | 'camera2.cts.ImageReaderTest#testImageReaderFromCameraJpeg', |
| 195 | 'cts.CameraTest#testImmediateZoom', |
| 196 | 'cts.CameraTest#testPreviewCallback', |
| 197 | 'cts.CameraTest#testSmoothZoom', |
| 198 | 'cts.CameraTest#testVideoSnapshot', |
| 199 | 'cts.CameraGLTest#testCameraToSurfaceTextureMetadata', |
| 200 | 'cts.CameraGLTest#testSetPreviewTextureBothCallbacks', |
| 201 | 'cts.CameraGLTest#testSetPreviewTexturePreviewCallback',], |
| 202 | 'android.media' : [ |
| 203 | 'cts.DecoderTest#testCodecResetsH264WithSurface', |
| 204 | 'cts.StreamingMediaPlayerTest#testHLS',], |
| 205 | 'android.mediastress' : [ |
| 206 | 'cts.NativeMediaTest#test480pPlay',], |
| 207 | 'android.net' : [ |
| 208 | 'cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri', |
| 209 | 'cts.DnsTest#testDnsWorks', |
| 210 | 'cts.SSLCertificateSocketFactoryTest#testCreateSocket', |
| 211 | 'cts.SSLCertificateSocketFactoryTest#test_createSocket_bind', |
| 212 | 'cts.SSLCertificateSocketFactoryTest#test_createSocket_simple', |
| 213 | 'cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping', |
| 214 | 'cts.TrafficStatsTest#testTrafficStatsForLocalhost', |
| 215 | 'wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly', |
| 216 | 'wifi.cts.ScanResultTest#testAndroidTestCaseSetupProperly', |
| 217 | 'wifi.cts.ScanResultTest#testScanResultTimeStamp',], |
Unsuk Jung | 1d72924 | 2014-09-08 09:23:03 -0700 | [diff] [blame^] | 218 | 'android.os' : [ |
| 219 | 'cts.BuildVersionTest#testReleaseVersion', |
| 220 | 'cts.BuildTest#testIsSecureUserBuild',], |
Nicholas Sauer | 471c601 | 2014-05-28 15:28:28 -0700 | [diff] [blame] | 221 | 'android.security' : [ |
| 222 | 'cts.BannedFilesTest#testNoSu', |
| 223 | 'cts.BannedFilesTest#testNoSuInPath', |
| 224 | 'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports', |
| 225 | 'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts', |
| 226 | 'cts.PackageSignatureTest#testPackageSignatures',], |
| 227 | 'android.webkit' : [ |
| 228 | 'cts.WebViewClientTest#testDoUpdateVisitedHistory', |
| 229 | 'cts.WebViewClientTest#testLoadPage', |
| 230 | 'cts.WebViewClientTest#testOnFormResubmission', |
| 231 | 'cts.WebViewClientTest#testOnReceivedError', |
| 232 | 'cts.WebViewClientTest#testOnReceivedHttpAuthRequest', |
| 233 | 'cts.WebViewClientTest#testOnScaleChanged', |
| 234 | 'cts.WebViewClientTest#testOnUnhandledKeyEvent', |
| 235 | 'cts.WebViewTest#testSetInitialScale',]} |
Unsuk Jung | 2a692d1 | 2013-09-29 20:57:32 -0700 | [diff] [blame] | 236 | |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 237 | def LogGenerateDescription(name): |
| 238 | print 'Generating test description for package %s' % name |
| 239 | |
| 240 | def GenerateSignatureCheckDescription(test_repository): |
| 241 | """Generate the test description for the signature check.""" |
| 242 | LogGenerateDescription('android.tests.sigtest') |
| 243 | package = tools.TestPackage('SignatureTest', 'android.tests.sigtest') |
| 244 | package.AddAttribute('appNameSpace', 'android.tests.sigtest') |
| 245 | package.AddAttribute('signatureCheck', 'true') |
| 246 | package.AddAttribute('runner', '.InstrumentationRunner') |
Stuart Scott | c09a2e0 | 2013-11-15 13:03:29 -0800 | [diff] [blame] | 247 | package.AddTest('android.tests.sigtest.SignatureTest.testSignature') |
Brian Muramatsu | 9157e0a | 2011-04-05 18:06:15 -0700 | [diff] [blame] | 248 | description = open(os.path.join(test_repository, 'SignatureTest.xml'), 'w') |
| 249 | package.WriteDescription(description) |
| 250 | description.close() |
| 251 | |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 252 | if __name__ == '__main__': |
| 253 | builder = CtsBuilder(sys.argv) |
Keun young Park | d93d0d1 | 2013-01-10 14:11:35 -0800 | [diff] [blame] | 254 | result = builder.GenerateTestDescriptions() |
| 255 | if result != 0: |
| 256 | sys.exit(result) |
Phil Dubach | 0d6ef06 | 2009-08-12 18:13:16 -0700 | [diff] [blame] | 257 | builder.GenerateTestPlans() |
Keun young Park | 3769d33 | 2012-08-29 10:16:37 -0700 | [diff] [blame] | 258 | |