blob: a64153955f7c829309ae27ff82280d5e58a3f784 [file] [log] [blame]
Alan Viverettef5cb0e62017-11-17 15:15:23 -05001#!/usr/bin/python3
Alan Viverette95f6d362017-04-06 09:40:50 -04002
Anvesh Renikindi18680e62022-10-26 19:15:50 +00003"""Updates prebuilt libraries used by Android builds.
4
5For details on how to use this script, visit go/update-prebuilts.
6"""
7import os
8import sys
9import zipfile
10import re
Alan Viveretted4527e62017-05-11 15:03:25 -040011import argparse
12import subprocess
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +000013import six
Anvesh Renikindi18680e62022-10-26 19:15:50 +000014import shlex
15
16from urllib import request
17from shutil import which
18from distutils.version import LooseVersion
19from pathlib import Path
20from maven import MavenLibraryInfo, GMavenArtifact, maven_path_for_artifact
21from buildserver import fetch_and_extract, fetch_artifacts, fetch_artifact, extract_artifact, \
22 parse_build_id
23from utils import print_e, append, cp, mv, rm
24
Alan Viverette95f6d362017-04-06 09:40:50 -040025
Alan Viverette45837092017-05-12 14:50:53 -040026current_path = 'current'
Anvesh Renikindi18680e62022-10-26 19:15:50 +000027framework_sdk_target = 'sdk'
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -040028androidx_dir = os.path.join(current_path, 'androidx')
Anvesh Renikindi18680e62022-10-26 19:15:50 +000029androidx_owners = os.path.join(androidx_dir, 'OWNERS')
30java_plugins_bp_path = os.path.join(androidx_dir, 'JavaPlugins.bp')
31test_mapping_file = os.path.join(androidx_dir, 'TEST_MAPPING')
Nick Anthonybac2fca2022-11-03 18:56:57 +000032compose_test_mapping_file = os.path.join(androidx_dir, 'm2repository/androidx/compose/TEST_MAPPING')
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +000033gmaven_dir = os.path.join(current_path, 'gmaven')
Alan Viverette3e57a4a2017-08-11 15:49:47 -040034extras_dir = os.path.join(current_path, 'extras')
Alan Viverettec1c32b62017-12-20 09:40:36 -050035buildtools_dir = 'tools'
Jeff Gaston553a4372018-03-29 18:34:22 -040036jetifier_dir = os.path.join(buildtools_dir, 'jetifier', 'jetifier-standalone')
Anton Hansson4bcebac2022-04-07 17:23:18 +010037repo_root_dir = Path(sys.argv[0]).resolve().parents[3]
Anvesh Renikindi18680e62022-10-26 19:15:50 +000038extension_sdk_finalization_cmd = '%s -r "{readme}" -b {bug} -f {extension_version} {build_id}' % (
Anton Hansson4bcebac2022-04-07 17:23:18 +010039 "packages/modules/common/tools/finalize_sdk.py"
40)
Anvesh Renikindi18680e62022-10-26 19:15:50 +000041temp_dir = os.path.join(os.getcwd(), 'support_tmp')
Jeff Gastona68a8d42018-03-23 14:00:13 -040042os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))))
43git_dir = os.getcwd()
44
Alan Viverette0723aff2021-08-31 17:07:47 +000045# Leave map blank to automatically populate name and path:
46# - Name format is MAVEN.replaceAll(':','_')
47# - Path format is MAVEN.replaceAll(':','/').replaceAll('.','/')
Alan Viverette95f6d362017-04-06 09:40:50 -040048maven_to_make = {
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -040049 # AndroidX
Anvesh Renikindi18680e62022-10-26 19:15:50 +000050 'androidx.benchmark:benchmark-macro': {},
51 'androidx.benchmark:benchmark-macro-junit4': {},
52 'androidx.benchmark:benchmark-common': {},
53 'androidx.benchmark:benchmark-junit4': {},
54 'androidx.tracing:tracing': {},
55 'androidx.tracing:tracing-perfetto': {},
56 'androidx.tracing:tracing-perfetto-binary': {},
57 'androidx.tracing:tracing-perfetto-common': {},
58 'androidx.tracing:tracing-ktx': {},
59 'androidx.slice:slice-builders': {},
60 'androidx.slice:slice-core': {},
61 'androidx.slice:slice-view': {},
62 'androidx.remotecallback:remotecallback': {},
63 'androidx.remotecallback:remotecallback-processor': {
64 'host': True
65 },
66 'androidx.versionedparcelable:versionedparcelable': {},
67 'androidx.vectordrawable:vectordrawable-animated': {},
68 'androidx.activity:activity': {},
69 'androidx.activity:activity-ktx': {},
70 'androidx.annotation:annotation': {
71 'host_and_device': True,
72 'extra-static-libs': {
73 'androidx.annotation_annotation-jvm'
74 }
75 },
76 'androidx.annotation:annotation-jvm': {
77 'host_and_device': True
78 },
79 'androidx.annotation:annotation-experimental': {},
80 'androidx.asynclayoutinflater:asynclayoutinflater': {},
81 'androidx.collection:collection': {
82 'extra-static-libs': {
83 'androidx.collection_collection-jvm'
84 }
85 },
86 'androidx.collection:collection-ktx': {},
87 'androidx.collection:collection-jvm': {},
88 'androidx.concurrent:concurrent-futures': {},
89 'androidx.concurrent:concurrent-listenablefuture-callback': {},
90 'androidx.concurrent:concurrent-listenablefuture': {},
91 'androidx.core:core': {},
92 'androidx.core:core-animation': {},
Johannes Gallmann33504562022-11-17 09:32:02 +000093 'androidx.core:core-animation-testing': {},
Anvesh Renikindi18680e62022-10-26 19:15:50 +000094 'androidx.core:core-ktx': {},
95 'androidx.core.uwb:uwb': {},
96 'androidx.core.uwb:uwb-rxjava3': {},
97 'androidx.contentpaging:contentpaging': {},
98 'androidx.coordinatorlayout:coordinatorlayout': {},
99 'androidx.legacy:legacy-support-core-ui': {},
100 'androidx.legacy:legacy-support-core-utils': {},
101 'androidx.cursoradapter:cursoradapter': {},
102 'androidx.browser:browser': {},
103 'androidx.customview:customview': {},
104 'androidx.customview:customview-poolingcontainer': {},
105 'androidx.documentfile:documentfile': {},
106 'androidx.drawerlayout:drawerlayout': {},
107 'androidx.dynamicanimation:dynamicanimation': {},
108 'androidx.emoji:emoji': {},
109 'androidx.emoji:emoji-appcompat': {},
110 'androidx.emoji:emoji-bundled': {},
111 'androidx.emoji2:emoji2': {},
112 'androidx.emoji2:emoji2-views-helper': {},
113 'androidx.exifinterface:exifinterface': {},
114 'androidx.fragment:fragment': {},
115 'androidx.fragment:fragment-ktx': {},
116 'androidx.heifwriter:heifwriter': {},
117 'androidx.interpolator:interpolator': {},
118 'androidx.loader:loader': {},
119 'androidx.media:media': {},
120 'androidx.media2:media2-player': {},
121 'androidx.media2:media2-session': {},
122 'androidx.media2:media2-common': {},
123 'androidx.media2:media2-exoplayer': {},
124 'androidx.media2:media2-widget': {},
125 'androidx.navigation:navigation-common': {},
126 'androidx.navigation:navigation-common-ktx': {},
127 'androidx.navigation:navigation-fragment': {},
128 'androidx.navigation:navigation-fragment-ktx': {},
129 'androidx.navigation:navigation-runtime': {},
130 'androidx.navigation:navigation-runtime-ktx': {},
131 'androidx.navigation:navigation-ui': {},
132 'androidx.navigation:navigation-ui-ktx': {},
133 'androidx.percentlayout:percentlayout': {},
134 'androidx.print:print': {},
135 'androidx.recommendation:recommendation': {},
136 'androidx.recyclerview:recyclerview-selection': {},
137 'androidx.savedstate:savedstate': {},
138 'androidx.savedstate:savedstate-ktx': {},
139 'androidx.slidingpanelayout:slidingpanelayout': {},
140 'androidx.swiperefreshlayout:swiperefreshlayout': {},
141 'androidx.textclassifier:textclassifier': {},
142 'androidx.transition:transition': {},
143 'androidx.tvprovider:tvprovider': {},
144 'androidx.legacy:legacy-support-v13': {},
145 'androidx.legacy:legacy-preference-v14': {},
146 'androidx.leanback:leanback': {},
147 'androidx.leanback:leanback-grid': {},
148 'androidx.leanback:leanback-preference': {},
149 'androidx.legacy:legacy-support-v4': {},
150 'androidx.appcompat:appcompat': {},
151 'androidx.appcompat:appcompat-resources': {},
152 'androidx.cardview:cardview': {},
153 'androidx.gridlayout:gridlayout': {},
154 'androidx.mediarouter:mediarouter': {},
155 'androidx.palette:palette': {},
156 'androidx.preference:preference': {},
157 'androidx.recyclerview:recyclerview': {},
158 'androidx.vectordrawable:vectordrawable': {},
159 'androidx.viewpager:viewpager': {},
160 'androidx.viewpager2:viewpager2': {},
161 'androidx.wear:wear': {},
162 'androidx.wear:wear-ongoing': {},
163 'androidx.javascriptengine:javascriptengine': {},
164 'androidx.webkit:webkit': {},
165 'androidx.biometric:biometric': {},
166 'androidx.autofill:autofill': {},
167 'androidx.appsearch:appsearch': {},
168 'androidx.appsearch:appsearch-builtin-types': {},
169 'androidx.appsearch:appsearch-compiler': {
170 'name': 'androidx.appsearch_appsearch-compiler',
171 'host': True
172 },
173 'androidx.appsearch:appsearch-local-storage': {
174 'name': 'androidx.appsearch_appsearch_local_storage'
175 },
176 'androidx.appsearch:appsearch-platform-storage': {},
177 'androidx.car.app:app': {},
178 'androidx.car.app:app-automotive': {},
179 'androidx.car.app:app-testing': {},
180 'androidx.startup:startup-runtime': {},
181 'androidx.window:window': {
182 'optional-uses-libs': {
183 'androidx.window.extensions',
184 'androidx.window.sidecar'
185 }
186 },
187 'androidx.resourceinspection:resourceinspection-annotation': {},
188 'androidx.profileinstaller:profileinstaller': {},
189 'androidx.test.uiautomator:uiautomator': {},
Aurimas Liutikas6e87bbf2021-07-30 10:04:05 -0700190
Alan Viverettef44e2f92021-08-31 17:26:26 +0000191 # AndroidX for Compose
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000192 'androidx.compose.compiler:compiler-hosted': {
193 'host': True
194 },
195 'androidx.compose.runtime:runtime': {},
196 'androidx.compose.runtime:runtime-saveable': {},
197 'androidx.compose.runtime:runtime-livedata': {},
Jordan Demeulenaere13e19082022-11-24 17:43:06 +0100198 'androidx.compose.runtime:runtime-tracing': {},
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000199 'androidx.compose.foundation:foundation': {},
200 'androidx.compose.foundation:foundation-layout': {},
201 'androidx.compose.foundation:foundation-text': {},
202 'androidx.compose.ui:ui': {},
203 'androidx.compose.ui:ui-geometry': {},
204 'androidx.compose.ui:ui-graphics': {},
205 'androidx.compose.ui:ui-text': {},
206 'androidx.compose.ui:ui-tooling': {},
207 'androidx.compose.ui:ui-tooling-preview': {},
208 'androidx.compose.ui:ui-tooling-data': {},
209 'androidx.compose.ui:ui-unit': {},
210 'androidx.compose.ui:ui-util': {},
Jordan Demeulenaere606dca92022-07-29 16:09:28 +0200211 'androidx.compose.ui:ui-test': { },
212 'androidx.compose.ui:ui-test-junit4': { },
Jordan Demeulenaere76170242022-05-16 12:18:39 +0200213 'androidx.compose.ui:ui-test-manifest': { },
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000214 'androidx.compose.animation:animation-core': {},
215 'androidx.compose.animation:animation': {},
216 'androidx.compose.material:material-icons-core': {},
Jordan Demeulenaere606dca92022-07-29 16:09:28 +0200217 'androidx.compose.material:material-icons-extended': { },
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000218 'androidx.compose.material:material-ripple': {},
219 'androidx.compose.material:material': {},
220 'androidx.compose.material3:material3': {},
221 'androidx.activity:activity-compose': {},
Jordan Demeulenaere606dca92022-07-29 16:09:28 +0200222 'androidx.navigation:navigation-compose': { },
223 'androidx.lifecycle:lifecycle-viewmodel-compose': { },
Alan Viverettef44e2f92021-08-31 17:26:26 +0000224
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400225 # AndroidX for Multidex
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000226 'androidx.multidex:multidex': {},
227 'androidx.multidex:multidex-instrumentation': {},
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400228
229 # AndroidX for Constraint Layout
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000230 'androidx.constraintlayout:constraintlayout': {
231 'name': 'androidx-constraintlayout_constraintlayout'
232 },
233 'androidx.constraintlayout:constraintlayout-solver': {
234 'name': 'androidx-constraintlayout_constraintlayout-solver'
235 },
Anvesh Renikindi82b59882022-11-22 01:33:01 +0000236 'androidx.constraintlayout:constraintlayout-core': {},
237 'androidx.constraintlayout:constraintlayout-compose': {},
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400238 # AndroidX for Architecture Components
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000239 'androidx.arch.core:core-common': {},
240 'androidx.arch.core:core-runtime': {},
241 'androidx.lifecycle:lifecycle-common': {},
242 'androidx.lifecycle:lifecycle-common-java8': {},
243 'androidx.lifecycle:lifecycle-extensions': {},
244 'androidx.lifecycle:lifecycle-livedata': {},
245 'androidx.lifecycle:lifecycle-livedata-ktx': {},
246 'androidx.lifecycle:lifecycle-livedata-core': {},
247 'androidx.lifecycle:lifecycle-livedata-core-ktx': {},
248 'androidx.lifecycle:lifecycle-process': {},
249 'androidx.lifecycle:lifecycle-runtime': {},
250 'androidx.lifecycle:lifecycle-runtime-ktx': {},
251 'androidx.lifecycle:lifecycle-service': {},
252 'androidx.lifecycle:lifecycle-viewmodel': {},
253 'androidx.lifecycle:lifecycle-viewmodel-ktx': {},
254 'androidx.lifecycle:lifecycle-viewmodel-savedstate': {},
255 'androidx.paging:paging-common': {},
256 'androidx.paging:paging-common-ktx': {},
257 'androidx.paging:paging-runtime': {},
258 'androidx.sqlite:sqlite': {},
259 'androidx.sqlite:sqlite-framework': {},
260 'androidx.room:room-common': {
261 'host_and_device': True
262 },
263 'androidx.room:room-compiler': {
264 'host': True,
265 'extra-static-libs': {
Krzysztof Kosiński2450c552022-11-04 21:35:51 +0000266 'guava'
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000267 }
268 },
269 'androidx.room:room-migration': {
270 'host_and_device': True
271 },
272 'androidx.room:room-runtime': {},
273 'androidx.room:room-testing': {},
274 'androidx.room:room-compiler-processing': {
275 'host': True
276 },
277 'androidx.work:work-runtime': {},
278 'androidx.work:work-runtime-ktx': {},
279 'androidx.work:work-testing': {},
Allenab72f012018-01-29 18:10:23 -0800280
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400281 # Third-party dependencies
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000282 'com.google.android:flexbox': {
283 'name': 'flexbox',
284 'path': 'flexbox'
285 },
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400286
Jeff Gastonfa7c7e82018-04-06 13:35:48 -0400287 # Androidx Material Design Components
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000288 'com.google.android.material:material': {},
Alan Viverette95f6d362017-04-06 09:40:50 -0400289}
290
Alan Viverettecdfc9892021-08-31 19:35:58 +0000291# Mapping of POM dependencies to Soong build targets
292deps_rewrite = {
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000293 'auto-common': 'auto_common',
294 'auto-value-annotations': 'auto_value_annotations',
295 'com.google.auto.value:auto-value': 'libauto_value_plugin',
296 'monitor': 'androidx.test.monitor',
297 'rules': 'androidx.test.rules',
298 'runner': 'androidx.test.runner',
299 'androidx.test:core': 'androidx.test.core',
300 'com.squareup:javapoet': 'javapoet',
301 'com.google.guava:listenablefuture': 'guava-listenablefuture-prebuilt-jar',
302 'sqlite-jdbc': 'xerial-sqlite-jdbc',
303 'com.intellij:annotations': 'jetbrains-annotations',
304 'javax.annotation:javax.annotation-api': 'javax-annotation-api-prebuilt-host-jar',
305 'org.robolectric:robolectric': 'Robolectric_all-target',
306 'org.jetbrains.kotlin:kotlin-stdlib-common': 'kotlin-stdlib',
307 'org.jetbrains.kotlinx:kotlinx-coroutines-core': 'kotlinx_coroutines',
308 'org.jetbrains.kotlinx:kotlinx-coroutines-android': 'kotlinx_coroutines_android',
Jordan Demeulenaere606dca92022-07-29 16:09:28 +0200309 'org.jetbrains.kotlinx:kotlinx-coroutines-test':'kotlinx_coroutines_test',
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000310 'org.jetbrains.kotlinx:kotlinx-metadata-jvm': 'kotlinx_metadata_jvm',
Jordan Demeulenaere606dca92022-07-29 16:09:28 +0200311 'androidx.test.espresso:espresso-core':'androidx.test.espresso.core',
312 'androidx.test.espresso:espresso-idling-resource':'androidx.test.espresso.idling-resource',
Alan Viverettecdfc9892021-08-31 19:35:58 +0000313}
314
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000315# List of artifacts that will be updated from GMaven
316# Use pattern: `group:library:version:extension`
317# e.g.:
318# androidx.appcompat:appcompat:1.2.0:aar
319# Use `latest` to always fetch the latest version.
320# e.g.:
321# androidx.appcompat:appcompat:latest:aar
322# Also make sure you add `group:library`:{} to maven_to_make as well.
323gmaven_artifacts = {}
Alan Viverettef3c12722021-08-30 19:50:33 +0000324
Alan Viverette95f6d362017-04-06 09:40:50 -0400325# Always remove these files.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000326denylist_files = [
Alan Viverette95f6d362017-04-06 09:40:50 -0400327 'annotations.zip',
328 'public.txt',
329 'R.txt',
Alan Viverette44ad4e42017-08-09 17:45:00 -0400330 'AndroidManifest.xml',
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000331 os.path.join('libs', 'noto-emoji-compat-java.jar')
Alan Viverette95f6d362017-04-06 09:40:50 -0400332]
333
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000334artifact_pattern = re.compile(r'^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$')
Alan Viverette95f6d362017-04-06 09:40:50 -0400335
Alan Viverettec960cfb2017-12-04 13:09:22 -0500336
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000337def name_for_artifact(group_artifact):
338 """Returns the build system target name for a given library's Maven coordinate.
339
340 Args:
341 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core
342 Returns:
343 The build system target name for the artifact, ex. androidx.core_core.
344 """
345 return group_artifact.replace(':', '_')
Alan Viverettec960cfb2017-12-04 13:09:22 -0500346
347
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000348def path_for_artifact(group_artifact):
349 """Returns the file system path for a given library's Maven coordinate.
350
351 Args:
352 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core
353 Returns:
354 The file system path for the artifact, ex. androidx/core/core.
355 """
356 return group_artifact.replace('.', '/').replace(':', '/')
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500357
Alan Viverette95f6d362017-04-06 09:40:50 -0400358
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000359def populate_maven_to_make(mapping):
360 """Modifies the input mapping by expanding Maven coordinate keys into build target names and
361 paths.
Alan Viverette95f6d362017-04-06 09:40:50 -0400362
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000363 Args:
364 mapping: a map where the keys are Maven coordinates
365 """
366 for key in mapping:
367 if 'name' not in mapping[key]:
368 mapping[key]['name'] = name_for_artifact(key)
369 if 'path' not in maven_to_make[key]:
370 mapping[key]['path'] = path_for_artifact(key)
Alan Viverette45837092017-05-12 14:50:53 -0400371
372
Jeff Gastonc302bb92018-03-23 13:53:48 -0400373def detect_artifacts(maven_repo_dirs):
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000374 """Parses Maven libraries from the specified directories.
375
376 Args:
377 maven_repo_dirs: a list of maven repository roots
378 Returns:
379 A map of Maven coordinate keys to MavenLibraryInfo objects parsed from POM files.
380 """
Alan Viveretted4527e62017-05-11 15:03:25 -0400381 maven_lib_info = {}
382
Dan Willemsen814152e2017-11-06 13:22:11 -0800383 # Find the latest revision for each artifact, remove others
Jeff Gastonc302bb92018-03-23 13:53:48 -0400384 for repo_dir in maven_repo_dirs:
Dan Willemsen814152e2017-11-06 13:22:11 -0800385 for root, dirs, files in os.walk(repo_dir):
386 for file in files:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000387 if file[-4:] == '.pom':
Alan Viverettec960cfb2017-12-04 13:09:22 -0500388 # Read the POM (hack hack hack).
389 group_id = ''
390 artifact_id = ''
391 version = ''
392 file = os.path.join(root, file)
393 with open(file) as pom_file:
394 for line in pom_file:
395 if line[:11] == ' <groupId>':
396 group_id = line[11:-11]
397 elif line[:14] == ' <artifactId>':
398 artifact_id = line[14:-14]
399 elif line[:11] == ' <version>':
400 version = line[11:-11]
401 if group_id == '' or artifact_id == '' or version == '':
402 print_e('Failed to find Maven artifact data in ' + file)
403 continue
Alan Viverette95f6d362017-04-06 09:40:50 -0400404
Alan Viverettec960cfb2017-12-04 13:09:22 -0500405 # Locate the artifact.
406 artifact_file = file[:-4]
407 if os.path.exists(artifact_file + '.jar'):
408 artifact_file = artifact_file + '.jar'
409 elif os.path.exists(artifact_file + '.aar'):
410 artifact_file = artifact_file + '.aar'
411 else:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000412 # This error only occurs for a handful of gradle.plugin artifacts that only
413 # ship POM files, so we probably don't need to log unless we're debugging.
414 # print_e('Failed to find artifact for ' + artifact_file)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500415 continue
416
417 # Make relative to root.
418 artifact_file = artifact_file[len(root) + 1:]
419
420 # Find the mapping.
421 group_artifact = group_id + ':' + artifact_id
Jeff Gaston3e622ba2018-03-26 23:24:46 -0400422 if group_artifact in maven_to_make:
Alan Viverettec960cfb2017-12-04 13:09:22 -0500423 key = group_artifact
Jeff Gaston3e622ba2018-03-26 23:24:46 -0400424 elif artifact_id in maven_to_make:
425 key = artifact_id
Alan Viverettec960cfb2017-12-04 13:09:22 -0500426 else:
Alan Viverettecdfc9892021-08-31 19:35:58 +0000427 # No mapping entry, skip this library.
Alan Viverettec960cfb2017-12-04 13:09:22 -0500428 continue
429
430 # Store the latest version.
431 version = LooseVersion(version)
432 if key not in maven_lib_info \
433 or version > maven_lib_info[key].version:
434 maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version,
435 root, repo_dir, artifact_file)
Alan Viverette95f6d362017-04-06 09:40:50 -0400436
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400437 return maven_lib_info
438
439
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000440def transform_maven_repos(maven_repo_dirs, transformed_dir, extract_res=True,
441 include_static_deps=True, include=None, exclude=None, prepend=None):
442 """Transforms a standard Maven repository to be compatible with the Android build system.
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400443
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000444 When using the include argument by itself, all other libraries will be excluded. When using the
445 exclude argument by itself, all other libraries will be included. When using both arguments, the
446 inclusion list will be applied followed by the exclusion list.
447
448 Args:
449 maven_repo_dirs: path to local Maven repository
450 transformed_dir: relative path for output, ex. androidx
451 extract_res: whether to extract Android resources like AndroidManifest.xml from AARs
452 include_static_deps: whether to pass --static-deps to pom2bp
453 include: list of Maven groupIds or unversioned artifact coordinates to include for
454 updates, ex. androidx.core or androidx.core:core
455 exclude: list of Maven groupIds or unversioned artifact coordinates to exclude from
456 updates, ex. androidx.core or androidx.core:core
457 prepend: Path to a file containing text to be inserted at the beginning of the generated
458 build file
459 Returns:
460 True if successful, false otherwise.
461 """
462 if exclude is None:
463 exclude = []
464 if include is None:
465 include = []
466
467 cwd = os.getcwd()
468 local_repo = os.path.join(cwd, transformed_dir)
Jeff Gastona68a8d42018-03-23 14:00:13 -0400469 working_dir = temp_dir
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400470
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000471 # Handle inclusions by stashing the remote artifacts for the inclusions, replacing the entire
472 # remote repo with the local repo, then restoring the stashed artifacts.
473 for remote_repo in maven_repo_dirs:
474 remote_repo = os.path.join(cwd, remote_repo)
475 paths_to_copy = []
476 for group_artifact in include:
477 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact))
478 remote_path = os.path.join(remote_repo, artifact_path)
479 working_path = os.path.join(working_dir, artifact_path)
480 if os.path.exists(remote_path):
481 print(f'Included {group_artifact} in update')
482 paths_to_copy.append([remote_path, working_path])
483
484 # Move included artifacts from repo to temp.
485 for [remote_path, working_path] in paths_to_copy:
486 mv(remote_path, working_path)
487
488 # Replace all remaining artifacts in remote repo with local repo.
489 cp(local_repo, remote_repo)
490
491 # Restore included artifacts to remote repo.
492 for [remote_path, working_path] in paths_to_copy:
493 mv(working_path, remote_path)
494
495 # Handle exclusions by replacing the remote artifacts for the exclusions with local artifacts.
496 # This must happen before we parse the artifacts.
497 for remote_repo in maven_repo_dirs:
498 for group_artifact in exclude:
499 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact))
500 remote_path = os.path.join(remote_repo, artifact_path)
501 if os.path.exists(remote_path):
502 rm(remote_path)
503 local_path = os.path.join(local_repo, artifact_path)
504 if os.path.exists(local_path):
505 print(f'Excluded {group_artifact} from update, used local artifact')
506 mv(local_path, remote_path)
507 else:
508 print(f'Excluded {group_artifact} from update, no local artifact present')
509
510 # Parse artifacts.
511 maven_lib_info = detect_artifacts(maven_repo_dirs)
512
Alan Viverettec960cfb2017-12-04 13:09:22 -0500513 if not maven_lib_info:
514 print_e('Failed to detect artifacts')
515 return False
516
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000517 # Move libraries into the working directory, performing any necessary transformations.
Alan Viveretted4527e62017-05-11 15:03:25 -0400518 for info in maven_lib_info.values():
Alan Viveretted8ce7222017-12-11 17:24:43 -0500519 transform_maven_lib(working_dir, info, extract_res)
Dan Willemsen814152e2017-11-06 13:22:11 -0800520
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000521 # Generate a single Android.bp that specifies to use all of the above artifacts.
Colin Cross74683bc2018-04-11 17:36:18 -0700522 makefile = os.path.join(working_dir, 'Android.bp')
Alan Viverette4ec9a172018-02-20 16:19:31 -0500523 with open(makefile, 'w') as f:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000524 args = ['pom2bp']
525 args.extend(['-sdk-version', '31'])
526 args.extend(['-default-min-sdk-version', '24'])
Jeff Gaston1cc06092018-03-28 15:51:02 -0400527 if include_static_deps:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000528 args.append('-static-deps')
529 if prepend:
530 args.append(f'-prepend={prepend}')
531 rewrite_names = sorted(maven_to_make.keys())
532 args.extend([f'-rewrite=^{name}$={maven_to_make[name]["name"]}' for name in rewrite_names])
533 args.extend([f'-rewrite=^{key}$={value}' for key, value in deps_rewrite.items()])
534 args.extend(["-extra-static-libs=" + maven_to_make[name]['name'] + "=" + ",".join(
535 sorted(maven_to_make[name]['extra-static-libs'])) for name in maven_to_make if
536 'extra-static-libs' in maven_to_make[name]])
537 args.extend(["-optional-uses-libs=" + maven_to_make[name]['name'] + "=" + ",".join(
538 sorted(maven_to_make[name]['optional-uses-libs'])) for name in maven_to_make if
539 'optional-uses-libs' in maven_to_make[name]])
540 args.extend([f'-host={name}' for name in maven_to_make
541 if maven_to_make[name].get('host')])
542 args.extend([f'-host-and-device={name}' for name in maven_to_make
543 if maven_to_make[name].get('host_and_device')])
544 args.extend(['.'])
Dan Willemsen814152e2017-11-06 13:22:11 -0800545 subprocess.check_call(args, stdout=f, cwd=working_dir)
Alan Viverette95f6d362017-04-06 09:40:50 -0400546
547 # Replace the old directory.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000548 local_repo = os.path.join(cwd, transformed_dir)
549 mv(working_dir, local_repo)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500550 return True
Alan Viverette95f6d362017-04-06 09:40:50 -0400551
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000552
Alan Viveretted8ce7222017-12-11 17:24:43 -0500553def transform_maven_lib(working_dir, artifact_info, extract_res):
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000554 """Transforms the specified artifact for use in the Android build system.
555
556 Moves relevant files for the artifact represented by artifact_info of type MavenLibraryInfo into
557 the appropriate path inside working_dir, unpacking files needed by the build system from AARs.
558
559 Args:
560 working_dir: The directory into which the artifact should be moved
561 artifact_info: A MavenLibraryInfo representing the library artifact
562 extract_res: True to extract resources from AARs, false otherwise.
563 """
Dan Willemsen814152e2017-11-06 13:22:11 -0800564 # Move library into working dir
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000565 new_dir = os.path.normpath(
566 os.path.join(working_dir, os.path.relpath(artifact_info.dir, artifact_info.repo_dir)))
Jeff Gastonc302bb92018-03-23 13:53:48 -0400567 mv(artifact_info.dir, new_dir)
Dan Willemsen814152e2017-11-06 13:22:11 -0800568
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000569 maven_lib_type = os.path.splitext(artifact_info.file)[1][1:]
Alan Viveretted4527e62017-05-11 15:03:25 -0400570
Alan Viverettef3c12722021-08-30 19:50:33 +0000571 group_artifact = artifact_info.key
572 make_lib_name = maven_to_make[group_artifact]['name']
573 make_dir_name = maven_to_make[group_artifact]['path']
Alan Viveretted4527e62017-05-11 15:03:25 -0400574
Alan Viverette129555e2018-01-30 09:57:57 -0500575 artifact_file = os.path.join(new_dir, artifact_info.file)
576
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000577 if maven_lib_type == 'aar':
Colin Cross59b59db2018-04-24 13:58:05 -0700578 if extract_res:
579 target_dir = os.path.join(working_dir, make_dir_name)
580 if not os.path.exists(target_dir):
581 os.makedirs(target_dir)
Dan Willemsen814152e2017-11-06 13:22:11 -0800582
Alan Viverettec960cfb2017-12-04 13:09:22 -0500583 process_aar(artifact_file, target_dir)
Alan Viveretted4527e62017-05-11 15:03:25 -0400584
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000585 with zipfile.ZipFile(artifact_file) as zip_file:
586 manifests_dir = os.path.join(working_dir, 'manifests')
587 zip_file.extract('AndroidManifest.xml', os.path.join(manifests_dir, make_lib_name))
Alan Viveretted4527e62017-05-11 15:03:25 -0400588
Alan Viverette95f6d362017-04-06 09:40:50 -0400589
Alan Viverettec960cfb2017-12-04 13:09:22 -0500590def process_aar(artifact_file, target_dir):
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000591 """Extracts and cleans up the contents of an AAR file to the specified directory.
592
593 Removes classes.jar, empty directories, and denylisted files.
594
595 Args:
596 artifact_file: path to the AAR to extract
597 target_dir: directory into which the contents should be extracted
598 """
Alan Viverette95f6d362017-04-06 09:40:50 -0400599 # Extract AAR file to target_dir.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000600 with zipfile.ZipFile(artifact_file) as zip_file:
601 zip_file.extractall(target_dir)
Alan Viverette95f6d362017-04-06 09:40:50 -0400602
Dan Willemsen814152e2017-11-06 13:22:11 -0800603 # Remove classes.jar
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000604 classes_jar = os.path.join(target_dir, 'classes.jar')
Alan Viverette95f6d362017-04-06 09:40:50 -0400605 if os.path.exists(classes_jar):
Dan Willemsen814152e2017-11-06 13:22:11 -0800606 os.remove(classes_jar)
Alan Viverette95f6d362017-04-06 09:40:50 -0400607
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000608 # Remove empty dirs.
609 for root, dirs, files in os.walk(target_dir, topdown=False):
610 for dir_name in dirs:
611 dir_path = os.path.join(root, dir_name)
Alan Viverette95f6d362017-04-06 09:40:50 -0400612 if not os.listdir(dir_path):
613 os.rmdir(dir_path)
614
615 # Remove top-level cruft.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000616 for file in denylist_files:
Alan Viverette95f6d362017-04-06 09:40:50 -0400617 file_path = os.path.join(target_dir, file)
618 if os.path.exists(file_path):
619 os.remove(file_path)
620
Alan Viverettef17c9402017-07-19 12:57:40 -0400621
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000622def fetch_gmaven_artifact(artifact):
623 """Fetch a GMaven artifact.
624
625 Downloads a GMaven artifact
626 (https://developer.android.com/studio/build/dependencies#gmaven-access)
627
628 Args:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000629 artifact: an instance of GMavenArtifact.
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000630 """
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000631 pom_path = maven_path_for_artifact(
632 'gmaven', artifact.group, artifact.library, artifact.version, 'pom')
633 artifact_path = maven_path_for_artifact(
634 'gmaven', artifact.group, artifact.library, artifact.version, artifact.ext)
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000635
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000636 download_file_to_disk(artifact.get_pom_file_url(), pom_path)
637 download_file_to_disk(artifact.get_artifact_url(), artifact_path)
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000638
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000639 return os.path.dirname(artifact_path)
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000640
641
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000642def download_file_to_disk(url, filepath):
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000643 """Download the file at URL to the location dictated by the path.
644
645 Args:
646 url: Remote URL to download file from.
647 filepath: Filesystem path to write the file to.
648 """
649 print(f'Downloading URL: {url}')
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000650 file_data = request.urlopen(url)
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000651
652 try:
653 os.makedirs(os.path.dirname(filepath))
654 except os.error:
655 # This is a common situation - os.makedirs fails if dir already exists.
656 pass
657 try:
658 with open(filepath, 'wb') as f:
659 f.write(six.ensure_binary(file_data.read()))
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000660 except Exception as e:
661 print_e(e.__class__, 'occurred while reading', filepath)
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000662 os.remove(os.path.dirname(filepath))
663 raise
664
665
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000666def update_gmaven(gmaven_artifacts_list):
667 artifacts = [GMavenArtifact(artifact) for artifact in gmaven_artifacts_list]
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000668 for artifact in artifacts:
669 if artifact.version == 'latest':
670 artifact.version = artifact.get_latest_version()
671
Saeid Farivar Asanjan54cd34e2021-10-01 16:47:54 +0000672 if not transform_maven_repos(['gmaven'], gmaven_dir, extract_res=False):
673 return []
674 return [artifact.key for artifact in artifacts]
675
676
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000677def update_androidx(target, build_id, local_file, include, exclude, beyond_corp):
678 """Fetches and extracts Jetpack library prebuilts.
679
680 Args:
681 target: Android build server target name, must be specified if local_file is empty
682 build_id: Optional Android build server ID, must be specified if local_file is empty
683 local_file: Optional local top-of-tree ZIP, must be specified if build_id is empty
684 include: List of Maven groupIds or unversioned artifact coordinates to include for
685 updates, ex. android.core or androidx.core:core
686 exclude: List of Maven groupIds or unversioned artifact coordinates to exclude from
687 updates, ex. android.core or androidx.core:core
688 beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher
689 Returns:
690 True if successful, false otherwise.
691 """
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400692 if build_id:
Anthony Chenacbb4872018-07-02 11:22:48 -0700693 repo_file = 'top-of-tree-m2repository-all-%s.zip' % build_id.fs_id
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000694 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp, None)
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400695 else:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000696 repo_dir = fetch_and_extract(target, None, None, beyond_corp, local_file)
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400697 if not repo_dir:
698 print_e('Failed to extract AndroidX repository')
699 return False
700
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000701 prepend_path = os.path.relpath('update_prebuilts/prepend_androidx_license', start=temp_dir)
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400702
Tony Mak0f658752019-07-19 11:11:05 +0100703 # Transform the repo archive into a Makefile-compatible format.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000704 if not transform_maven_repos([repo_dir], androidx_dir, extract_res=False, include=include,
705 exclude=exclude, prepend=prepend_path):
Tony Mak0f658752019-07-19 11:11:05 +0100706 return False
707
708 # Import JavaPlugins.bp in Android.bp.
709 makefile = os.path.join(androidx_dir, 'Android.bp')
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000710 with open(makefile, 'a+') as f:
Tony Mak0f658752019-07-19 11:11:05 +0100711 f.write('\nbuild = ["JavaPlugins.bp"]\n')
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000712
713 # Keep OWNERs file, JavaPlugins.bp file, and TEST_MAPPING files untouched.
Nick Anthonybac2fca2022-11-03 18:56:57 +0000714 files_to_restore = [androidx_owners, java_plugins_bp_path, test_mapping_file,
715 compose_test_mapping_file]
716 for file_to_restore in files_to_restore:
717 # Ignore any output or error - these files are not gauranteed to exist, but
718 # if they do, we want to restore them.
719 subprocess.call(['git', 'restore', file_to_restore],
720 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
Tony Mak0f658752019-07-19 11:11:05 +0100721
722 return True
Jeff Gastonbbd6bfc2018-03-27 17:35:49 -0400723
Alan Viverette6dc45752020-04-16 14:56:20 +0000724
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000725def update_jetifier(target, build_id, beyond_corp):
726 """
727 Fetches and extracts Jetifier tool prebuilts.
728
729 Args:
730 target: Android build server target name
731 build_id: Android build server ID
732 beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher
733 Return:
734 Whether the prebuilt was successfully updated.
735 """
Jeff Gaston782c3e32018-02-06 14:36:17 -0500736 repo_file = 'jetifier-standalone.zip'
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000737 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp)
Jeff Gaston782c3e32018-02-06 14:36:17 -0500738 if not repo_dir:
739 print_e('Failed to extract Jetifier')
740 return False
741
742 rm(jetifier_dir)
Jeff Gaston553a4372018-03-29 18:34:22 -0400743 mv(os.path.join(repo_dir, 'jetifier-standalone'), jetifier_dir)
744 os.chmod(os.path.join(jetifier_dir, 'bin', 'jetifier-standalone'), 0o755)
Jeff Gaston782c3e32018-02-06 14:36:17 -0500745 return True
746
Alan Viverette7e897e22018-03-09 15:24:10 -0500747
Alan Viverette6dc45752020-04-16 14:56:20 +0000748def update_constraint(local_file):
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000749 """
750 Extracts ConstraintLayout library prebuilts.
751
752 Args:
753 local_file: local Maven repository ZIP containing library artifacts
754 Return:
755 Whether the prebuilts were successfully updated.
756 """
Jeff Gaston9edf86c2018-04-18 13:45:23 -0400757 repo_dir = extract_artifact(local_file)
758 if not repo_dir:
Alan Viverette6dc45752020-04-16 14:56:20 +0000759 print_e('Failed to extract Constraint Layout')
Jeff Gaston9edf86c2018-04-18 13:45:23 -0400760 return False
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000761 return transform_maven_repos([repo_dir], os.path.join(extras_dir, 'constraint-layout-x'),
762 extract_res=False)
Jeff Gaston9edf86c2018-04-18 13:45:23 -0400763
Alan Viverette45837092017-05-12 14:50:53 -0400764
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000765def update_material(local_file):
766 """
767 Extracts Material Design Components library prebuilts.
768
769 Args:
770 local_file: local Maven repository ZIP containing library artifacts
771 Return:
772 Whether the prebuilts were successfully updated.
773 """
774 design_dir = extract_artifact(local_file)
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400775 if not design_dir:
Alan Viverette6dc45752020-04-16 14:56:20 +0000776 print_e('Failed to extract Material Design Components')
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400777 return False
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000778 return transform_maven_repos([design_dir], os.path.join(extras_dir, 'material-design-x'),
779 extract_res=False)
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400780
Jeff Gaston6e7e7502018-04-04 00:22:45 -0400781
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000782def update_framework(target, build_id, sdk_dir, beyond_corp):
Anton Hansson83f92172020-03-26 11:12:03 +0000783 api_scope_list = ['public', 'system', 'test', 'module-lib', 'system-server']
Sundong Ahn66091f22018-08-29 18:54:28 +0900784 if sdk_dir == 'current':
785 api_scope_list.append('core')
786
787 for api_scope in api_scope_list:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000788 target_dir = os.path.join(sdk_dir, api_scope)
Sundong Ahn66091f22018-08-29 18:54:28 +0900789 if api_scope == 'core':
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000790 artifact_to_path = {'core.current.stubs.jar': os.path.join(target_dir, 'android.jar')}
Sundong Ahn66091f22018-08-29 18:54:28 +0900791 else:
Paul Duffin6815a312021-10-28 13:02:51 +0100792 artifact_to_path = {
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000793 'apistubs/android/' + api_scope + '/*.jar': os.path.join(target_dir, '*'),
Paul Duffin6815a312021-10-28 13:02:51 +0100794 }
795 if api_scope == 'public' or api_scope == 'module-lib':
796 # Distinct core-for-system-modules.jar files are only provided
797 # for the public and module-lib API surfaces.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000798 artifact_to_path[
799 'system-modules/' + api_scope + '/core-for-system-modules.jar'] = os.path.join(
800 target_dir, '*')
Anton Hansson1d01a032018-04-09 10:29:37 +0100801
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000802 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp):
Anton Hansson1d01a032018-04-09 10:29:37 +0100803 return False
804
Jiyong Park1d1c9632018-05-29 17:45:27 +0900805 if api_scope == 'public':
Anton Hansson1d01a032018-04-09 10:29:37 +0100806 # Fetch a few artifacts from the public sdk.
Sundong Ahn66091f22018-08-29 18:54:28 +0900807 artifact = 'sdk-repo-linux-platforms-%s.zip' % build_id.fs_id
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000808 artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp)
Anton Hansson9709fd42018-04-03 16:52:13 +0100809 if not artifact_path:
810 return False
Anton Hansson1d01a032018-04-09 10:29:37 +0100811
812 with zipfile.ZipFile(artifact_path) as zipFile:
Colin Crossd338d702020-07-10 19:17:54 -0700813 extra_files = [
814 'android.jar',
815 'framework.aidl',
816 'uiautomator.jar',
Colin Crossd338d702020-07-10 19:17:54 -0700817 'data/annotations.zip',
818 'data/api-versions.xml']
819 for filename in extra_files:
Anton Hanssona26e4812020-03-25 12:54:39 +0000820 matches = list(filter(lambda path: filename in path, zipFile.namelist()))
821 if len(matches) != 1:
822 print_e('Expected 1 file named \'%s\' in zip %s, found %d' %
823 (filename, zipFile.filename, len(matches)))
824 return False
825 zip_path = matches[0]
826 src_path = zipFile.extract(zip_path)
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000827 dst_path = os.path.join(target_dir, filename)
Anton Hanssona26e4812020-03-25 12:54:39 +0000828 mv(src_path, dst_path)
Anton Hansson9709fd42018-04-03 16:52:13 +0100829
Anton Hansson73be1512021-06-24 10:56:05 +0100830 # Filtered API DB is currently only available for "public"
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000831 fetch_artifacts(target, build_id, {'api-versions-public-filtered.xml': os.path.join(
832 target_dir, 'data/api-versions-filtered.xml')}, beyond_corp)
Anton Hansson73be1512021-06-24 10:56:05 +0100833
Alan Viverettecd3de262017-08-14 09:51:30 -0400834 return True
Alan Viverette45837092017-05-12 14:50:53 -0400835
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000836
Sundong Ahncc34cc32018-07-11 15:18:47 +0900837def update_makefile(build_id):
838 template = '"%s",\n\
839 "current"'
840 makefile = os.path.join(git_dir, 'Android.bp')
841
842 with open(makefile, 'r+') as f:
843 contents = f.read().replace('"current"', template % build_id)
844 f.seek(0)
845 f.write(contents)
846
847 return True
Alan Viverette45837092017-05-12 14:50:53 -0400848
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000849
850def finalize_sdk(target, build_id, sdk_version, beyond_corp):
Anton Hansson1d01a032018-04-09 10:29:37 +0100851 target_finalize_dir = '%d' % sdk_version
Anton Hansson9709fd42018-04-03 16:52:13 +0100852
Anton Hanssonaa554c02020-04-30 14:06:20 +0100853 for api_scope in ['public', 'system', 'test', 'module-lib', 'system-server']:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000854 artifact_to_path = {f'apistubs/android/{api_scope}/api/*.txt': os.path.join(
855 target_finalize_dir, api_scope, 'api', '*')}
Sundong Ahn66091f22018-08-29 18:54:28 +0900856
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000857 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp):
Sundong Ahn66091f22018-08-29 18:54:28 +0900858 return False
859
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000860 return update_framework(target, build_id, target_finalize_dir, beyond_corp) and update_makefile(
861 target_finalize_dir)
Anton Hansson9709fd42018-04-03 16:52:13 +0100862
Anton Hansson9709fd42018-04-03 16:52:13 +0100863
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000864def update_framework_current(target, build_id, beyond_corp):
865 return update_framework(target, build_id, current_path, beyond_corp)
Anton Hansson9709fd42018-04-03 16:52:13 +0100866
867
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000868def update_buildtools(target, arch, build_id, beyond_corp):
Jeff Gaston13e38412018-02-06 14:45:36 -0500869 artifact_path = fetch_and_extract(target, build_id.url_id,
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000870 f'sdk-repo-{arch}-build-tools-{build_id.fs_id}.zip',
871 beyond_corp)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500872 if not artifact_path:
873 return False
874
875 top_level_dir = os.listdir(artifact_path)[0]
876 src_path = os.path.join(artifact_path, top_level_dir)
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000877 dst_path = os.path.join(buildtools_dir, arch)
Colin Cross20648d32021-02-12 13:30:51 -0800878
879 # There are a few libraries that have been manually added to the
880 # build tools, copy them from the destination back to the source
881 # before the destination is overwritten.
882 files_to_save = (
883 'lib64/libconscrypt_openjdk_jni.dylib',
884 'lib64/libconscrypt_openjdk_jni.so',
885 'bin/lib64/libwinpthread-1.dll',
886 )
887 for file in files_to_save:
888 src_file = os.path.join(dst_path, file)
889 dst_file = os.path.join(src_path, file)
890 if os.path.exists(dst_path):
891 mv(src_file, dst_file)
892
Alan Viverettec1c32b62017-12-20 09:40:36 -0500893 mv(src_path, dst_path)
894
895 # Move all top-level files to /bin and make them executable
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000896 bin_path = os.path.join(dst_path, 'bin')
897 top_level_files = filter(lambda e: os.path.isfile(os.path.join(dst_path, e)), os.listdir(dst_path))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500898 for file in top_level_files:
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000899 src_file = os.path.join(dst_path, file)
900 dst_file = os.path.join(bin_path, file)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500901 mv(src_file, dst_file)
902 os.chmod(dst_file, 0o755)
903
Colin Cross6673ceb2021-02-12 13:14:14 -0800904 # Make the files under lld-bin executable
905 lld_bin_files = os.listdir(os.path.join(dst_path, 'lld-bin'))
906 for file in lld_bin_files:
907 os.chmod(os.path.join(dst_path, 'lld-bin', file), 0o755)
908
Alan Viverettec1c32b62017-12-20 09:40:36 -0500909 # Remove renderscript
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000910 rm(os.path.join(dst_path, 'renderscript'))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500911
912 return True
913
914
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000915def has_uncommitted_changes():
Jeff Gastoncc296a82018-03-23 14:33:24 -0400916 try:
917 # Make sure we don't overwrite any pending changes.
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000918 diff_command = f'cd {git_dir} && git diff --quiet'
919 subprocess.check_call(diff_command, shell=True)
920 subprocess.check_call(f'{diff_command} --cached', shell=True)
Jeff Gastoncc296a82018-03-23 14:33:24 -0400921 return False
922 except subprocess.CalledProcessError:
923 return True
924
925
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000926def main():
927 parser = argparse.ArgumentParser(
928 description='Update current prebuilts')
929 parser.add_argument(
930 'source', nargs='?',
931 help='Build server build ID or local Maven ZIP file')
932 parser.add_argument(
933 '-m', '--material', action='store_true',
934 help='If specified, updates only Material Design Components')
935 parser.add_argument(
936 '-c', '--constraint', action='store_true',
937 help='If specified, updates only Constraint Layout')
938 parser.add_argument(
939 '-j', '--jetifier', action='store_true',
940 help='If specified, updates only Jetifier')
941 parser.add_argument(
942 '-p', '--platform', action='store_true',
943 help='If specified, updates only the Android Platform')
944 parser.add_argument(
945 '-f', '--finalize_sdk', type=int,
946 help='Finalize the build as the specified SDK version. Must be used together with -e')
947 parser.add_argument(
948 '-e', '--finalize_extension', type=int,
949 help='Finalize the build as the specified extension SDK version. Must be used together with -f')
950 parser.add_argument('--bug', type=int, help='The bug number to add to the commit message.')
951 parser.add_argument(
952 '--sdk_target',
953 default=framework_sdk_target,
954 help='If specified, the name of the build target from which to retrieve the SDK when -p or -f '
955 'is specified.')
956 parser.add_argument(
957 '-b', '--buildtools', action='store_true',
958 help='If specified, updates only the Build Tools')
959 parser.add_argument(
960 '-x', '--androidx', action='store_true',
961 help='If specified, updates only the Jetpack (androidx) libraries excluding those covered by '
962 'other arguments')
963 parser.add_argument(
964 '--include', action='append', default=[],
965 help='If specified with -x, includes the specified Jetpack library Maven group or artifact for '
966 'updates. Applied before exclude.')
967 parser.add_argument(
968 '--exclude', action='append', default=[],
969 help='If specified with -x, excludes the specified Jetpack library Maven group or artifact '
970 'from updates')
971 parser.add_argument(
972 '-g', '--gmaven', action='store_true',
973 help='If specified, updates only the artifact from GMaven libraries excluding those covered by '
974 'other arguments')
975 parser.add_argument(
976 '--commit-first', action='store_true',
977 help='If specified, then if uncommited changes exist, commit before continuing')
978 parser.add_argument(
979 '--beyond-corp', action='store_true',
980 help='If specified, then fetch artifacts with tooling that works on BeyondCorp devices')
Anton Hansson4bcebac2022-04-07 17:23:18 +0100981
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000982 rm(temp_dir)
Alan Viveretted4527e62017-05-11 15:03:25 -0400983
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000984 args = parser.parse_args()
Jeff Gastoncc296a82018-03-23 14:33:24 -0400985
Anvesh Renikindi18680e62022-10-26 19:15:50 +0000986 # Validate combinations of arguments.
987 if not args.source and (args.platform or args.buildtools or args.jetifier
988 or args.androidx or args.material or args.finalize_sdk
989 or args.constraint):
990 parser.error('You must specify a build ID or local Maven ZIP file')
991 sys.exit(1)
992 if not (args.gmaven or args.platform or args.buildtools or args.jetifier
993 or args.androidx or args.material or args.finalize_sdk
994 or args.finalize_extension or args.constraint):
995 parser.error('You must specify at least one target to update')
996 sys.exit(1)
997 if (args.finalize_sdk is None) != (args.finalize_extension is None):
998 parser.error('Either both or neither of -e and -f must be specified.')
999 sys.exit(1)
1000 if args.finalize_sdk and not args.bug:
1001 parser.error('Specifying a bug ID with --bug is required when finalizing an SDK.')
1002 sys.exit(1)
Alan Viveretted4527e62017-05-11 15:03:25 -04001003
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001004 # Validate the build environment for POM-dependent targets.
1005 if (args.constraint or args.material or args.androidx or args.gmaven) \
1006 and which('pom2bp') is None:
1007 parser.error('Cannot find pom2bp in path; please run lunch to set up build environment. '
1008 'You may also need to run \'m pom2bp\' if it hasn\'t been built already.')
1009 sys.exit(1)
Anton Hansson57a1b142022-04-07 17:27:41 +01001010
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001011 # Validate the git status.
1012 if has_uncommitted_changes():
1013 if args.commit_first:
1014 subprocess.check_call(f'cd {git_dir} && git add -u', shell=True)
1015 subprocess.check_call(f'cd {git_dir} && git commit -m \'save working state\'',
1016 shell=True)
1017 if has_uncommitted_changes():
1018 self_file = os.path.basename(__file__)
1019 print_e(f'FAIL: There are uncommitted changes here. Please commit or stash before '
1020 f'continuing, because {self_file} will run "git reset --hard" if execution fails')
1021 sys.exit(1)
Anton Hansson4bcebac2022-04-07 17:23:18 +01001022
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001023 if args.bug:
1024 commit_msg_suffix = '\n\nBug: {args.bug}'
Alan Viverette129555e2018-01-30 09:57:57 -05001025 else:
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001026 commit_msg_suffix = ''
Alan Viveretted4527e62017-05-11 15:03:25 -04001027
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001028 # Are we fetching a build ID or using a local file?
1029 build_id = None
1030 file = None
1031 if args.source:
1032 build_id = parse_build_id(args.source)
1033 if build_id is None:
1034 file = args.source
1035
Alan Viveretted4527e62017-05-11 15:03:25 -04001036 try:
Anvesh Renikindi18680e62022-10-26 19:15:50 +00001037 components = []
1038 if args.constraint:
1039 if update_constraint(file):
1040 components.append('Constraint Layout')
1041 else:
1042 print_e('Failed to update Constraint Layout, aborting...')
1043 sys.exit(1)
1044 if args.material:
1045 if update_material(file):
1046 components.append('Material Design Components')
1047 else:
1048 print_e('Failed to update Material Design Components, aborting...')
1049 sys.exit(1)
1050 if args.gmaven:
1051 updated_artifacts = update_gmaven(gmaven_artifacts)
1052 if updated_artifacts:
1053 components.append('\n'.join(updated_artifacts))
1054 else:
1055 print_e('Failed to update GMaven, aborting...')
1056 sys.exit(1)
1057 if args.androidx:
1058 if update_androidx('androidx', build_id, file, args.include, args.exclude,
1059 args.beyond_corp):
1060 components.append('AndroidX')
1061 else:
1062 print_e('Failed to update AndroidX, aborting...')
1063 sys.exit(1)
1064 if args.jetifier:
1065 if update_jetifier('androidx', build_id, args.beyond_corp):
1066 components.append('Jetifier')
1067 else:
1068 print_e('Failed to update Jetifier, aborting...')
1069 sys.exit(1)
1070 if args.platform or args.finalize_sdk:
1071 if update_framework_current(args.sdk_target, build_id, args.beyond_corp):
1072 components.append('platform SDK')
1073 else:
1074 print_e('Failed to update platform SDK, aborting...')
1075 sys.exit(1)
1076 if args.finalize_sdk:
1077 n = args.finalize_sdk
1078 if not finalize_sdk(args.sdk_target, build_id, n, args.beyond_corp):
1079 print_e('Failed to finalize SDK %d, aborting...' % n)
1080 sys.exit(1)
1081
1082 # We commit the finalized dir separately from the current sdk update.
1083 msg = f'Import final sdk version {n} from build {build_id.url_id}{commit_msg_suffix}'
1084 subprocess.check_call(['git', 'add', '%d' % n])
1085 subprocess.check_call(['git', 'add', 'Android.bp'])
1086 subprocess.check_call(['git', 'commit', '-m', msg])
1087
1088 # Finalize extension sdk level
1089 readme = (f'- {args.finalize_extension}: Finalized together with '
1090 'Android {args.finalize_sdk} (all modules)')
1091 cmd = extension_sdk_finalization_cmd.format(
1092 readme=readme,
1093 bug=args.bug,
1094 extension_version=args.finalize_extension,
1095 build_id=build_id.url_id)
1096 subprocess.check_call(shlex.split(cmd), cwd=repo_root_dir.resolve())
1097 if args.buildtools:
1098 if update_buildtools('sdk_mac', 'darwin', build_id, args.beyond_corp) \
1099 and update_buildtools('sdk', 'linux', build_id, args.beyond_corp) \
1100 and update_buildtools('sdk', 'windows', build_id, args.beyond_corp):
1101 components.append('build tools')
1102 else:
1103 print_e('Failed to update build tools, aborting...')
1104 sys.exit(1)
1105
1106 # Build the git commit.
1107 subprocess.check_call(['git', 'add', current_path, buildtools_dir])
1108
1109 # Build the commit message.
1110 components_msg = ', '.join(components)
1111 argv_msg = ' '.join(sys.argv)
1112 if not args.source and args.gmaven:
1113 src_msg = 'GMaven'
1114 elif not args.source.isnumeric():
1115 src_msg = 'local Maven ZIP'
1116 else:
1117 src_msg = f'build {build_id.url_id}'
1118 msg = f'Import {components_msg} from {src_msg}\n\n{argv_msg}{commit_msg_suffix}'
1119
1120 # Create the git commit.
1121 subprocess.check_call(['git', 'commit', '-q', '-m', msg])
1122
1123 if args.finalize_sdk:
1124 print('NOTE: Created three commits:')
1125 subprocess.check_call(['git', 'log', '-3', '--oneline'])
1126 else:
1127 print('Created commit:')
1128 subprocess.check_call(['git', 'log', '-1', '--oneline'])
1129 print('Remember to test this change before uploading it to Gerrit!')
1130
1131 finally:
1132 # Revert all stray files, including the downloaded zip.
1133 try:
1134 with open(os.devnull, 'w') as bitbucket:
1135 subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket)
1136 subprocess.check_call(
1137 ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!', '--allow-empty'],
1138 stdout=bitbucket)
1139 subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket)
1140 except subprocess.CalledProcessError:
1141 print_e('ERROR: Failed cleaning up, manual cleanup required!!!')
1142
1143
1144# Add automatic entries to maven_to_make.
1145populate_maven_to_make(maven_to_make)
1146
1147if __name__ == '__main__':
1148 main()