blob: c1a2af8dd19853ddc14cb9c79c7d67073afb26c5 [file] [log] [blame]
Alan Viverettef5cb0e62017-11-17 15:15:23 -05001#!/usr/bin/python3
Alan Viverette95f6d362017-04-06 09:40:50 -04002
Alan Viverettecd3de262017-08-14 09:51:30 -04003# This script is used to update platform SDK prebuilts, Support Library, and a variety of other
4# prebuilt libraries used by Android's Makefile builds. For details on how to use this script,
5# visit go/update-prebuilts.
Alan Viverette95f6d362017-04-06 09:40:50 -04006import os, sys, getopt, zipfile, re
Alan Viveretted4527e62017-05-11 15:03:25 -04007import argparse
8import subprocess
Alan Viverettef5cb0e62017-11-17 15:15:23 -05009from shutil import copyfile, rmtree, which
Alan Viveretted4527e62017-05-11 15:03:25 -040010from distutils.version import LooseVersion
Alan Viverettef5cb0e62017-11-17 15:15:23 -050011from functools import reduce
Alan Viverette95f6d362017-04-06 09:40:50 -040012
Alan Viverette45837092017-05-12 14:50:53 -040013current_path = 'current'
14system_path = 'system_current'
Alan Viverette3e57a4a2017-08-11 15:49:47 -040015support_dir = os.path.join(current_path, 'support')
16extras_dir = os.path.join(current_path, 'extras')
Alan Viverettec1c32b62017-12-20 09:40:36 -050017buildtools_dir = 'tools'
Jeff Gaston13e38412018-02-06 14:45:36 -050018jetifier_dir = os.path.join(buildtools_dir, 'jetifier')
Alan Viverette95f6d362017-04-06 09:40:50 -040019
Alan Viverettecd3de262017-08-14 09:51:30 -040020# See go/fetch_artifact for details on this script.
Alan Viveretted4527e62017-05-11 15:03:25 -040021FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact'
22
Alan Viverette95f6d362017-04-06 09:40:50 -040023# Does not import support-v4, which is handled as a separate Android.mk (../support-v4) to
24# statically include dependencies. Similarly, the support-v13 module is imported as
25# support-v13-nodeps and then handled as a separate Android.mk (../support-v13) to statically
26# include dependencies.
27maven_to_make = {
28 'animated-vector-drawable': ['android-support-animatedvectordrawable', 'graphics/drawable'],
Alan Viveretted8ce7222017-12-11 17:24:43 -050029 'appcompat-v7': ['android-support-v7-appcompat', 'v7/appcompat'],
Alan Viverette95f6d362017-04-06 09:40:50 -040030 'cardview-v7': ['android-support-v7-cardview', 'v7/cardview'],
31 'customtabs': ['android-support-customtabs', 'customtabs'],
Alan Viverette95f6d362017-04-06 09:40:50 -040032 'exifinterface': ['android-support-exifinterface', 'exifinterface'],
33 'gridlayout-v7': ['android-support-v7-gridlayout', 'v7/gridlayout'],
34 'leanback-v17': ['android-support-v17-leanback', 'v17/leanback'],
35 'mediarouter-v7': ['android-support-v7-mediarouter', 'v7/mediarouter'],
Alan Viverette95f6d362017-04-06 09:40:50 -040036 'palette-v7': ['android-support-v7-palette', 'v7/palette'],
37 'percent': ['android-support-percent', 'percent'],
38 'preference-leanback-v17': ['android-support-v17-preference-leanback', 'v17/preference-leanback'],
39 'preference-v14': ['android-support-v14-preference', 'v14/preference'],
40 'preference-v7': ['android-support-v7-preference', 'v7/preference'],
41 'recommendation': ['android-support-recommendation', 'recommendation'],
42 'recyclerview-v7': ['android-support-v7-recyclerview', 'v7/recyclerview'],
Alan Viverette129555e2018-01-30 09:57:57 -050043 'support-annotations': ['android-support-annotations', 'annotations', 'jar'],
Alan Viverette95f6d362017-04-06 09:40:50 -040044 'support-compat': ['android-support-compat', 'compat'],
45 'support-core-ui': ['android-support-core-ui', 'core-ui'],
46 'support-core-utils': ['android-support-core-utils', 'core-utils'],
47 'support-dynamic-animation': ['android-support-dynamic-animation', 'dynamic-animation'],
48 'support-emoji': ['android-support-emoji', 'emoji'],
49 'support-emoji-appcompat': ['android-support-emoji-appcompat', 'emoji-appcompat'],
50 'support-emoji-bundled': ['android-support-emoji-bundled', 'emoji-bundled'],
51 'support-fragment': ['android-support-fragment', 'fragment'],
52 'support-media-compat': ['android-support-media-compat', 'media-compat'],
53 'support-tv-provider': ['android-support-tv-provider', 'tv-provider'],
Alan Viveretted8ce7222017-12-11 17:24:43 -050054 'support-v4': ['android-support-v4', 'v4'],
55 'support-v13': ['android-support-v13', 'v13'],
Alan Viverette95f6d362017-04-06 09:40:50 -040056 'support-vector-drawable': ['android-support-vectordrawable', 'graphics/drawable'],
57 'transition': ['android-support-transition', 'transition'],
Alan Viverette3e57a4a2017-08-11 15:49:47 -040058 'wear': ['android-support-wear', 'wear'],
Alan Viverette7bb86da2018-02-21 18:29:02 -050059
60 # AndroidX splits
61 'collections': ['android-support-collections', 'collections'],
62 'heifwriter': ['android-support-heifwriter', 'heifwriter'],
63 'webkit': ['android-support-webkit', 'webkit'],
64 'customview': ['android-support-customview', 'customview'],
65 'textclassifier': ['android-support-textclassifier', 'textclassifier'],
66 'swiperefreshlayout': ['android-support-swiperefreshlayout', 'swiperefreshlayout'],
67 'viewpager': ['android-support-viewpager', 'viewpager'],
68 'coordinatorlayout': ['android-support-coordinatorlayout', 'coordinatorlayout'],
69 'asynclayoutinflater': ['android-support-asynclayoutinflater', 'asynclayoutinflater'],
70 'support-content': ['android-support-support-content', 'support-content'],
71 'documentfile': ['android-support-documentfile', 'documentfile'],
72 'drawerlayout': ['android-support-drawerlayout', 'drawerlayout'],
73 'localbroadcastmanager': ['android-support-localbroadcastmanager', 'localbroadcastmanager'],
74 'print': ['android-support-print', 'print'],
75 'slidingpanelayout': ['android-support-slidingpanelayout', 'slidingpanelayout'],
76 'interpolator': ['android-support-interpolator', 'interpolator'],
77 'cursoradapter': ['android-support-cursoradapter', 'cursoradapter'],
78 'loader': ['android-support-loader', 'loader'],
79
Alan Viverette19ecd502018-01-05 13:52:16 -050080 # Slices
81 'slices-core': ['android-slices-core', 'slices-core'],
82 'slices-view': ['android-slices-view', 'slices-view'],
83 'slices-builders': ['android-slices-builders', 'slices-builders'],
84
85 # Multidex
86 'multidex': ['android-support-multidex', 'multidex/library'],
87 'multidex-instrumentation': ['android-support-multidex-instrumentation', 'multidex/instrumentation'],
88
89 # Constraint Layout
90 'constraint-layout': ['android-support-constraint-layout', 'constraint-layout'],
91 'constraint-layout-solver': ['android-support-constraint-layout-solver', 'constraint-layout-solver'],
92
93 # App Arch Core
94 'android.arch.core:runtime': ['android-arch-core-runtime', 'arch-core/runtime'],
95 'android.arch.core:common': ['android-arch-core-common', 'arch-core/common'],
Allenab72f012018-01-29 18:10:23 -080096 'android.arch.core:testing': ['android-arch-core-testing', 'arch-core/testing'],
97
Alan Viverette19ecd502018-01-05 13:52:16 -050098 # Paging
99 'android.arch.paging:common': ['android-arch-paging-common', 'arch-paging/common'],
100 'android.arch.paging:runtime': ['android-arch-paging-runtime', 'arch-paging/runtime'],
Allenab72f012018-01-29 18:10:23 -0800101
Alan Viverette19ecd502018-01-05 13:52:16 -0500102 # Lifecycle
Allenab72f012018-01-29 18:10:23 -0800103 'android.arch.lifecycle:livedata-core': ['android-arch-lifecycle-livedata-core', 'arch-lifecycle/livedata-core'],
104 'android.arch.lifecycle:livedata': ['android-arch-lifecycle-livedata', 'arch-lifecycle/livedata'],
105 'android.arch.lifecycle:viewmodel': ['android-arch-lifecycle-viewmodel', 'arch-lifecycle/viewmodel'],
106 'android.arch.lifecycle:extensions': ['android-arch-lifecycle-extensions', 'arch-lifecycle/extensions'],
107 'android.arch.lifecycle:runtime': ['android-arch-lifecycle-runtime', 'arch-lifecycle/runtime'],
108 'android.arch.lifecycle:common': ['android-arch-lifecycle-common', 'arch-lifecycle/common'],
109 'android.arch.lifecycle:common-java8': ['android-arch-lifecycle-common-java8', 'arch-lifecycle/common-java8'],
Alan Viverettec7e88f12018-02-13 15:29:50 -0500110 # Missing dependencies:
111 # - auto-common
112 # - javapoet
113 #'android.arch.lifecycle:compiler': ['android-arch-lifecycle-compiler', 'arch-lifecycle/compiler'],
114 # Missing dependencies:
115 # - reactive-streams
116 #'android.arch.lifecycle:reactivestreams': ['android-arch-lifecycle-reactivestreams','arch-lifecycle/reactivestreams'],
Allenab72f012018-01-29 18:10:23 -0800117
Alan Viverette19ecd502018-01-05 13:52:16 -0500118 # Persistence
119 'android.arch.persistence:db': ['android-arch-persistence-db', 'arch-persistence/db'],
120 'android.arch.persistence:db-framework': ['android-arch-persistence-db-framework', 'arch-persistence/db-framework'],
Allenab72f012018-01-29 18:10:23 -0800121
Alan Viverette19ecd502018-01-05 13:52:16 -0500122 # Room
123 'android.arch.persistence.room:common': ['android-arch-room-common', 'arch-room/common'],
Alan Viverettec7e88f12018-02-13 15:29:50 -0500124 # Missing dependencies:
125 # - auto-common
126 # - javapoet
127 # - antlr4
128 # - kotlin-metadata
129 # - commons-codec
130 #'android.arch.persistence.room:compiler': ['android-arch-room-compiler', 'arch-room/compiler'],
Alan Viverette19ecd502018-01-05 13:52:16 -0500131 'android.arch.persistence.room:runtime': ['android-arch-room-runtime', 'arch-room/runtime'],
132 'android.arch.persistence.room:migration': ['android-arch-room-migration', 'arch-room/migration'],
Alan Viverettec7e88f12018-02-13 15:29:50 -0500133 # Missing dependencies:
134 # - rxjava
135 #'android.arch.persistence.room:rxjava2': ['android-arch-room-rxjava2', 'arch-room/rxjava2'],
Alan Viverette19ecd502018-01-05 13:52:16 -0500136 'android.arch.persistence.room:testing': ['android-arch-room-testing', 'arch-room/testing'],
Allenab72f012018-01-29 18:10:23 -0800137
Alan Viverette129555e2018-01-30 09:57:57 -0500138 # Material Design Components
139 'com.google.android:flexbox': ['flexbox', 'flexbox'],
140 'design': ['android-support-design', 'design'],
141 'design-animation': ['android-support-design-animation', 'design-animation'],
142 'design-bottomnavigation': ['android-support-design-bottomnavigation', 'design-bottomnavigation'],
143 'design-bottomsheet': ['android-support-design-bottomsheet', 'design-bottomsheet'],
144 'design-button': ['android-support-design-button', 'design-button'],
145 'design-canvas': ['android-support-design-canvas', 'design-canvas'],
146 'design-card': ['android-support-design-card', 'design-card'],
147 'design-chip': ['android-support-design-chip', 'design-chip'],
148 'design-circularreveal': ['android-support-design-circularreveal', 'design-circularreveal'],
149 'design-circularreveal-cardview': ['android-support-design-circularreveal-cardview', 'design-circularreveal-cardview'],
150 'design-circularreveal-coordinatorlayout': ['android-support-design-circularreveal-coordinatorlayout', 'design-circularreveal-coordinatorlayout'],
151 'design-color': ['android-support-design-color', 'design-color'],
152 'design-dialog': ['android-support-design-dialog', 'design-dialog'],
153 'design-drawable': ['android-support-design-drawable', 'design-drawable'],
154 'design-expandable': ['android-support-design-expandable', 'design-expandable'],
155 'design-floatingactionbutton': ['android-support-design-floatingactionbutton', 'design-floatingactionbutton'],
156 'design-internal': ['android-support-design-internal', 'design-internal'],
157 'design-math': ['android-support-design-math', 'design-math'],
158 'design-resources': ['android-support-design-resources', 'design-resources'],
159 'design-ripple': ['android-support-design-ripple', 'design-ripple'],
160 'design-snackbar': ['android-support-design-snackbar', 'design-snackbar'],
161 'design-stateful': ['android-support-design-stateful', 'design-stateful'],
162 'design-textfield': ['android-support-design-textfield', 'design-textfield'],
163 'design-theme': ['android-support-design-theme', 'design-theme'],
164 'design-transformation': ['android-support-design-transformation', 'design-transformation'],
165 'design-typography': ['android-support-design-typography', 'design-typography'],
166 'design-widget': ['android-support-design-widget', 'design-widget'],
Alan Viverettea59082b2018-02-22 10:27:46 -0500167 'design-navigation': ['android-support-design-navigation', 'design-navigation'],
168 'design-tabs': ['android-support-design-tabs', 'design-tabs'],
169 'design-bottomappbar': ['android-support-design-bottomappbar', 'design-bottomappbar'],
170 'design-shape': ['android-support-design-shape', 'design-shape'],
171
Alan Viverette95f6d362017-04-06 09:40:50 -0400172}
173
174# Always remove these files.
175blacklist_files = [
176 'annotations.zip',
177 'public.txt',
178 'R.txt',
Alan Viverette44ad4e42017-08-09 17:45:00 -0400179 'AndroidManifest.xml',
Alan Viverettef48d9b42017-08-17 14:45:46 -0400180 os.path.join('libs','noto-emoji-compat-java.jar')
Alan Viverette95f6d362017-04-06 09:40:50 -0400181]
182
Alan Viverette44ad4e42017-08-09 17:45:00 -0400183artifact_pattern = re.compile(r"^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$")
Alan Viverette95f6d362017-04-06 09:40:50 -0400184
Alan Viverettec960cfb2017-12-04 13:09:22 -0500185
186class MavenLibraryInfo:
187 def __init__(self, key, group_id, artifact_id, version, root, repo_dir, file):
188 self.key = key
189 self.group_id = group_id
190 self.artifact_id = artifact_id
191 self.version = version
192 self.root = root
193 self.repo_dir = repo_dir
194 self.file = file
195
196
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500197def print_e(*args, **kwargs):
198 print(*args, file=sys.stderr, **kwargs)
199
Alan Viverette95f6d362017-04-06 09:40:50 -0400200
201def touch(fname, times=None):
202 with open(fname, 'a'):
203 os.utime(fname, times)
204
205
Alan Viverette45837092017-05-12 14:50:53 -0400206def path(*path_parts):
207 return reduce((lambda x, y: os.path.join(x, y)), path_parts)
208
209
Alan Viverettecd3de262017-08-14 09:51:30 -0400210def flatten(list):
211 return reduce((lambda x, y: "%s %s" % (x, y)), list)
212
213
Alan Viverette45837092017-05-12 14:50:53 -0400214def rm(path):
215 if os.path.isdir(path):
216 rmtree(path)
217 elif os.path.exists(path):
218 os.remove(path)
219
220
221def mv(src_path, dst_path):
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400222 if os.path.exists(dst_path):
Alan Viverettecd3de262017-08-14 09:51:30 -0400223 rm(dst_path)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400224 if not os.path.exists(os.path.dirname(dst_path)):
225 os.makedirs(os.path.dirname(dst_path))
Alan Viverette45837092017-05-12 14:50:53 -0400226 os.rename(src_path, dst_path)
227
228
Dan Willemsen814152e2017-11-06 13:22:11 -0800229def detect_artifacts(repo_dirs):
Alan Viveretted4527e62017-05-11 15:03:25 -0400230 maven_lib_info = {}
231
Dan Willemsen814152e2017-11-06 13:22:11 -0800232 # Find the latest revision for each artifact, remove others
233 for repo_dir in repo_dirs:
234 for root, dirs, files in os.walk(repo_dir):
235 for file in files:
Alan Viverettec960cfb2017-12-04 13:09:22 -0500236 if file[-4:] == ".pom":
237 # Read the POM (hack hack hack).
238 group_id = ''
239 artifact_id = ''
240 version = ''
241 file = os.path.join(root, file)
242 with open(file) as pom_file:
243 for line in pom_file:
244 if line[:11] == ' <groupId>':
245 group_id = line[11:-11]
246 elif line[:14] == ' <artifactId>':
247 artifact_id = line[14:-14]
248 elif line[:11] == ' <version>':
249 version = line[11:-11]
250 if group_id == '' or artifact_id == '' or version == '':
251 print_e('Failed to find Maven artifact data in ' + file)
252 continue
Alan Viverette95f6d362017-04-06 09:40:50 -0400253
Alan Viverettec960cfb2017-12-04 13:09:22 -0500254 # Locate the artifact.
255 artifact_file = file[:-4]
256 if os.path.exists(artifact_file + '.jar'):
257 artifact_file = artifact_file + '.jar'
258 elif os.path.exists(artifact_file + '.aar'):
259 artifact_file = artifact_file + '.aar'
260 else:
261 print_e('Failed to find artifact for ' + artifact_file)
262 continue
263
264 # Make relative to root.
265 artifact_file = artifact_file[len(root) + 1:]
266
267 # Find the mapping.
268 group_artifact = group_id + ':' + artifact_id
269 if artifact_id in maven_to_make:
270 key = artifact_id
271 elif group_artifact in maven_to_make:
272 key = group_artifact
273 else:
274 print_e('Failed to find artifact mapping for ' + group_artifact)
275 continue
276
277 # Store the latest version.
278 version = LooseVersion(version)
279 if key not in maven_lib_info \
280 or version > maven_lib_info[key].version:
281 maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version,
282 root, repo_dir, artifact_file)
Alan Viverette95f6d362017-04-06 09:40:50 -0400283
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400284 return maven_lib_info
285
286
Alan Viveretted8ce7222017-12-11 17:24:43 -0500287def transform_maven_repo(repo_dirs, update_dir, extract_res=True):
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400288 cwd = os.getcwd()
289
290 # Use a temporary working directory.
291 working_dir = os.path.join(cwd, 'support_tmp')
Dan Willemsen814152e2017-11-06 13:22:11 -0800292 maven_lib_info = detect_artifacts(repo_dirs)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400293
Alan Viverettec960cfb2017-12-04 13:09:22 -0500294 if not maven_lib_info:
295 print_e('Failed to detect artifacts')
296 return False
297
Alan Viveretted4527e62017-05-11 15:03:25 -0400298 for info in maven_lib_info.values():
Alan Viveretted8ce7222017-12-11 17:24:43 -0500299 transform_maven_lib(working_dir, info, extract_res)
Dan Willemsen814152e2017-11-06 13:22:11 -0800300
301 with open(os.path.join(working_dir, 'Android.mk'), 'w') as f:
Alan Viverette229a53d2018-02-06 16:25:22 -0500302 args = ["pom2mk", "-static-deps", "-sdk-version", "current"]
Dan Willemsen814152e2017-11-06 13:22:11 -0800303 args.extend(["-rewrite=^" + name + "$=" + maven_to_make[name][0] for name in maven_to_make])
304 args.extend(["."])
305 subprocess.check_call(args, stdout=f, cwd=working_dir)
Alan Viverette95f6d362017-04-06 09:40:50 -0400306
307 # Replace the old directory.
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400308 output_dir = os.path.join(cwd, update_dir)
309 mv(working_dir, output_dir)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500310 return True
Alan Viverette95f6d362017-04-06 09:40:50 -0400311
Alan Viveretted4527e62017-05-11 15:03:25 -0400312
Alan Viveretted8ce7222017-12-11 17:24:43 -0500313def transform_maven_lib(working_dir, artifact_info, extract_res):
Dan Willemsen814152e2017-11-06 13:22:11 -0800314 # Move library into working dir
Alan Viverettec960cfb2017-12-04 13:09:22 -0500315 new_dir = os.path.join(working_dir, os.path.relpath(artifact_info.root, artifact_info.repo_dir))
316 mv(artifact_info.root, new_dir)
Dan Willemsen814152e2017-11-06 13:22:11 -0800317
318 for dirpath, dirs, files in os.walk(new_dir):
319 for f in files:
320 if '-sources.jar' in f:
321 os.remove(os.path.join(dirpath, f))
322
Alan Viverettec960cfb2017-12-04 13:09:22 -0500323 matcher = artifact_pattern.match(artifact_info.file)
324 maven_lib_name = artifact_info.key
Alan Viveretted4527e62017-05-11 15:03:25 -0400325 maven_lib_vers = matcher.group(2)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500326 maven_lib_type = artifact_info.file[-3:]
Alan Viveretted4527e62017-05-11 15:03:25 -0400327
Alan Viverettec960cfb2017-12-04 13:09:22 -0500328 make_lib_name = maven_to_make[artifact_info.key][0]
329 make_dir_name = maven_to_make[artifact_info.key][1]
Alan Viveretted4527e62017-05-11 15:03:25 -0400330
Alan Viverette129555e2018-01-30 09:57:57 -0500331 artifact_file = os.path.join(new_dir, artifact_info.file)
332
Dan Willemsen814152e2017-11-06 13:22:11 -0800333 if extract_res:
Dan Willemsen814152e2017-11-06 13:22:11 -0800334 target_dir = os.path.join(working_dir, make_dir_name)
335 if not os.path.exists(target_dir):
336 os.makedirs(target_dir)
337
338 if maven_lib_type == "aar":
Alan Viverettec960cfb2017-12-04 13:09:22 -0500339 process_aar(artifact_file, target_dir)
Alan Viveretted4527e62017-05-11 15:03:25 -0400340
Alan Viveretted8ce7222017-12-11 17:24:43 -0500341 if maven_lib_type == "aar":
Dan Willemsend677b602017-11-08 22:13:17 -0800342 with zipfile.ZipFile(artifact_file) as zip:
Alan Viveretted8ce7222017-12-11 17:24:43 -0500343 manifests_dir = os.path.join(working_dir, "manifests")
344 zip.extract("AndroidManifest.xml", os.path.join(manifests_dir, make_lib_name))
Dan Willemsend677b602017-11-08 22:13:17 -0800345
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500346 print(maven_lib_vers, ":", maven_lib_name, "->", make_lib_name)
Alan Viveretted4527e62017-05-11 15:03:25 -0400347
Alan Viverette95f6d362017-04-06 09:40:50 -0400348
Alan Viverettec960cfb2017-12-04 13:09:22 -0500349def process_aar(artifact_file, target_dir):
Alan Viverette95f6d362017-04-06 09:40:50 -0400350 # Extract AAR file to target_dir.
351 with zipfile.ZipFile(artifact_file) as zip:
352 zip.extractall(target_dir)
353
Dan Willemsen814152e2017-11-06 13:22:11 -0800354 # Remove classes.jar
Alan Viverette95f6d362017-04-06 09:40:50 -0400355 classes_jar = os.path.join(target_dir, "classes.jar")
356 if os.path.exists(classes_jar):
Dan Willemsen814152e2017-11-06 13:22:11 -0800357 os.remove(classes_jar)
Alan Viverette95f6d362017-04-06 09:40:50 -0400358
359 # Remove or preserve empty dirs.
360 for root, dirs, files in os.walk(target_dir):
361 for dir in dirs:
362 dir_path = os.path.join(root, dir)
363 if not os.listdir(dir_path):
364 os.rmdir(dir_path)
365
366 # Remove top-level cruft.
367 for file in blacklist_files:
368 file_path = os.path.join(target_dir, file)
369 if os.path.exists(file_path):
370 os.remove(file_path)
371
Alan Viverettef17c9402017-07-19 12:57:40 -0400372
Alan Viverettecd3de262017-08-14 09:51:30 -0400373def fetch_artifact(target, build_id, artifact_path):
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500374 print('Fetching %s from %s...' % (artifact_path, target))
Alan Viverettecd3de262017-08-14 09:51:30 -0400375 fetch_cmd = [FETCH_ARTIFACT, '--bid', str(build_id), '--target', target, artifact_path]
Alan Viverette45837092017-05-12 14:50:53 -0400376 try:
Alan Viverettecd3de262017-08-14 09:51:30 -0400377 subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
Alan Viverette45837092017-05-12 14:50:53 -0400378 except subprocess.CalledProcessError:
Jeff Gaston13e38412018-02-06 14:45:36 -0500379 print_e('FAIL: Unable to retrieve %s artifact for build ID %s' % (artifact_path, build_id))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500380 print_e('Please make sure you are authenticated for build server access!')
Alan Viverette45837092017-05-12 14:50:53 -0400381 return None
382 return artifact_path
383
384
Alan Viverette129555e2018-01-30 09:57:57 -0500385def extract_artifact(artifact_path):
Alan Viverette45837092017-05-12 14:50:53 -0400386 # Unzip the repo archive into a separate directory.
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400387 repo_dir = os.path.basename(artifact_path)[:-4]
Alan Viverette45837092017-05-12 14:50:53 -0400388 with zipfile.ZipFile(artifact_path) as zipFile:
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400389 zipFile.extractall(repo_dir)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400390 return repo_dir
391
392
Alan Viverette129555e2018-01-30 09:57:57 -0500393def fetch_and_extract(target, build_id, file):
394 artifact_path = fetch_artifact(target, build_id, file)
395 if not artifact_path:
396 return None
397 return extract_artifact(artifact_path)
398
399
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400400def update_support(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500401 repo_file = 'top-of-tree-m2repository-%s.zip' % build_id.fs_id
402 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400403 if not repo_dir:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500404 print_e('Failed to extract Support Library repository')
Alan Viverettef48d9b42017-08-17 14:45:46 -0400405 return False
Alan Viverette45837092017-05-12 14:50:53 -0400406
407 # Transform the repo archive into a Makefile-compatible format.
Alan Viveretted8ce7222017-12-11 17:24:43 -0500408 return transform_maven_repo([repo_dir], support_dir)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500409
Jeff Gaston782c3e32018-02-06 14:36:17 -0500410def update_jetifier(target, build_id):
411 repo_file = 'jetifier-standalone.zip'
412 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file)
413 if not repo_dir:
414 print_e('Failed to extract Jetifier')
415 return False
416
417 rm(jetifier_dir)
418 mv(repo_dir, jetifier_dir)
Jeff Gaston6afb1e42018-02-07 18:58:07 -0500419 os.chmod(os.path.join(jetifier_dir, 'jetifier-standalone', 'bin', 'jetifier-standalone'), 0o755)
Jeff Gaston782c3e32018-02-06 14:36:17 -0500420 return True
421
Alan Viverettec960cfb2017-12-04 13:09:22 -0500422def update_toolkit(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500423 repo_dir = fetch_and_extract(target, build_id.url_id, 'top-of-tree-m2repository-%s.zip' % build_id.fs_id)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500424 if not repo_dir:
425 print_e('Failed to extract App Toolkit repository')
426 return False
427
428 # Transform the repo archive into a Makefile-compatible format.
429 return transform_maven_repo([repo_dir], os.path.join(extras_dir, 'app-toolkit'))
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400430
431
432def update_constraint(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500433 layout_dir = fetch_and_extract(target, build_id.url_id,
434 'com.android.support.constraint-constraint-layout-%s.zip' % build_id.fs_id)
435 solver_dir = fetch_and_extract(target, build_id.url_id,
436 'com.android.support.constraint-constraint-layout-solver-%s.zip' % build_id.fs_id)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400437 if not layout_dir or not solver_dir:
Alan Viverettec960cfb2017-12-04 13:09:22 -0500438 print_e('Failed to extract Constraint Layout repositories')
Alan Viverettecd3de262017-08-14 09:51:30 -0400439 return False
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400440
441 # Passing False here is an inelegant solution, but it means we can replace
442 # the top-level directory without worrying about other child directories.
Alan Viverettec960cfb2017-12-04 13:09:22 -0500443 return transform_maven_repo([layout_dir, solver_dir],
444 os.path.join(extras_dir, 'constraint-layout'), extract_res=False)
Alan Viverette45837092017-05-12 14:50:53 -0400445
446
Alan Viverette129555e2018-01-30 09:57:57 -0500447def update_design(file):
448 design_dir = extract_artifact(file)
449 if not design_dir:
450 print_e('Failed to extract Design Library repositories')
451 return False
452
453 # Don't bother extracting resources -- this should only be used with AAPT2.
454 return transform_maven_repo([design_dir],
455 os.path.join(extras_dir, 'material-design'), extract_res=False)
456
457
Alan Viverette45837092017-05-12 14:50:53 -0400458def extract_to(zip_file, paths, filename, parent_path):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500459 zip_path = next(filter(lambda path: filename in path, paths))
Alan Viverette45837092017-05-12 14:50:53 -0400460 src_path = zip_file.extract(zip_path)
461 dst_path = path(parent_path, filename)
462 mv(src_path, dst_path)
463
464
Alan Viverettecd3de262017-08-14 09:51:30 -0400465def update_sdk_repo(target, build_id):
Alan Viverette45837092017-05-12 14:50:53 -0400466 platform = 'darwin' if 'mac' in target else 'linux'
Alan Viverettecd3de262017-08-14 09:51:30 -0400467 artifact_path = fetch_artifact(
Jeff Gaston13e38412018-02-06 14:45:36 -0500468 target, build_id.url_id, 'sdk-repo-%s-platforms-%s.zip' % (platform, build_id.fs_id))
Alan Viverette45837092017-05-12 14:50:53 -0400469 if not artifact_path:
Alan Viverettecd3de262017-08-14 09:51:30 -0400470 return False
Alan Viverette45837092017-05-12 14:50:53 -0400471
472 with zipfile.ZipFile(artifact_path) as zipFile:
473 paths = zipFile.namelist()
474
475 extract_to(zipFile, paths, 'android.jar', current_path)
476 extract_to(zipFile, paths, 'uiautomator.jar', current_path)
477 extract_to(zipFile, paths, 'framework.aidl', current_path)
Paul Duffind891f3c2017-12-06 11:32:56 +0000478 extract_to(zipFile, paths, 'optional/android.test.base.jar', current_path)
Paul Duffin2b1a1462017-07-14 16:21:33 +0100479 extract_to(zipFile, paths, 'optional/android.test.mock.jar', current_path)
480 extract_to(zipFile, paths, 'optional/android.test.runner.jar', current_path)
Alan Viverette45837092017-05-12 14:50:53 -0400481
482 # Unclear if this is actually necessary.
483 extract_to(zipFile, paths, 'framework.aidl', system_path)
Jiyong Parkb773d292018-01-30 23:33:19 +0900484
Colin Cross4f4b9062018-02-14 10:03:19 -0800485 artifact_path = fetch_artifact(target, build_id.fs_id, 'core.current.stubs.jar')
Jiyong Parkb773d292018-01-30 23:33:19 +0900486 if not artifact_path:
487 return False
488
489 mv(artifact_path, path(current_path, 'core.jar'))
Alan Viverettecd3de262017-08-14 09:51:30 -0400490 return True
Alan Viverette45837092017-05-12 14:50:53 -0400491
492
Alan Viverettecd3de262017-08-14 09:51:30 -0400493def update_system(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500494 artifact_path = fetch_artifact(target, build_id.url_id, 'android_system.jar')
Alan Viverette45837092017-05-12 14:50:53 -0400495 if not artifact_path:
Alan Viverettecd3de262017-08-14 09:51:30 -0400496 return False
Alan Viverette45837092017-05-12 14:50:53 -0400497
498 mv(artifact_path, path(system_path, 'android.jar'))
Paul Duffin960e1ee2017-12-21 08:50:14 +0000499
Jeff Gaston13e38412018-02-06 14:45:36 -0500500 artifact_path = fetch_artifact(target, build_id.url_id, 'android.test.mock.stubs_system.jar')
Paul Duffin960e1ee2017-12-21 08:50:14 +0000501 if not artifact_path:
502 return False
503
504 mv(artifact_path, path(system_path, 'optional/android.test.mock.jar'))
505
Alan Viverettecd3de262017-08-14 09:51:30 -0400506 return True
507
508
Alan Viverettec1c32b62017-12-20 09:40:36 -0500509def update_buildtools(target, arch, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500510 artifact_path = fetch_and_extract(target, build_id.url_id,
511 "sdk-repo-%s-build-tools-%s.zip" % (arch, build_id.fs_id))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500512 if not artifact_path:
513 return False
514
515 top_level_dir = os.listdir(artifact_path)[0]
516 src_path = os.path.join(artifact_path, top_level_dir)
517 dst_path = path(buildtools_dir, arch)
518 mv(src_path, dst_path)
519
520 # Move all top-level files to /bin and make them executable
521 bin_path = path(dst_path, 'bin')
522 top_level_files = filter(lambda e: os.path.isfile(path(dst_path, e)), os.listdir(dst_path))
523 for file in top_level_files:
524 src_file = path(dst_path, file)
525 dst_file = path(bin_path, file)
526 mv(src_file, dst_file)
527 os.chmod(dst_file, 0o755)
528
529 # Remove renderscript
530 rm(path(dst_path, 'renderscript'))
531
532 return True
533
534
Alan Viverettecd3de262017-08-14 09:51:30 -0400535def append(text, more_text):
536 if text:
537 return "%s, %s" % (text, more_text)
538 return more_text
Alan Viverette45837092017-05-12 14:50:53 -0400539
Alan Viverette95f6d362017-04-06 09:40:50 -0400540
Jeff Gaston13e38412018-02-06 14:45:36 -0500541class buildId(object):
542 def __init__(self, url_id, fs_id):
543 # id when used in build server urls
544 self.url_id = url_id
545 # id when used in build commands
546 self.fs_id = fs_id
547
Jeff Gastonb9b09052018-02-05 14:16:05 -0500548def getBuildId(args):
Jeff Gaston13e38412018-02-06 14:45:36 -0500549 # must be in the format 12345 or P12345
Jeff Gastonb9b09052018-02-05 14:16:05 -0500550 source = args.source
Jeff Gaston13e38412018-02-06 14:45:36 -0500551 number_text = source[:]
552 presubmit = False
553 if number_text.startswith("P"):
554 presubmit = True
555 number_text = number_text[1:]
556 if not number_text.isnumeric():
557 raise Exception('Updating this set of prebuilts requires <source> to be a build id, not "' + source + '"')
558 url_id = source
559 fs_id = url_id
560 if presubmit:
561 fs_id = "0"
562 args.file = False
563 return buildId(url_id, fs_id)
Jeff Gastonb9b09052018-02-05 14:16:05 -0500564
Colin Cross573e1212018-02-12 14:59:52 -0800565def getFile(args):
566 source = args.source
567 if not source.isnumeric():
568 return args.source
569 else:
570 raise Exception('Updating this set of prebuilts requires <source> to be file, not a numeric build id')
571
Alan Viveretted4527e62017-05-11 15:03:25 -0400572parser = argparse.ArgumentParser(
Alan Viverette45837092017-05-12 14:50:53 -0400573 description=('Update current prebuilts'))
Alan Viveretted4527e62017-05-11 15:03:25 -0400574parser.add_argument(
Alan Viverette129555e2018-01-30 09:57:57 -0500575 'source',
576 help='Build server build ID or local Maven ZIP file')
577parser.add_argument(
578 '-d', '--design', action="store_true",
579 help='If specified, updates only the Design Library')
Alan Viveretted4527e62017-05-11 15:03:25 -0400580parser.add_argument(
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400581 '-c', '--constraint', action="store_true",
582 help='If specified, updates only Constraint Layout')
583parser.add_argument(
Alan Viverette5bfe05b2017-06-06 14:21:29 -0400584 '-s', '--support', action="store_true",
585 help='If specified, updates only the Support Library')
Alan Viverette45837092017-05-12 14:50:53 -0400586parser.add_argument(
Jeff Gaston782c3e32018-02-06 14:36:17 -0500587 '-j', '--jetifier', action="store_true",
588 help='If specified, updates only Jetifier')
589parser.add_argument(
Alan Viverettec960cfb2017-12-04 13:09:22 -0500590 '-t', '--toolkit', action="store_true",
591 help='If specified, updates only the App Toolkit')
592parser.add_argument(
Alan Viverette5bfe05b2017-06-06 14:21:29 -0400593 '-p', '--platform', action="store_true",
594 help='If specified, updates only the Android Platform')
Alan Viverettec1c32b62017-12-20 09:40:36 -0500595parser.add_argument(
596 '-b', '--buildtools', action="store_true",
597 help='If specified, updates only the Build Tools')
Alan Viveretted4527e62017-05-11 15:03:25 -0400598args = parser.parse_args()
Jeff Gaston56fe2762018-02-06 14:54:37 -0500599args.file = True
Alan Viverette129555e2018-01-30 09:57:57 -0500600if not args.source:
601 parser.error("You must specify a build ID or local Maven ZIP file")
Alan Viveretted4527e62017-05-11 15:03:25 -0400602 sys.exit(1)
Alan Viverette129555e2018-01-30 09:57:57 -0500603if not (args.support or args.platform or args.constraint or args.toolkit or args.buildtools \
Jeff Gaston782c3e32018-02-06 14:36:17 -0500604 or args.design or args.jetifier):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500605 parser.error("You must specify at least one target to update")
Alan Viverettecd3de262017-08-14 09:51:30 -0400606 sys.exit(1)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500607if (args.support or args.constraint or args.toolkit) and which('pom2mk') is None:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500608 parser.error("Cannot find pom2mk in path; please run lunch to set up build environment")
609 sys.exit(1)
Alan Viveretted4527e62017-05-11 15:03:25 -0400610
611try:
612 # Make sure we don't overwrite any pending changes.
613 subprocess.check_call(['git', 'diff', '--quiet', '--', '**'])
614 subprocess.check_call(['git', 'diff', '--quiet', '--cached', '--', '**'])
615except subprocess.CalledProcessError:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500616 print_e('FAIL: There are uncommitted changes here; please revert or stash')
Alan Viveretted4527e62017-05-11 15:03:25 -0400617 sys.exit(1)
618
619try:
Alan Viverettecd3de262017-08-14 09:51:30 -0400620 components = None
621 if args.constraint:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500622 if update_constraint('studio', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400623 components = append(components, 'Constraint Layout')
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500624 print_e('Failed to update Constraint Layout, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400625 else:
626 sys.exit(1)
627 if args.support:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500628 if update_support('support_library', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400629 components = append(components, 'Support Library')
630 else:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500631 print_e('Failed to update Support Library, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400632 sys.exit(1)
Jeff Gaston782c3e32018-02-06 14:36:17 -0500633 if args.jetifier:
634 if update_jetifier('support_library', getBuildId(args)):
635 components = append(components, 'Jetifier')
636 else:
637 print_e('Failed to update Jetifier, aborting...')
638 sys.exit(1)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500639 if args.toolkit:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500640 if update_toolkit('support_library_app_toolkit', getBuildId(args)):
Alan Viverettec960cfb2017-12-04 13:09:22 -0500641 components = append(components, 'App Toolkit')
642 else:
643 print_e('Failed to update App Toolkit, aborting...')
644 sys.exit(1)
Alan Viverettecd3de262017-08-14 09:51:30 -0400645 if args.platform:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500646 if update_sdk_repo('sdk_phone_armv7-sdk_mac', getBuildId(args)) \
647 and update_system('sdk_phone_armv7-sdk_mac', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400648 components = append(components, 'platform SDK')
649 else:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500650 print_e('Failed to update platform SDK, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400651 sys.exit(1)
Alan Viverette129555e2018-01-30 09:57:57 -0500652 if args.design:
Colin Cross573e1212018-02-12 14:59:52 -0800653 if update_design(getFile(args)):
Alan Viverette129555e2018-01-30 09:57:57 -0500654 components = append(components, 'Design Library')
655 else:
656 print_e('Failed to update platform SDK, aborting...')
657 sys.exit(1)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500658 if args.buildtools:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500659 if update_buildtools('sdk_phone_armv7-sdk_mac', 'darwin', getBuildId(args)) \
660 and update_buildtools('sdk_phone_x86_64-sdk', 'linux', getBuildId(args)) \
661 and update_buildtools('sdk_phone_armv7-win_sdk', 'windows', getBuildId(args)):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500662 components = append(components, 'build tools')
663 else:
664 print_e('Failed to update build tools, aborting...')
665 sys.exit(1)
Alan Viveretted4527e62017-05-11 15:03:25 -0400666
667 # Commit all changes.
Alan Viverette45837092017-05-12 14:50:53 -0400668 subprocess.check_call(['git', 'add', current_path])
669 subprocess.check_call(['git', 'add', system_path])
Alan Viverettec1c32b62017-12-20 09:40:36 -0500670 subprocess.check_call(['git', 'add', buildtools_dir])
Colin Cross573e1212018-02-12 14:59:52 -0800671 if not args.source.isnumeric():
Alan Viverette129555e2018-01-30 09:57:57 -0500672 src_msg = "local Maven ZIP"
673 else:
Jeff Gaston13e38412018-02-06 14:45:36 -0500674 src_msg = "build %s" % (getBuildId(args).url_id)
Alan Viverette129555e2018-01-30 09:57:57 -0500675 msg = "Import %s from %s\n\n%s" % (components, src_msg, flatten(sys.argv))
Alan Viveretted4527e62017-05-11 15:03:25 -0400676 subprocess.check_call(['git', 'commit', '-m', msg])
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500677 print('Remember to test this change before uploading it to Gerrit!')
Alan Viveretted4527e62017-05-11 15:03:25 -0400678
679finally:
680 # Revert all stray files, including the downloaded zip.
681 try:
682 with open(os.devnull, 'w') as bitbucket:
683 subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket)
684 subprocess.check_call(
685 ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!'], stdout=bitbucket)
686 subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket)
687 except subprocess.CalledProcessError:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500688 print_e('ERROR: Failed cleaning up, manual cleanup required!!!')