blob: ff2c9688c057d9a5fb41ffa485e0221c9e4e762e [file] [log] [blame]
Alan Viverettef5cb0e62017-11-17 15:15:23 -05001#!/usr/bin/python3
Alan Viverette95f6d362017-04-06 09:40:50 -04002
Alan Viverettecd3de262017-08-14 09:51:30 -04003# This script is used to update platform SDK prebuilts, Support Library, and a variety of other
4# prebuilt libraries used by Android's Makefile builds. For details on how to use this script,
5# visit go/update-prebuilts.
Alan Viverette95f6d362017-04-06 09:40:50 -04006import os, sys, getopt, zipfile, re
Alan Viveretted4527e62017-05-11 15:03:25 -04007import argparse
8import subprocess
Alan Viverettef5cb0e62017-11-17 15:15:23 -05009from shutil import copyfile, rmtree, which
Alan Viveretted4527e62017-05-11 15:03:25 -040010from distutils.version import LooseVersion
Alan Viverettef5cb0e62017-11-17 15:15:23 -050011from functools import reduce
Alan Viverette95f6d362017-04-06 09:40:50 -040012
Alan Viverette45837092017-05-12 14:50:53 -040013current_path = 'current'
14system_path = 'system_current'
Alan Viverette3e57a4a2017-08-11 15:49:47 -040015support_dir = os.path.join(current_path, 'support')
16extras_dir = os.path.join(current_path, 'extras')
Alan Viverettec1c32b62017-12-20 09:40:36 -050017buildtools_dir = 'tools'
Jeff Gaston13e38412018-02-06 14:45:36 -050018jetifier_dir = os.path.join(buildtools_dir, 'jetifier')
Alan Viverette95f6d362017-04-06 09:40:50 -040019
Alan Viverettecd3de262017-08-14 09:51:30 -040020# See go/fetch_artifact for details on this script.
Alan Viveretted4527e62017-05-11 15:03:25 -040021FETCH_ARTIFACT = '/google/data/ro/projects/android/fetch_artifact'
22
Alan Viverette95f6d362017-04-06 09:40:50 -040023# Does not import support-v4, which is handled as a separate Android.mk (../support-v4) to
24# statically include dependencies. Similarly, the support-v13 module is imported as
25# support-v13-nodeps and then handled as a separate Android.mk (../support-v13) to statically
26# include dependencies.
27maven_to_make = {
28 'animated-vector-drawable': ['android-support-animatedvectordrawable', 'graphics/drawable'],
Alan Viveretted8ce7222017-12-11 17:24:43 -050029 'appcompat-v7': ['android-support-v7-appcompat', 'v7/appcompat'],
Alan Viverette95f6d362017-04-06 09:40:50 -040030 'cardview-v7': ['android-support-v7-cardview', 'v7/cardview'],
31 'customtabs': ['android-support-customtabs', 'customtabs'],
Alan Viverette95f6d362017-04-06 09:40:50 -040032 'exifinterface': ['android-support-exifinterface', 'exifinterface'],
33 'gridlayout-v7': ['android-support-v7-gridlayout', 'v7/gridlayout'],
34 'leanback-v17': ['android-support-v17-leanback', 'v17/leanback'],
35 'mediarouter-v7': ['android-support-v7-mediarouter', 'v7/mediarouter'],
Alan Viverette95f6d362017-04-06 09:40:50 -040036 'palette-v7': ['android-support-v7-palette', 'v7/palette'],
37 'percent': ['android-support-percent', 'percent'],
38 'preference-leanback-v17': ['android-support-v17-preference-leanback', 'v17/preference-leanback'],
39 'preference-v14': ['android-support-v14-preference', 'v14/preference'],
40 'preference-v7': ['android-support-v7-preference', 'v7/preference'],
41 'recommendation': ['android-support-recommendation', 'recommendation'],
42 'recyclerview-v7': ['android-support-v7-recyclerview', 'v7/recyclerview'],
Alan Viverette129555e2018-01-30 09:57:57 -050043 'support-annotations': ['android-support-annotations', 'annotations', 'jar'],
Alan Viverette95f6d362017-04-06 09:40:50 -040044 'support-compat': ['android-support-compat', 'compat'],
45 'support-core-ui': ['android-support-core-ui', 'core-ui'],
46 'support-core-utils': ['android-support-core-utils', 'core-utils'],
47 'support-dynamic-animation': ['android-support-dynamic-animation', 'dynamic-animation'],
48 'support-emoji': ['android-support-emoji', 'emoji'],
49 'support-emoji-appcompat': ['android-support-emoji-appcompat', 'emoji-appcompat'],
50 'support-emoji-bundled': ['android-support-emoji-bundled', 'emoji-bundled'],
51 'support-fragment': ['android-support-fragment', 'fragment'],
52 'support-media-compat': ['android-support-media-compat', 'media-compat'],
53 'support-tv-provider': ['android-support-tv-provider', 'tv-provider'],
Alan Viveretted8ce7222017-12-11 17:24:43 -050054 'support-v4': ['android-support-v4', 'v4'],
55 'support-v13': ['android-support-v13', 'v13'],
Alan Viverette95f6d362017-04-06 09:40:50 -040056 'support-vector-drawable': ['android-support-vectordrawable', 'graphics/drawable'],
57 'transition': ['android-support-transition', 'transition'],
Alan Viverette3e57a4a2017-08-11 15:49:47 -040058 'wear': ['android-support-wear', 'wear'],
Alan Viverette19ecd502018-01-05 13:52:16 -050059 # Slices
60 'slices-core': ['android-slices-core', 'slices-core'],
61 'slices-view': ['android-slices-view', 'slices-view'],
62 'slices-builders': ['android-slices-builders', 'slices-builders'],
63
64 # Multidex
65 'multidex': ['android-support-multidex', 'multidex/library'],
66 'multidex-instrumentation': ['android-support-multidex-instrumentation', 'multidex/instrumentation'],
67
68 # Constraint Layout
69 'constraint-layout': ['android-support-constraint-layout', 'constraint-layout'],
70 'constraint-layout-solver': ['android-support-constraint-layout-solver', 'constraint-layout-solver'],
71
72 # App Arch Core
73 'android.arch.core:runtime': ['android-arch-core-runtime', 'arch-core/runtime'],
74 'android.arch.core:common': ['android-arch-core-common', 'arch-core/common'],
75 # Paging
76 'android.arch.paging:common': ['android-arch-paging-common', 'arch-paging/common'],
77 'android.arch.paging:runtime': ['android-arch-paging-runtime', 'arch-paging/runtime'],
78 # Lifecycle
Ian Lake7afc6372018-01-24 13:11:03 -080079 'android.arch.lifecycle:livedata-core': ['android-arch-lifecycle-livedata-core', 'arch-lifecycle/livedata-core'],
80 'android.arch.lifecycle:livedata': ['android-arch-lifecycle-livedata', 'arch-lifecycle/livedata'],
81 'android.arch.lifecycle:viewmodel': ['android-arch-lifecycle-viewmodel', 'arch-lifecycle/viewmodel'],
82 'android.arch.lifecycle:extensions': ['android-arch-lifecycle-extensions', 'arch-lifecycle/extensions'],
83 'android.arch.lifecycle:runtime': ['android-arch-lifecycle-runtime', 'arch-lifecycle/runtime'],
84 'android.arch.lifecycle:common': ['android-arch-lifecycle-common', 'arch-lifecycle/common'],
85 'android.arch.lifecycle:common-java8': ['android-arch-lifecycle-common-java8', 'arch-lifecycle/common-java8'],
Alan Viverette19ecd502018-01-05 13:52:16 -050086 # Persistence
87 'android.arch.persistence:db': ['android-arch-persistence-db', 'arch-persistence/db'],
88 'android.arch.persistence:db-framework': ['android-arch-persistence-db-framework', 'arch-persistence/db-framework'],
89 # Room
90 'android.arch.persistence.room:common': ['android-arch-room-common', 'arch-room/common'],
91 'android.arch.persistence.room:runtime': ['android-arch-room-runtime', 'arch-room/runtime'],
92 'android.arch.persistence.room:migration': ['android-arch-room-migration', 'arch-room/migration'],
93 'android.arch.persistence.room:testing': ['android-arch-room-testing', 'arch-room/testing'],
Alan Viverette129555e2018-01-30 09:57:57 -050094 # Material Design Components
95 'com.google.android:flexbox': ['flexbox', 'flexbox'],
96 'design': ['android-support-design', 'design'],
97 'design-animation': ['android-support-design-animation', 'design-animation'],
98 'design-bottomnavigation': ['android-support-design-bottomnavigation', 'design-bottomnavigation'],
99 'design-bottomsheet': ['android-support-design-bottomsheet', 'design-bottomsheet'],
100 'design-button': ['android-support-design-button', 'design-button'],
101 'design-canvas': ['android-support-design-canvas', 'design-canvas'],
102 'design-card': ['android-support-design-card', 'design-card'],
103 'design-chip': ['android-support-design-chip', 'design-chip'],
104 'design-circularreveal': ['android-support-design-circularreveal', 'design-circularreveal'],
105 'design-circularreveal-cardview': ['android-support-design-circularreveal-cardview', 'design-circularreveal-cardview'],
106 'design-circularreveal-coordinatorlayout': ['android-support-design-circularreveal-coordinatorlayout', 'design-circularreveal-coordinatorlayout'],
107 'design-color': ['android-support-design-color', 'design-color'],
108 'design-dialog': ['android-support-design-dialog', 'design-dialog'],
109 'design-drawable': ['android-support-design-drawable', 'design-drawable'],
110 'design-expandable': ['android-support-design-expandable', 'design-expandable'],
111 'design-floatingactionbutton': ['android-support-design-floatingactionbutton', 'design-floatingactionbutton'],
112 'design-internal': ['android-support-design-internal', 'design-internal'],
113 'design-math': ['android-support-design-math', 'design-math'],
114 'design-resources': ['android-support-design-resources', 'design-resources'],
115 'design-ripple': ['android-support-design-ripple', 'design-ripple'],
116 'design-snackbar': ['android-support-design-snackbar', 'design-snackbar'],
117 'design-stateful': ['android-support-design-stateful', 'design-stateful'],
118 'design-textfield': ['android-support-design-textfield', 'design-textfield'],
119 'design-theme': ['android-support-design-theme', 'design-theme'],
120 'design-transformation': ['android-support-design-transformation', 'design-transformation'],
121 'design-typography': ['android-support-design-typography', 'design-typography'],
122 'design-widget': ['android-support-design-widget', 'design-widget'],
Alan Viverette95f6d362017-04-06 09:40:50 -0400123}
124
125# Always remove these files.
126blacklist_files = [
127 'annotations.zip',
128 'public.txt',
129 'R.txt',
Alan Viverette44ad4e42017-08-09 17:45:00 -0400130 'AndroidManifest.xml',
Alan Viverettef48d9b42017-08-17 14:45:46 -0400131 os.path.join('libs','noto-emoji-compat-java.jar')
Alan Viverette95f6d362017-04-06 09:40:50 -0400132]
133
Alan Viverette44ad4e42017-08-09 17:45:00 -0400134artifact_pattern = re.compile(r"^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$")
Alan Viverette95f6d362017-04-06 09:40:50 -0400135
Alan Viverettec960cfb2017-12-04 13:09:22 -0500136
137class MavenLibraryInfo:
138 def __init__(self, key, group_id, artifact_id, version, root, repo_dir, file):
139 self.key = key
140 self.group_id = group_id
141 self.artifact_id = artifact_id
142 self.version = version
143 self.root = root
144 self.repo_dir = repo_dir
145 self.file = file
146
147
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500148def print_e(*args, **kwargs):
149 print(*args, file=sys.stderr, **kwargs)
150
Alan Viverette95f6d362017-04-06 09:40:50 -0400151
152def touch(fname, times=None):
153 with open(fname, 'a'):
154 os.utime(fname, times)
155
156
Alan Viverette45837092017-05-12 14:50:53 -0400157def path(*path_parts):
158 return reduce((lambda x, y: os.path.join(x, y)), path_parts)
159
160
Alan Viverettecd3de262017-08-14 09:51:30 -0400161def flatten(list):
162 return reduce((lambda x, y: "%s %s" % (x, y)), list)
163
164
Alan Viverette45837092017-05-12 14:50:53 -0400165def rm(path):
166 if os.path.isdir(path):
167 rmtree(path)
168 elif os.path.exists(path):
169 os.remove(path)
170
171
172def mv(src_path, dst_path):
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400173 if os.path.exists(dst_path):
Alan Viverettecd3de262017-08-14 09:51:30 -0400174 rm(dst_path)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400175 if not os.path.exists(os.path.dirname(dst_path)):
176 os.makedirs(os.path.dirname(dst_path))
Alan Viverette45837092017-05-12 14:50:53 -0400177 os.rename(src_path, dst_path)
178
179
Dan Willemsen814152e2017-11-06 13:22:11 -0800180def detect_artifacts(repo_dirs):
Alan Viveretted4527e62017-05-11 15:03:25 -0400181 maven_lib_info = {}
182
Dan Willemsen814152e2017-11-06 13:22:11 -0800183 # Find the latest revision for each artifact, remove others
184 for repo_dir in repo_dirs:
185 for root, dirs, files in os.walk(repo_dir):
186 for file in files:
Alan Viverettec960cfb2017-12-04 13:09:22 -0500187 if file[-4:] == ".pom":
188 # Read the POM (hack hack hack).
189 group_id = ''
190 artifact_id = ''
191 version = ''
192 file = os.path.join(root, file)
193 with open(file) as pom_file:
194 for line in pom_file:
195 if line[:11] == ' <groupId>':
196 group_id = line[11:-11]
197 elif line[:14] == ' <artifactId>':
198 artifact_id = line[14:-14]
199 elif line[:11] == ' <version>':
200 version = line[11:-11]
201 if group_id == '' or artifact_id == '' or version == '':
202 print_e('Failed to find Maven artifact data in ' + file)
203 continue
Alan Viverette95f6d362017-04-06 09:40:50 -0400204
Alan Viverettec960cfb2017-12-04 13:09:22 -0500205 # Locate the artifact.
206 artifact_file = file[:-4]
207 if os.path.exists(artifact_file + '.jar'):
208 artifact_file = artifact_file + '.jar'
209 elif os.path.exists(artifact_file + '.aar'):
210 artifact_file = artifact_file + '.aar'
211 else:
212 print_e('Failed to find artifact for ' + artifact_file)
213 continue
214
215 # Make relative to root.
216 artifact_file = artifact_file[len(root) + 1:]
217
218 # Find the mapping.
219 group_artifact = group_id + ':' + artifact_id
220 if artifact_id in maven_to_make:
221 key = artifact_id
222 elif group_artifact in maven_to_make:
223 key = group_artifact
224 else:
225 print_e('Failed to find artifact mapping for ' + group_artifact)
226 continue
227
228 # Store the latest version.
229 version = LooseVersion(version)
230 if key not in maven_lib_info \
231 or version > maven_lib_info[key].version:
232 maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version,
233 root, repo_dir, artifact_file)
Alan Viverette95f6d362017-04-06 09:40:50 -0400234
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400235 return maven_lib_info
236
237
Alan Viveretted8ce7222017-12-11 17:24:43 -0500238def transform_maven_repo(repo_dirs, update_dir, extract_res=True):
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400239 cwd = os.getcwd()
240
241 # Use a temporary working directory.
242 working_dir = os.path.join(cwd, 'support_tmp')
Dan Willemsen814152e2017-11-06 13:22:11 -0800243 maven_lib_info = detect_artifacts(repo_dirs)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400244
Alan Viverettec960cfb2017-12-04 13:09:22 -0500245 if not maven_lib_info:
246 print_e('Failed to detect artifacts')
247 return False
248
Alan Viveretted4527e62017-05-11 15:03:25 -0400249 for info in maven_lib_info.values():
Alan Viveretted8ce7222017-12-11 17:24:43 -0500250 transform_maven_lib(working_dir, info, extract_res)
Dan Willemsen814152e2017-11-06 13:22:11 -0800251
252 with open(os.path.join(working_dir, 'Android.mk'), 'w') as f:
Alan Viverette129555e2018-01-30 09:57:57 -0500253 args = ["pom2mk", "-static-deps", "-allow-missing", "-sdk-version", "current"]
254 for name in maven_to_make:
255 if len(maven_to_make[name]) == 3:
256 args.extend(["-type=" + maven_to_make[name][0] + "=" + maven_to_make[name][2]])
Dan Willemsen814152e2017-11-06 13:22:11 -0800257 args.extend(["-rewrite=^" + name + "$=" + maven_to_make[name][0] for name in maven_to_make])
258 args.extend(["."])
259 subprocess.check_call(args, stdout=f, cwd=working_dir)
Alan Viverette95f6d362017-04-06 09:40:50 -0400260
261 # Replace the old directory.
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400262 output_dir = os.path.join(cwd, update_dir)
263 mv(working_dir, output_dir)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500264 return True
Alan Viverette95f6d362017-04-06 09:40:50 -0400265
Alan Viveretted4527e62017-05-11 15:03:25 -0400266
Alan Viveretted8ce7222017-12-11 17:24:43 -0500267def transform_maven_lib(working_dir, artifact_info, extract_res):
Dan Willemsen814152e2017-11-06 13:22:11 -0800268 # Move library into working dir
Alan Viverettec960cfb2017-12-04 13:09:22 -0500269 new_dir = os.path.join(working_dir, os.path.relpath(artifact_info.root, artifact_info.repo_dir))
270 mv(artifact_info.root, new_dir)
Dan Willemsen814152e2017-11-06 13:22:11 -0800271
272 for dirpath, dirs, files in os.walk(new_dir):
273 for f in files:
274 if '-sources.jar' in f:
275 os.remove(os.path.join(dirpath, f))
276
Alan Viverettec960cfb2017-12-04 13:09:22 -0500277 matcher = artifact_pattern.match(artifact_info.file)
278 maven_lib_name = artifact_info.key
Alan Viveretted4527e62017-05-11 15:03:25 -0400279 maven_lib_vers = matcher.group(2)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500280 maven_lib_type = artifact_info.file[-3:]
Alan Viveretted4527e62017-05-11 15:03:25 -0400281
Alan Viverettec960cfb2017-12-04 13:09:22 -0500282 make_lib_name = maven_to_make[artifact_info.key][0]
283 make_dir_name = maven_to_make[artifact_info.key][1]
Alan Viveretted4527e62017-05-11 15:03:25 -0400284
Alan Viverette129555e2018-01-30 09:57:57 -0500285 artifact_file = os.path.join(new_dir, artifact_info.file)
286
Dan Willemsen814152e2017-11-06 13:22:11 -0800287 if extract_res:
Dan Willemsen814152e2017-11-06 13:22:11 -0800288 target_dir = os.path.join(working_dir, make_dir_name)
289 if not os.path.exists(target_dir):
290 os.makedirs(target_dir)
291
292 if maven_lib_type == "aar":
Alan Viverettec960cfb2017-12-04 13:09:22 -0500293 process_aar(artifact_file, target_dir)
Alan Viveretted4527e62017-05-11 15:03:25 -0400294
Alan Viveretted8ce7222017-12-11 17:24:43 -0500295 if maven_lib_type == "aar":
Dan Willemsend677b602017-11-08 22:13:17 -0800296 with zipfile.ZipFile(artifact_file) as zip:
Alan Viveretted8ce7222017-12-11 17:24:43 -0500297 manifests_dir = os.path.join(working_dir, "manifests")
298 zip.extract("AndroidManifest.xml", os.path.join(manifests_dir, make_lib_name))
Dan Willemsend677b602017-11-08 22:13:17 -0800299
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500300 print(maven_lib_vers, ":", maven_lib_name, "->", make_lib_name)
Alan Viveretted4527e62017-05-11 15:03:25 -0400301
Alan Viverette95f6d362017-04-06 09:40:50 -0400302
Alan Viverettec960cfb2017-12-04 13:09:22 -0500303def process_aar(artifact_file, target_dir):
Alan Viverette95f6d362017-04-06 09:40:50 -0400304 # Extract AAR file to target_dir.
305 with zipfile.ZipFile(artifact_file) as zip:
306 zip.extractall(target_dir)
307
Dan Willemsen814152e2017-11-06 13:22:11 -0800308 # Remove classes.jar
Alan Viverette95f6d362017-04-06 09:40:50 -0400309 classes_jar = os.path.join(target_dir, "classes.jar")
310 if os.path.exists(classes_jar):
Dan Willemsen814152e2017-11-06 13:22:11 -0800311 os.remove(classes_jar)
Alan Viverette95f6d362017-04-06 09:40:50 -0400312
313 # Remove or preserve empty dirs.
314 for root, dirs, files in os.walk(target_dir):
315 for dir in dirs:
316 dir_path = os.path.join(root, dir)
317 if not os.listdir(dir_path):
318 os.rmdir(dir_path)
319
320 # Remove top-level cruft.
321 for file in blacklist_files:
322 file_path = os.path.join(target_dir, file)
323 if os.path.exists(file_path):
324 os.remove(file_path)
325
Alan Viverettef17c9402017-07-19 12:57:40 -0400326
Alan Viverettecd3de262017-08-14 09:51:30 -0400327def fetch_artifact(target, build_id, artifact_path):
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500328 print('Fetching %s from %s...' % (artifact_path, target))
Alan Viverettecd3de262017-08-14 09:51:30 -0400329 fetch_cmd = [FETCH_ARTIFACT, '--bid', str(build_id), '--target', target, artifact_path]
Alan Viverette45837092017-05-12 14:50:53 -0400330 try:
Alan Viverettecd3de262017-08-14 09:51:30 -0400331 subprocess.check_output(fetch_cmd, stderr=subprocess.STDOUT)
Alan Viverette45837092017-05-12 14:50:53 -0400332 except subprocess.CalledProcessError:
Jeff Gaston13e38412018-02-06 14:45:36 -0500333 print_e('FAIL: Unable to retrieve %s artifact for build ID %s' % (artifact_path, build_id))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500334 print_e('Please make sure you are authenticated for build server access!')
Alan Viverette45837092017-05-12 14:50:53 -0400335 return None
336 return artifact_path
337
338
Alan Viverette129555e2018-01-30 09:57:57 -0500339def extract_artifact(artifact_path):
Alan Viverette45837092017-05-12 14:50:53 -0400340 # Unzip the repo archive into a separate directory.
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400341 repo_dir = os.path.basename(artifact_path)[:-4]
Alan Viverette45837092017-05-12 14:50:53 -0400342 with zipfile.ZipFile(artifact_path) as zipFile:
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400343 zipFile.extractall(repo_dir)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400344 return repo_dir
345
346
Alan Viverette129555e2018-01-30 09:57:57 -0500347def fetch_and_extract(target, build_id, file):
348 artifact_path = fetch_artifact(target, build_id, file)
349 if not artifact_path:
350 return None
351 return extract_artifact(artifact_path)
352
353
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400354def update_support(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500355 repo_file = 'top-of-tree-m2repository-%s.zip' % build_id.fs_id
356 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400357 if not repo_dir:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500358 print_e('Failed to extract Support Library repository')
Alan Viverettef48d9b42017-08-17 14:45:46 -0400359 return False
Alan Viverette45837092017-05-12 14:50:53 -0400360
361 # Transform the repo archive into a Makefile-compatible format.
Alan Viveretted8ce7222017-12-11 17:24:43 -0500362 return transform_maven_repo([repo_dir], support_dir)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500363
Alan Viverettec960cfb2017-12-04 13:09:22 -0500364def update_toolkit(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500365 repo_dir = fetch_and_extract(target, build_id.url_id, 'top-of-tree-m2repository-%s.zip' % build_id.fs_id)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500366 if not repo_dir:
367 print_e('Failed to extract App Toolkit repository')
368 return False
369
370 # Transform the repo archive into a Makefile-compatible format.
371 return transform_maven_repo([repo_dir], os.path.join(extras_dir, 'app-toolkit'))
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400372
373
374def update_constraint(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500375 layout_dir = fetch_and_extract(target, build_id.url_id,
376 'com.android.support.constraint-constraint-layout-%s.zip' % build_id.fs_id)
377 solver_dir = fetch_and_extract(target, build_id.url_id,
378 'com.android.support.constraint-constraint-layout-solver-%s.zip' % build_id.fs_id)
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400379 if not layout_dir or not solver_dir:
Alan Viverettec960cfb2017-12-04 13:09:22 -0500380 print_e('Failed to extract Constraint Layout repositories')
Alan Viverettecd3de262017-08-14 09:51:30 -0400381 return False
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400382
383 # Passing False here is an inelegant solution, but it means we can replace
384 # the top-level directory without worrying about other child directories.
Alan Viverettec960cfb2017-12-04 13:09:22 -0500385 return transform_maven_repo([layout_dir, solver_dir],
386 os.path.join(extras_dir, 'constraint-layout'), extract_res=False)
Alan Viverette45837092017-05-12 14:50:53 -0400387
388
Alan Viverette129555e2018-01-30 09:57:57 -0500389def update_design(file):
390 design_dir = extract_artifact(file)
391 if not design_dir:
392 print_e('Failed to extract Design Library repositories')
393 return False
394
395 # Don't bother extracting resources -- this should only be used with AAPT2.
396 return transform_maven_repo([design_dir],
397 os.path.join(extras_dir, 'material-design'), extract_res=False)
398
399
Alan Viverette45837092017-05-12 14:50:53 -0400400def extract_to(zip_file, paths, filename, parent_path):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500401 zip_path = next(filter(lambda path: filename in path, paths))
Alan Viverette45837092017-05-12 14:50:53 -0400402 src_path = zip_file.extract(zip_path)
403 dst_path = path(parent_path, filename)
404 mv(src_path, dst_path)
405
406
Alan Viverettecd3de262017-08-14 09:51:30 -0400407def update_sdk_repo(target, build_id):
Alan Viverette45837092017-05-12 14:50:53 -0400408 platform = 'darwin' if 'mac' in target else 'linux'
Alan Viverettecd3de262017-08-14 09:51:30 -0400409 artifact_path = fetch_artifact(
Jeff Gaston13e38412018-02-06 14:45:36 -0500410 target, build_id.url_id, 'sdk-repo-%s-platforms-%s.zip' % (platform, build_id.fs_id))
Alan Viverette45837092017-05-12 14:50:53 -0400411 if not artifact_path:
Alan Viverettecd3de262017-08-14 09:51:30 -0400412 return False
Alan Viverette45837092017-05-12 14:50:53 -0400413
414 with zipfile.ZipFile(artifact_path) as zipFile:
415 paths = zipFile.namelist()
416
417 extract_to(zipFile, paths, 'android.jar', current_path)
418 extract_to(zipFile, paths, 'uiautomator.jar', current_path)
419 extract_to(zipFile, paths, 'framework.aidl', current_path)
Paul Duffind891f3c2017-12-06 11:32:56 +0000420 extract_to(zipFile, paths, 'optional/android.test.base.jar', current_path)
Paul Duffin2b1a1462017-07-14 16:21:33 +0100421 extract_to(zipFile, paths, 'optional/android.test.mock.jar', current_path)
422 extract_to(zipFile, paths, 'optional/android.test.runner.jar', current_path)
Alan Viverette45837092017-05-12 14:50:53 -0400423
424 # Unclear if this is actually necessary.
425 extract_to(zipFile, paths, 'framework.aidl', system_path)
Alan Viverettecd3de262017-08-14 09:51:30 -0400426 return True
Alan Viverette45837092017-05-12 14:50:53 -0400427
428
Alan Viverettecd3de262017-08-14 09:51:30 -0400429def update_system(target, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500430 artifact_path = fetch_artifact(target, build_id.url_id, 'android_system.jar')
Alan Viverette45837092017-05-12 14:50:53 -0400431 if not artifact_path:
Alan Viverettecd3de262017-08-14 09:51:30 -0400432 return False
Alan Viverette45837092017-05-12 14:50:53 -0400433
434 mv(artifact_path, path(system_path, 'android.jar'))
Paul Duffin960e1ee2017-12-21 08:50:14 +0000435
Jeff Gaston13e38412018-02-06 14:45:36 -0500436 artifact_path = fetch_artifact(target, build_id.url_id, 'android.test.mock.stubs_system.jar')
Paul Duffin960e1ee2017-12-21 08:50:14 +0000437 if not artifact_path:
438 return False
439
440 mv(artifact_path, path(system_path, 'optional/android.test.mock.jar'))
441
Alan Viverettecd3de262017-08-14 09:51:30 -0400442 return True
443
444
Alan Viverettec1c32b62017-12-20 09:40:36 -0500445def update_buildtools(target, arch, build_id):
Jeff Gaston13e38412018-02-06 14:45:36 -0500446 artifact_path = fetch_and_extract(target, build_id.url_id,
447 "sdk-repo-%s-build-tools-%s.zip" % (arch, build_id.fs_id))
Alan Viverettec1c32b62017-12-20 09:40:36 -0500448 if not artifact_path:
449 return False
450
451 top_level_dir = os.listdir(artifact_path)[0]
452 src_path = os.path.join(artifact_path, top_level_dir)
453 dst_path = path(buildtools_dir, arch)
454 mv(src_path, dst_path)
455
456 # Move all top-level files to /bin and make them executable
457 bin_path = path(dst_path, 'bin')
458 top_level_files = filter(lambda e: os.path.isfile(path(dst_path, e)), os.listdir(dst_path))
459 for file in top_level_files:
460 src_file = path(dst_path, file)
461 dst_file = path(bin_path, file)
462 mv(src_file, dst_file)
463 os.chmod(dst_file, 0o755)
464
465 # Remove renderscript
466 rm(path(dst_path, 'renderscript'))
467
468 return True
469
470
Alan Viverettecd3de262017-08-14 09:51:30 -0400471def append(text, more_text):
472 if text:
473 return "%s, %s" % (text, more_text)
474 return more_text
Alan Viverette45837092017-05-12 14:50:53 -0400475
Alan Viverette95f6d362017-04-06 09:40:50 -0400476
Jeff Gaston13e38412018-02-06 14:45:36 -0500477class buildId(object):
478 def __init__(self, url_id, fs_id):
479 # id when used in build server urls
480 self.url_id = url_id
481 # id when used in build commands
482 self.fs_id = fs_id
483
Jeff Gastonb9b09052018-02-05 14:16:05 -0500484def getBuildId(args):
Jeff Gaston13e38412018-02-06 14:45:36 -0500485 # must be in the format 12345 or P12345
Jeff Gastonb9b09052018-02-05 14:16:05 -0500486 source = args.source
Jeff Gaston13e38412018-02-06 14:45:36 -0500487 number_text = source[:]
488 presubmit = False
489 if number_text.startswith("P"):
490 presubmit = True
491 number_text = number_text[1:]
492 if not number_text.isnumeric():
493 raise Exception('Updating this set of prebuilts requires <source> to be a build id, not "' + source + '"')
494 url_id = source
495 fs_id = url_id
496 if presubmit:
497 fs_id = "0"
498 args.file = False
499 return buildId(url_id, fs_id)
Jeff Gastonb9b09052018-02-05 14:16:05 -0500500
Alan Viveretted4527e62017-05-11 15:03:25 -0400501parser = argparse.ArgumentParser(
Alan Viverette45837092017-05-12 14:50:53 -0400502 description=('Update current prebuilts'))
Alan Viveretted4527e62017-05-11 15:03:25 -0400503parser.add_argument(
Alan Viverette129555e2018-01-30 09:57:57 -0500504 'source',
505 help='Build server build ID or local Maven ZIP file')
506parser.add_argument(
507 '-d', '--design', action="store_true",
508 help='If specified, updates only the Design Library')
Alan Viveretted4527e62017-05-11 15:03:25 -0400509parser.add_argument(
Alan Viverette3e57a4a2017-08-11 15:49:47 -0400510 '-c', '--constraint', action="store_true",
511 help='If specified, updates only Constraint Layout')
512parser.add_argument(
Alan Viverette5bfe05b2017-06-06 14:21:29 -0400513 '-s', '--support', action="store_true",
514 help='If specified, updates only the Support Library')
Alan Viverette45837092017-05-12 14:50:53 -0400515parser.add_argument(
Alan Viverettec960cfb2017-12-04 13:09:22 -0500516 '-t', '--toolkit', action="store_true",
517 help='If specified, updates only the App Toolkit')
518parser.add_argument(
Alan Viverette5bfe05b2017-06-06 14:21:29 -0400519 '-p', '--platform', action="store_true",
520 help='If specified, updates only the Android Platform')
Alan Viverettec1c32b62017-12-20 09:40:36 -0500521parser.add_argument(
522 '-b', '--buildtools', action="store_true",
523 help='If specified, updates only the Build Tools')
Alan Viveretted4527e62017-05-11 15:03:25 -0400524args = parser.parse_args()
Jeff Gaston56fe2762018-02-06 14:54:37 -0500525args.file = True
Alan Viverette129555e2018-01-30 09:57:57 -0500526if not args.source:
527 parser.error("You must specify a build ID or local Maven ZIP file")
Alan Viveretted4527e62017-05-11 15:03:25 -0400528 sys.exit(1)
Alan Viverette129555e2018-01-30 09:57:57 -0500529if not (args.support or args.platform or args.constraint or args.toolkit or args.buildtools \
530 or args.design):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500531 parser.error("You must specify at least one target to update")
Alan Viverettecd3de262017-08-14 09:51:30 -0400532 sys.exit(1)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500533if (args.support or args.constraint or args.toolkit) and which('pom2mk') is None:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500534 parser.error("Cannot find pom2mk in path; please run lunch to set up build environment")
535 sys.exit(1)
Alan Viveretted4527e62017-05-11 15:03:25 -0400536
537try:
538 # Make sure we don't overwrite any pending changes.
539 subprocess.check_call(['git', 'diff', '--quiet', '--', '**'])
540 subprocess.check_call(['git', 'diff', '--quiet', '--cached', '--', '**'])
541except subprocess.CalledProcessError:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500542 print_e('FAIL: There are uncommitted changes here; please revert or stash')
Alan Viveretted4527e62017-05-11 15:03:25 -0400543 sys.exit(1)
544
545try:
Alan Viverettecd3de262017-08-14 09:51:30 -0400546 components = None
547 if args.constraint:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500548 if update_constraint('studio', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400549 components = append(components, 'Constraint Layout')
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500550 print_e('Failed to update Constraint Layout, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400551 else:
552 sys.exit(1)
553 if args.support:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500554 if update_support('support_library', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400555 components = append(components, 'Support Library')
556 else:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500557 print_e('Failed to update Support Library, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400558 sys.exit(1)
Alan Viverettec960cfb2017-12-04 13:09:22 -0500559 if args.toolkit:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500560 if update_toolkit('support_library_app_toolkit', getBuildId(args)):
Alan Viverettec960cfb2017-12-04 13:09:22 -0500561 components = append(components, 'App Toolkit')
562 else:
563 print_e('Failed to update App Toolkit, aborting...')
564 sys.exit(1)
Alan Viverettecd3de262017-08-14 09:51:30 -0400565 if args.platform:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500566 if update_sdk_repo('sdk_phone_armv7-sdk_mac', getBuildId(args)) \
567 and update_system('sdk_phone_armv7-sdk_mac', getBuildId(args)):
Alan Viverettecd3de262017-08-14 09:51:30 -0400568 components = append(components, 'platform SDK')
569 else:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500570 print_e('Failed to update platform SDK, aborting...')
Alan Viverettecd3de262017-08-14 09:51:30 -0400571 sys.exit(1)
Alan Viverette129555e2018-01-30 09:57:57 -0500572 if args.design:
573 if not args.file:
574 print_e('Design Library must have --file specified')
575 sys.exit(1)
576 elif update_design(args.file):
577 components = append(components, 'Design Library')
578 else:
579 print_e('Failed to update platform SDK, aborting...')
580 sys.exit(1)
Alan Viverettec1c32b62017-12-20 09:40:36 -0500581 if args.buildtools:
Jeff Gastonb9b09052018-02-05 14:16:05 -0500582 if update_buildtools('sdk_phone_armv7-sdk_mac', 'darwin', getBuildId(args)) \
583 and update_buildtools('sdk_phone_x86_64-sdk', 'linux', getBuildId(args)) \
584 and update_buildtools('sdk_phone_armv7-win_sdk', 'windows', getBuildId(args)):
Alan Viverettec1c32b62017-12-20 09:40:36 -0500585 components = append(components, 'build tools')
586 else:
587 print_e('Failed to update build tools, aborting...')
588 sys.exit(1)
Alan Viveretted4527e62017-05-11 15:03:25 -0400589
590 # Commit all changes.
Alan Viverette45837092017-05-12 14:50:53 -0400591 subprocess.check_call(['git', 'add', current_path])
592 subprocess.check_call(['git', 'add', system_path])
Alan Viverettec1c32b62017-12-20 09:40:36 -0500593 subprocess.check_call(['git', 'add', buildtools_dir])
Alan Viverette129555e2018-01-30 09:57:57 -0500594 if args.file:
595 src_msg = "local Maven ZIP"
596 else:
Jeff Gaston13e38412018-02-06 14:45:36 -0500597 src_msg = "build %s" % (getBuildId(args).url_id)
Alan Viverette129555e2018-01-30 09:57:57 -0500598 msg = "Import %s from %s\n\n%s" % (components, src_msg, flatten(sys.argv))
Alan Viveretted4527e62017-05-11 15:03:25 -0400599 subprocess.check_call(['git', 'commit', '-m', msg])
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500600 print('Remember to test this change before uploading it to Gerrit!')
Alan Viveretted4527e62017-05-11 15:03:25 -0400601
602finally:
603 # Revert all stray files, including the downloaded zip.
604 try:
605 with open(os.devnull, 'w') as bitbucket:
606 subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket)
607 subprocess.check_call(
608 ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!'], stdout=bitbucket)
609 subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket)
610 except subprocess.CalledProcessError:
Alan Viverettef5cb0e62017-11-17 15:15:23 -0500611 print_e('ERROR: Failed cleaning up, manual cleanup required!!!')