blob: 46d8e4483ebf7803934ac9a3dfe52e552f928450 [file] [log] [blame]
#!/usr/bin/python3
"""Updates prebuilt libraries used by Android builds.
For details on how to use this script, visit go/update-prebuilts.
"""
import os
import sys
import zipfile
import re
import argparse
import subprocess
import six
import shlex
from urllib import request
from shutil import which
from distutils.version import LooseVersion
from pathlib import Path
from maven import MavenLibraryInfo, GMavenArtifact, maven_path_for_artifact
from buildserver import fetch_and_extract, fetch_artifacts, fetch_artifact, extract_artifact, \
parse_build_id
from utils import print_e, append, cp, mv, rm
current_path = 'current'
framework_sdk_target = 'sdk'
androidx_dir = os.path.join(current_path, 'androidx')
androidx_owners = os.path.join(androidx_dir, 'OWNERS')
java_plugins_bp_path = os.path.join(androidx_dir, 'JavaPlugins.bp')
test_mapping_file = os.path.join(androidx_dir, 'TEST_MAPPING')
compose_test_mapping_file = os.path.join(androidx_dir, 'm2repository/androidx/compose/TEST_MAPPING')
gmaven_dir = os.path.join(current_path, 'gmaven')
extras_dir = os.path.join(current_path, 'extras')
buildtools_dir = 'tools'
jetifier_dir = os.path.join(buildtools_dir, 'jetifier', 'jetifier-standalone')
repo_root_dir = Path(sys.argv[0]).resolve().parents[3]
extension_sdk_finalization_cmd = '%s -r "{readme}" -b {bug} -f {extension_version} {build_id}' % (
"packages/modules/common/tools/finalize_sdk.py"
)
temp_dir = os.path.join(os.getcwd(), 'support_tmp')
os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))))
git_dir = os.getcwd()
# Leave map blank to automatically populate name and path:
# - Name format is MAVEN.replaceAll(':','_')
# - Path format is MAVEN.replaceAll(':','/').replaceAll('.','/')
maven_to_make = {
# AndroidX
'androidx.benchmark:benchmark-macro': {},
'androidx.benchmark:benchmark-macro-junit4': {},
'androidx.benchmark:benchmark-common': {},
'androidx.benchmark:benchmark-junit4': {},
'androidx.tracing:tracing': {},
'androidx.tracing:tracing-perfetto': {},
'androidx.tracing:tracing-perfetto-binary': {},
'androidx.tracing:tracing-perfetto-common': {},
'androidx.tracing:tracing-ktx': {},
'androidx.slice:slice-builders': {},
'androidx.slice:slice-core': {},
'androidx.slice:slice-view': {},
'androidx.remotecallback:remotecallback': {},
'androidx.remotecallback:remotecallback-processor': {
'host': True
},
'androidx.versionedparcelable:versionedparcelable': {},
'androidx.vectordrawable:vectordrawable-animated': {},
'androidx.activity:activity': {},
'androidx.activity:activity-ktx': {},
'androidx.annotation:annotation': {
'host_and_device': True,
'extra-static-libs': {
'androidx.annotation_annotation-jvm'
}
},
'androidx.annotation:annotation-jvm': {
'host_and_device': True
},
'androidx.annotation:annotation-experimental': {},
'androidx.asynclayoutinflater:asynclayoutinflater': {},
'androidx.collection:collection': {
'extra-static-libs': {
'androidx.collection_collection-jvm'
}
},
'androidx.collection:collection-ktx': {},
'androidx.collection:collection-jvm': {},
'androidx.concurrent:concurrent-futures': {},
'androidx.concurrent:concurrent-listenablefuture-callback': {},
'androidx.concurrent:concurrent-listenablefuture': {},
'androidx.core:core': {},
'androidx.core:core-animation': {},
'androidx.core:core-ktx': {},
'androidx.core.uwb:uwb': {},
'androidx.core.uwb:uwb-rxjava3': {},
'androidx.contentpaging:contentpaging': {},
'androidx.coordinatorlayout:coordinatorlayout': {},
'androidx.legacy:legacy-support-core-ui': {},
'androidx.legacy:legacy-support-core-utils': {},
'androidx.cursoradapter:cursoradapter': {},
'androidx.browser:browser': {},
'androidx.customview:customview': {},
'androidx.customview:customview-poolingcontainer': {},
'androidx.documentfile:documentfile': {},
'androidx.drawerlayout:drawerlayout': {},
'androidx.dynamicanimation:dynamicanimation': {},
'androidx.emoji:emoji': {},
'androidx.emoji:emoji-appcompat': {},
'androidx.emoji:emoji-bundled': {},
'androidx.emoji2:emoji2': {},
'androidx.emoji2:emoji2-views-helper': {},
'androidx.exifinterface:exifinterface': {},
'androidx.fragment:fragment': {},
'androidx.fragment:fragment-ktx': {},
'androidx.heifwriter:heifwriter': {},
'androidx.interpolator:interpolator': {},
'androidx.loader:loader': {},
'androidx.media:media': {},
'androidx.media2:media2-player': {},
'androidx.media2:media2-session': {},
'androidx.media2:media2-common': {},
'androidx.media2:media2-exoplayer': {},
'androidx.media2:media2-widget': {},
'androidx.navigation:navigation-common': {},
'androidx.navigation:navigation-common-ktx': {},
'androidx.navigation:navigation-fragment': {},
'androidx.navigation:navigation-fragment-ktx': {},
'androidx.navigation:navigation-runtime': {},
'androidx.navigation:navigation-runtime-ktx': {},
'androidx.navigation:navigation-ui': {},
'androidx.navigation:navigation-ui-ktx': {},
'androidx.percentlayout:percentlayout': {},
'androidx.print:print': {},
'androidx.recommendation:recommendation': {},
'androidx.recyclerview:recyclerview-selection': {},
'androidx.savedstate:savedstate': {},
'androidx.savedstate:savedstate-ktx': {},
'androidx.slidingpanelayout:slidingpanelayout': {},
'androidx.swiperefreshlayout:swiperefreshlayout': {},
'androidx.textclassifier:textclassifier': {},
'androidx.transition:transition': {},
'androidx.tvprovider:tvprovider': {},
'androidx.legacy:legacy-support-v13': {},
'androidx.legacy:legacy-preference-v14': {},
'androidx.leanback:leanback': {},
'androidx.leanback:leanback-grid': {},
'androidx.leanback:leanback-preference': {},
'androidx.legacy:legacy-support-v4': {},
'androidx.appcompat:appcompat': {},
'androidx.appcompat:appcompat-resources': {},
'androidx.cardview:cardview': {},
'androidx.gridlayout:gridlayout': {},
'androidx.mediarouter:mediarouter': {},
'androidx.palette:palette': {},
'androidx.preference:preference': {},
'androidx.recyclerview:recyclerview': {},
'androidx.vectordrawable:vectordrawable': {},
'androidx.viewpager:viewpager': {},
'androidx.viewpager2:viewpager2': {},
'androidx.wear:wear': {},
'androidx.wear:wear-ongoing': {},
'androidx.javascriptengine:javascriptengine': {},
'androidx.webkit:webkit': {},
'androidx.biometric:biometric': {},
'androidx.autofill:autofill': {},
'androidx.appsearch:appsearch': {},
'androidx.appsearch:appsearch-builtin-types': {},
'androidx.appsearch:appsearch-compiler': {
'name': 'androidx.appsearch_appsearch-compiler',
'host': True
},
'androidx.appsearch:appsearch-local-storage': {
'name': 'androidx.appsearch_appsearch_local_storage'
},
'androidx.appsearch:appsearch-platform-storage': {},
'androidx.car.app:app': {},
'androidx.car.app:app-automotive': {},
'androidx.car.app:app-testing': {},
'androidx.startup:startup-runtime': {},
'androidx.window:window': {
'optional-uses-libs': {
'androidx.window.extensions',
'androidx.window.sidecar'
}
},
'androidx.resourceinspection:resourceinspection-annotation': {},
'androidx.profileinstaller:profileinstaller': {},
'androidx.test.uiautomator:uiautomator': {},
# AndroidX for Compose
'androidx.compose.compiler:compiler-hosted': {
'host': True
},
'androidx.compose.runtime:runtime': {},
'androidx.compose.runtime:runtime-saveable': {},
'androidx.compose.runtime:runtime-livedata': {},
'androidx.compose.foundation:foundation': {},
'androidx.compose.foundation:foundation-layout': {},
'androidx.compose.foundation:foundation-text': {},
'androidx.compose.ui:ui': {},
'androidx.compose.ui:ui-geometry': {},
'androidx.compose.ui:ui-graphics': {},
'androidx.compose.ui:ui-text': {},
'androidx.compose.ui:ui-tooling': {},
'androidx.compose.ui:ui-tooling-preview': {},
'androidx.compose.ui:ui-tooling-data': {},
'androidx.compose.ui:ui-unit': {},
'androidx.compose.ui:ui-util': {},
'androidx.compose.ui:ui-test': { },
'androidx.compose.ui:ui-test-junit4': { },
'androidx.compose.ui:ui-test-manifest': { },
'androidx.compose.animation:animation-core': {},
'androidx.compose.animation:animation': {},
'androidx.compose.material:material-icons-core': {},
'androidx.compose.material:material-icons-extended': { },
'androidx.compose.material:material-ripple': {},
'androidx.compose.material:material': {},
'androidx.compose.material3:material3': {},
'androidx.activity:activity-compose': {},
'androidx.navigation:navigation-compose': { },
'androidx.lifecycle:lifecycle-viewmodel-compose': { },
# AndroidX for Multidex
'androidx.multidex:multidex': {},
'androidx.multidex:multidex-instrumentation': {},
# AndroidX for Constraint Layout
'androidx.constraintlayout:constraintlayout': {
'name': 'androidx-constraintlayout_constraintlayout'
},
'androidx.constraintlayout:constraintlayout-solver': {
'name': 'androidx-constraintlayout_constraintlayout-solver'
},
# AndroidX for Architecture Components
'androidx.arch.core:core-common': {},
'androidx.arch.core:core-runtime': {},
'androidx.lifecycle:lifecycle-common': {},
'androidx.lifecycle:lifecycle-common-java8': {},
'androidx.lifecycle:lifecycle-extensions': {},
'androidx.lifecycle:lifecycle-livedata': {},
'androidx.lifecycle:lifecycle-livedata-ktx': {},
'androidx.lifecycle:lifecycle-livedata-core': {},
'androidx.lifecycle:lifecycle-livedata-core-ktx': {},
'androidx.lifecycle:lifecycle-process': {},
'androidx.lifecycle:lifecycle-runtime': {},
'androidx.lifecycle:lifecycle-runtime-ktx': {},
'androidx.lifecycle:lifecycle-service': {},
'androidx.lifecycle:lifecycle-viewmodel': {},
'androidx.lifecycle:lifecycle-viewmodel-ktx': {},
'androidx.lifecycle:lifecycle-viewmodel-savedstate': {},
'androidx.paging:paging-common': {},
'androidx.paging:paging-common-ktx': {},
'androidx.paging:paging-runtime': {},
'androidx.sqlite:sqlite': {},
'androidx.sqlite:sqlite-framework': {},
'androidx.room:room-common': {
'host_and_device': True
},
'androidx.room:room-compiler': {
'host': True,
'extra-static-libs': {
'guava-21.0'
}
},
'androidx.room:room-migration': {
'host_and_device': True
},
'androidx.room:room-runtime': {},
'androidx.room:room-testing': {},
'androidx.room:room-compiler-processing': {
'host': True
},
'androidx.work:work-runtime': {},
'androidx.work:work-runtime-ktx': {},
'androidx.work:work-testing': {},
# Third-party dependencies
'com.google.android:flexbox': {
'name': 'flexbox',
'path': 'flexbox'
},
# Androidx Material Design Components
'com.google.android.material:material': {},
}
# Mapping of POM dependencies to Soong build targets
deps_rewrite = {
'auto-common': 'auto_common',
'auto-value-annotations': 'auto_value_annotations',
'com.google.auto.value:auto-value': 'libauto_value_plugin',
'monitor': 'androidx.test.monitor',
'rules': 'androidx.test.rules',
'runner': 'androidx.test.runner',
'androidx.test:core': 'androidx.test.core',
'com.squareup:javapoet': 'javapoet',
'com.google.guava:listenablefuture': 'guava-listenablefuture-prebuilt-jar',
'sqlite-jdbc': 'xerial-sqlite-jdbc',
'com.intellij:annotations': 'jetbrains-annotations',
'javax.annotation:javax.annotation-api': 'javax-annotation-api-prebuilt-host-jar',
'org.robolectric:robolectric': 'Robolectric_all-target',
'org.jetbrains.kotlin:kotlin-stdlib-common': 'kotlin-stdlib',
'org.jetbrains.kotlinx:kotlinx-coroutines-core': 'kotlinx_coroutines',
'org.jetbrains.kotlinx:kotlinx-coroutines-android': 'kotlinx_coroutines_android',
'org.jetbrains.kotlinx:kotlinx-coroutines-test':'kotlinx_coroutines_test',
'org.jetbrains.kotlinx:kotlinx-metadata-jvm': 'kotlinx_metadata_jvm',
'androidx.test.espresso:espresso-core':'androidx.test.espresso.core',
'androidx.test.espresso:espresso-idling-resource':'androidx.test.espresso.idling-resource',
}
# List of artifacts that will be updated from GMaven
# Use pattern: `group:library:version:extension`
# e.g.:
# androidx.appcompat:appcompat:1.2.0:aar
# Use `latest` to always fetch the latest version.
# e.g.:
# androidx.appcompat:appcompat:latest:aar
# Also make sure you add `group:library`:{} to maven_to_make as well.
gmaven_artifacts = {}
# Always remove these files.
denylist_files = [
'annotations.zip',
'public.txt',
'R.txt',
'AndroidManifest.xml',
os.path.join('libs', 'noto-emoji-compat-java.jar')
]
artifact_pattern = re.compile(r'^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$')
def name_for_artifact(group_artifact):
"""Returns the build system target name for a given library's Maven coordinate.
Args:
group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core
Returns:
The build system target name for the artifact, ex. androidx.core_core.
"""
return group_artifact.replace(':', '_')
def path_for_artifact(group_artifact):
"""Returns the file system path for a given library's Maven coordinate.
Args:
group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core
Returns:
The file system path for the artifact, ex. androidx/core/core.
"""
return group_artifact.replace('.', '/').replace(':', '/')
def populate_maven_to_make(mapping):
"""Modifies the input mapping by expanding Maven coordinate keys into build target names and
paths.
Args:
mapping: a map where the keys are Maven coordinates
"""
for key in mapping:
if 'name' not in mapping[key]:
mapping[key]['name'] = name_for_artifact(key)
if 'path' not in maven_to_make[key]:
mapping[key]['path'] = path_for_artifact(key)
def detect_artifacts(maven_repo_dirs):
"""Parses Maven libraries from the specified directories.
Args:
maven_repo_dirs: a list of maven repository roots
Returns:
A map of Maven coordinate keys to MavenLibraryInfo objects parsed from POM files.
"""
maven_lib_info = {}
# Find the latest revision for each artifact, remove others
for repo_dir in maven_repo_dirs:
for root, dirs, files in os.walk(repo_dir):
for file in files:
if file[-4:] == '.pom':
# Read the POM (hack hack hack).
group_id = ''
artifact_id = ''
version = ''
file = os.path.join(root, file)
with open(file) as pom_file:
for line in pom_file:
if line[:11] == ' <groupId>':
group_id = line[11:-11]
elif line[:14] == ' <artifactId>':
artifact_id = line[14:-14]
elif line[:11] == ' <version>':
version = line[11:-11]
if group_id == '' or artifact_id == '' or version == '':
print_e('Failed to find Maven artifact data in ' + file)
continue
# Locate the artifact.
artifact_file = file[:-4]
if os.path.exists(artifact_file + '.jar'):
artifact_file = artifact_file + '.jar'
elif os.path.exists(artifact_file + '.aar'):
artifact_file = artifact_file + '.aar'
else:
# This error only occurs for a handful of gradle.plugin artifacts that only
# ship POM files, so we probably don't need to log unless we're debugging.
# print_e('Failed to find artifact for ' + artifact_file)
continue
# Make relative to root.
artifact_file = artifact_file[len(root) + 1:]
# Find the mapping.
group_artifact = group_id + ':' + artifact_id
if group_artifact in maven_to_make:
key = group_artifact
elif artifact_id in maven_to_make:
key = artifact_id
else:
# No mapping entry, skip this library.
continue
# Store the latest version.
version = LooseVersion(version)
if key not in maven_lib_info \
or version > maven_lib_info[key].version:
maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version,
root, repo_dir, artifact_file)
return maven_lib_info
def transform_maven_repos(maven_repo_dirs, transformed_dir, extract_res=True,
include_static_deps=True, include=None, exclude=None, prepend=None):
"""Transforms a standard Maven repository to be compatible with the Android build system.
When using the include argument by itself, all other libraries will be excluded. When using the
exclude argument by itself, all other libraries will be included. When using both arguments, the
inclusion list will be applied followed by the exclusion list.
Args:
maven_repo_dirs: path to local Maven repository
transformed_dir: relative path for output, ex. androidx
extract_res: whether to extract Android resources like AndroidManifest.xml from AARs
include_static_deps: whether to pass --static-deps to pom2bp
include: list of Maven groupIds or unversioned artifact coordinates to include for
updates, ex. androidx.core or androidx.core:core
exclude: list of Maven groupIds or unversioned artifact coordinates to exclude from
updates, ex. androidx.core or androidx.core:core
prepend: Path to a file containing text to be inserted at the beginning of the generated
build file
Returns:
True if successful, false otherwise.
"""
if exclude is None:
exclude = []
if include is None:
include = []
cwd = os.getcwd()
local_repo = os.path.join(cwd, transformed_dir)
working_dir = temp_dir
# Handle inclusions by stashing the remote artifacts for the inclusions, replacing the entire
# remote repo with the local repo, then restoring the stashed artifacts.
for remote_repo in maven_repo_dirs:
remote_repo = os.path.join(cwd, remote_repo)
paths_to_copy = []
for group_artifact in include:
artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact))
remote_path = os.path.join(remote_repo, artifact_path)
working_path = os.path.join(working_dir, artifact_path)
if os.path.exists(remote_path):
print(f'Included {group_artifact} in update')
paths_to_copy.append([remote_path, working_path])
# Move included artifacts from repo to temp.
for [remote_path, working_path] in paths_to_copy:
mv(remote_path, working_path)
# Replace all remaining artifacts in remote repo with local repo.
cp(local_repo, remote_repo)
# Restore included artifacts to remote repo.
for [remote_path, working_path] in paths_to_copy:
mv(working_path, remote_path)
# Handle exclusions by replacing the remote artifacts for the exclusions with local artifacts.
# This must happen before we parse the artifacts.
for remote_repo in maven_repo_dirs:
for group_artifact in exclude:
artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact))
remote_path = os.path.join(remote_repo, artifact_path)
if os.path.exists(remote_path):
rm(remote_path)
local_path = os.path.join(local_repo, artifact_path)
if os.path.exists(local_path):
print(f'Excluded {group_artifact} from update, used local artifact')
mv(local_path, remote_path)
else:
print(f'Excluded {group_artifact} from update, no local artifact present')
# Parse artifacts.
maven_lib_info = detect_artifacts(maven_repo_dirs)
if not maven_lib_info:
print_e('Failed to detect artifacts')
return False
# Move libraries into the working directory, performing any necessary transformations.
for info in maven_lib_info.values():
transform_maven_lib(working_dir, info, extract_res)
# Generate a single Android.bp that specifies to use all of the above artifacts.
makefile = os.path.join(working_dir, 'Android.bp')
with open(makefile, 'w') as f:
args = ['pom2bp']
args.extend(['-sdk-version', '31'])
args.extend(['-default-min-sdk-version', '24'])
if include_static_deps:
args.append('-static-deps')
if prepend:
args.append(f'-prepend={prepend}')
rewrite_names = sorted(maven_to_make.keys())
args.extend([f'-rewrite=^{name}$={maven_to_make[name]["name"]}' for name in rewrite_names])
args.extend([f'-rewrite=^{key}$={value}' for key, value in deps_rewrite.items()])
args.extend(["-extra-static-libs=" + maven_to_make[name]['name'] + "=" + ",".join(
sorted(maven_to_make[name]['extra-static-libs'])) for name in maven_to_make if
'extra-static-libs' in maven_to_make[name]])
args.extend(["-optional-uses-libs=" + maven_to_make[name]['name'] + "=" + ",".join(
sorted(maven_to_make[name]['optional-uses-libs'])) for name in maven_to_make if
'optional-uses-libs' in maven_to_make[name]])
args.extend([f'-host={name}' for name in maven_to_make
if maven_to_make[name].get('host')])
args.extend([f'-host-and-device={name}' for name in maven_to_make
if maven_to_make[name].get('host_and_device')])
args.extend(['.'])
subprocess.check_call(args, stdout=f, cwd=working_dir)
# Replace the old directory.
local_repo = os.path.join(cwd, transformed_dir)
mv(working_dir, local_repo)
return True
def transform_maven_lib(working_dir, artifact_info, extract_res):
"""Transforms the specified artifact for use in the Android build system.
Moves relevant files for the artifact represented by artifact_info of type MavenLibraryInfo into
the appropriate path inside working_dir, unpacking files needed by the build system from AARs.
Args:
working_dir: The directory into which the artifact should be moved
artifact_info: A MavenLibraryInfo representing the library artifact
extract_res: True to extract resources from AARs, false otherwise.
"""
# Move library into working dir
new_dir = os.path.normpath(
os.path.join(working_dir, os.path.relpath(artifact_info.dir, artifact_info.repo_dir)))
mv(artifact_info.dir, new_dir)
maven_lib_type = os.path.splitext(artifact_info.file)[1][1:]
group_artifact = artifact_info.key
make_lib_name = maven_to_make[group_artifact]['name']
make_dir_name = maven_to_make[group_artifact]['path']
artifact_file = os.path.join(new_dir, artifact_info.file)
if maven_lib_type == 'aar':
if extract_res:
target_dir = os.path.join(working_dir, make_dir_name)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
process_aar(artifact_file, target_dir)
with zipfile.ZipFile(artifact_file) as zip_file:
manifests_dir = os.path.join(working_dir, 'manifests')
zip_file.extract('AndroidManifest.xml', os.path.join(manifests_dir, make_lib_name))
def process_aar(artifact_file, target_dir):
"""Extracts and cleans up the contents of an AAR file to the specified directory.
Removes classes.jar, empty directories, and denylisted files.
Args:
artifact_file: path to the AAR to extract
target_dir: directory into which the contents should be extracted
"""
# Extract AAR file to target_dir.
with zipfile.ZipFile(artifact_file) as zip_file:
zip_file.extractall(target_dir)
# Remove classes.jar
classes_jar = os.path.join(target_dir, 'classes.jar')
if os.path.exists(classes_jar):
os.remove(classes_jar)
# Remove empty dirs.
for root, dirs, files in os.walk(target_dir, topdown=False):
for dir_name in dirs:
dir_path = os.path.join(root, dir_name)
if not os.listdir(dir_path):
os.rmdir(dir_path)
# Remove top-level cruft.
for file in denylist_files:
file_path = os.path.join(target_dir, file)
if os.path.exists(file_path):
os.remove(file_path)
def fetch_gmaven_artifact(artifact):
"""Fetch a GMaven artifact.
Downloads a GMaven artifact
(https://developer.android.com/studio/build/dependencies#gmaven-access)
Args:
artifact: an instance of GMavenArtifact.
"""
pom_path = maven_path_for_artifact(
'gmaven', artifact.group, artifact.library, artifact.version, 'pom')
artifact_path = maven_path_for_artifact(
'gmaven', artifact.group, artifact.library, artifact.version, artifact.ext)
download_file_to_disk(artifact.get_pom_file_url(), pom_path)
download_file_to_disk(artifact.get_artifact_url(), artifact_path)
return os.path.dirname(artifact_path)
def download_file_to_disk(url, filepath):
"""Download the file at URL to the location dictated by the path.
Args:
url: Remote URL to download file from.
filepath: Filesystem path to write the file to.
"""
print(f'Downloading URL: {url}')
file_data = request.urlopen(url)
try:
os.makedirs(os.path.dirname(filepath))
except os.error:
# This is a common situation - os.makedirs fails if dir already exists.
pass
try:
with open(filepath, 'wb') as f:
f.write(six.ensure_binary(file_data.read()))
except Exception as e:
print_e(e.__class__, 'occurred while reading', filepath)
os.remove(os.path.dirname(filepath))
raise
def update_gmaven(gmaven_artifacts_list):
artifacts = [GMavenArtifact(artifact) for artifact in gmaven_artifacts_list]
for artifact in artifacts:
if artifact.version == 'latest':
artifact.version = artifact.get_latest_version()
if not transform_maven_repos(['gmaven'], gmaven_dir, extract_res=False):
return []
return [artifact.key for artifact in artifacts]
def update_androidx(target, build_id, local_file, include, exclude, beyond_corp):
"""Fetches and extracts Jetpack library prebuilts.
Args:
target: Android build server target name, must be specified if local_file is empty
build_id: Optional Android build server ID, must be specified if local_file is empty
local_file: Optional local top-of-tree ZIP, must be specified if build_id is empty
include: List of Maven groupIds or unversioned artifact coordinates to include for
updates, ex. android.core or androidx.core:core
exclude: List of Maven groupIds or unversioned artifact coordinates to exclude from
updates, ex. android.core or androidx.core:core
beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher
Returns:
True if successful, false otherwise.
"""
if build_id:
repo_file = 'top-of-tree-m2repository-all-%s.zip' % build_id.fs_id
repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp, None)
else:
repo_dir = fetch_and_extract(target, None, None, beyond_corp, local_file)
if not repo_dir:
print_e('Failed to extract AndroidX repository')
return False
prepend_path = os.path.relpath('update_prebuilts/prepend_androidx_license', start=temp_dir)
# Transform the repo archive into a Makefile-compatible format.
if not transform_maven_repos([repo_dir], androidx_dir, extract_res=False, include=include,
exclude=exclude, prepend=prepend_path):
return False
# Import JavaPlugins.bp in Android.bp.
makefile = os.path.join(androidx_dir, 'Android.bp')
with open(makefile, 'a+') as f:
f.write('\nbuild = ["JavaPlugins.bp"]\n')
# Keep OWNERs file, JavaPlugins.bp file, and TEST_MAPPING files untouched.
files_to_restore = [androidx_owners, java_plugins_bp_path, test_mapping_file,
compose_test_mapping_file]
for file_to_restore in files_to_restore:
# Ignore any output or error - these files are not gauranteed to exist, but
# if they do, we want to restore them.
subprocess.call(['git', 'restore', file_to_restore],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return True
def update_jetifier(target, build_id, beyond_corp):
"""
Fetches and extracts Jetifier tool prebuilts.
Args:
target: Android build server target name
build_id: Android build server ID
beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher
Return:
Whether the prebuilt was successfully updated.
"""
repo_file = 'jetifier-standalone.zip'
repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp)
if not repo_dir:
print_e('Failed to extract Jetifier')
return False
rm(jetifier_dir)
mv(os.path.join(repo_dir, 'jetifier-standalone'), jetifier_dir)
os.chmod(os.path.join(jetifier_dir, 'bin', 'jetifier-standalone'), 0o755)
return True
def update_constraint(local_file):
"""
Extracts ConstraintLayout library prebuilts.
Args:
local_file: local Maven repository ZIP containing library artifacts
Return:
Whether the prebuilts were successfully updated.
"""
repo_dir = extract_artifact(local_file)
if not repo_dir:
print_e('Failed to extract Constraint Layout')
return False
return transform_maven_repos([repo_dir], os.path.join(extras_dir, 'constraint-layout-x'),
extract_res=False)
def update_material(local_file):
"""
Extracts Material Design Components library prebuilts.
Args:
local_file: local Maven repository ZIP containing library artifacts
Return:
Whether the prebuilts were successfully updated.
"""
design_dir = extract_artifact(local_file)
if not design_dir:
print_e('Failed to extract Material Design Components')
return False
return transform_maven_repos([design_dir], os.path.join(extras_dir, 'material-design-x'),
extract_res=False)
def update_framework(target, build_id, sdk_dir, beyond_corp):
api_scope_list = ['public', 'system', 'test', 'module-lib', 'system-server']
if sdk_dir == 'current':
api_scope_list.append('core')
for api_scope in api_scope_list:
target_dir = os.path.join(sdk_dir, api_scope)
if api_scope == 'core':
artifact_to_path = {'core.current.stubs.jar': os.path.join(target_dir, 'android.jar')}
else:
artifact_to_path = {
'apistubs/android/' + api_scope + '/*.jar': os.path.join(target_dir, '*'),
}
if api_scope == 'public' or api_scope == 'module-lib':
# Distinct core-for-system-modules.jar files are only provided
# for the public and module-lib API surfaces.
artifact_to_path[
'system-modules/' + api_scope + '/core-for-system-modules.jar'] = os.path.join(
target_dir, '*')
if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp):
return False
if api_scope == 'public':
# Fetch a few artifacts from the public sdk.
artifact = 'sdk-repo-linux-platforms-%s.zip' % build_id.fs_id
artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp)
if not artifact_path:
return False
with zipfile.ZipFile(artifact_path) as zipFile:
extra_files = [
'android.jar',
'framework.aidl',
'uiautomator.jar',
'data/annotations.zip',
'data/api-versions.xml']
for filename in extra_files:
matches = list(filter(lambda path: filename in path, zipFile.namelist()))
if len(matches) != 1:
print_e('Expected 1 file named \'%s\' in zip %s, found %d' %
(filename, zipFile.filename, len(matches)))
return False
zip_path = matches[0]
src_path = zipFile.extract(zip_path)
dst_path = os.path.join(target_dir, filename)
mv(src_path, dst_path)
# Filtered API DB is currently only available for "public"
fetch_artifacts(target, build_id, {'api-versions-public-filtered.xml': os.path.join(
target_dir, 'data/api-versions-filtered.xml')}, beyond_corp)
return True
def update_makefile(build_id):
template = '"%s",\n\
"current"'
makefile = os.path.join(git_dir, 'Android.bp')
with open(makefile, 'r+') as f:
contents = f.read().replace('"current"', template % build_id)
f.seek(0)
f.write(contents)
return True
def finalize_sdk(target, build_id, sdk_version, beyond_corp):
target_finalize_dir = '%d' % sdk_version
for api_scope in ['public', 'system', 'test', 'module-lib', 'system-server']:
artifact_to_path = {f'apistubs/android/{api_scope}/api/*.txt': os.path.join(
target_finalize_dir, api_scope, 'api', '*')}
if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp):
return False
return update_framework(target, build_id, target_finalize_dir, beyond_corp) and update_makefile(
target_finalize_dir)
def update_framework_current(target, build_id, beyond_corp):
return update_framework(target, build_id, current_path, beyond_corp)
def update_buildtools(target, arch, build_id, beyond_corp):
artifact_path = fetch_and_extract(target, build_id.url_id,
f'sdk-repo-{arch}-build-tools-{build_id.fs_id}.zip',
beyond_corp)
if not artifact_path:
return False
top_level_dir = os.listdir(artifact_path)[0]
src_path = os.path.join(artifact_path, top_level_dir)
dst_path = os.path.join(buildtools_dir, arch)
# There are a few libraries that have been manually added to the
# build tools, copy them from the destination back to the source
# before the destination is overwritten.
files_to_save = (
'lib64/libconscrypt_openjdk_jni.dylib',
'lib64/libconscrypt_openjdk_jni.so',
'bin/lib64/libwinpthread-1.dll',
)
for file in files_to_save:
src_file = os.path.join(dst_path, file)
dst_file = os.path.join(src_path, file)
if os.path.exists(dst_path):
mv(src_file, dst_file)
mv(src_path, dst_path)
# Move all top-level files to /bin and make them executable
bin_path = os.path.join(dst_path, 'bin')
top_level_files = filter(lambda e: os.path.isfile(os.path.join(dst_path, e)), os.listdir(dst_path))
for file in top_level_files:
src_file = os.path.join(dst_path, file)
dst_file = os.path.join(bin_path, file)
mv(src_file, dst_file)
os.chmod(dst_file, 0o755)
# Make the files under lld-bin executable
lld_bin_files = os.listdir(os.path.join(dst_path, 'lld-bin'))
for file in lld_bin_files:
os.chmod(os.path.join(dst_path, 'lld-bin', file), 0o755)
# Remove renderscript
rm(os.path.join(dst_path, 'renderscript'))
return True
def has_uncommitted_changes():
try:
# Make sure we don't overwrite any pending changes.
diff_command = f'cd {git_dir} && git diff --quiet'
subprocess.check_call(diff_command, shell=True)
subprocess.check_call(f'{diff_command} --cached', shell=True)
return False
except subprocess.CalledProcessError:
return True
def main():
parser = argparse.ArgumentParser(
description='Update current prebuilts')
parser.add_argument(
'source', nargs='?',
help='Build server build ID or local Maven ZIP file')
parser.add_argument(
'-m', '--material', action='store_true',
help='If specified, updates only Material Design Components')
parser.add_argument(
'-c', '--constraint', action='store_true',
help='If specified, updates only Constraint Layout')
parser.add_argument(
'-j', '--jetifier', action='store_true',
help='If specified, updates only Jetifier')
parser.add_argument(
'-p', '--platform', action='store_true',
help='If specified, updates only the Android Platform')
parser.add_argument(
'-f', '--finalize_sdk', type=int,
help='Finalize the build as the specified SDK version. Must be used together with -e')
parser.add_argument(
'-e', '--finalize_extension', type=int,
help='Finalize the build as the specified extension SDK version. Must be used together with -f')
parser.add_argument('--bug', type=int, help='The bug number to add to the commit message.')
parser.add_argument(
'--sdk_target',
default=framework_sdk_target,
help='If specified, the name of the build target from which to retrieve the SDK when -p or -f '
'is specified.')
parser.add_argument(
'-b', '--buildtools', action='store_true',
help='If specified, updates only the Build Tools')
parser.add_argument(
'-x', '--androidx', action='store_true',
help='If specified, updates only the Jetpack (androidx) libraries excluding those covered by '
'other arguments')
parser.add_argument(
'--include', action='append', default=[],
help='If specified with -x, includes the specified Jetpack library Maven group or artifact for '
'updates. Applied before exclude.')
parser.add_argument(
'--exclude', action='append', default=[],
help='If specified with -x, excludes the specified Jetpack library Maven group or artifact '
'from updates')
parser.add_argument(
'-g', '--gmaven', action='store_true',
help='If specified, updates only the artifact from GMaven libraries excluding those covered by '
'other arguments')
parser.add_argument(
'--commit-first', action='store_true',
help='If specified, then if uncommited changes exist, commit before continuing')
parser.add_argument(
'--beyond-corp', action='store_true',
help='If specified, then fetch artifacts with tooling that works on BeyondCorp devices')
rm(temp_dir)
args = parser.parse_args()
# Validate combinations of arguments.
if not args.source and (args.platform or args.buildtools or args.jetifier
or args.androidx or args.material or args.finalize_sdk
or args.constraint):
parser.error('You must specify a build ID or local Maven ZIP file')
sys.exit(1)
if not (args.gmaven or args.platform or args.buildtools or args.jetifier
or args.androidx or args.material or args.finalize_sdk
or args.finalize_extension or args.constraint):
parser.error('You must specify at least one target to update')
sys.exit(1)
if (args.finalize_sdk is None) != (args.finalize_extension is None):
parser.error('Either both or neither of -e and -f must be specified.')
sys.exit(1)
if args.finalize_sdk and not args.bug:
parser.error('Specifying a bug ID with --bug is required when finalizing an SDK.')
sys.exit(1)
# Validate the build environment for POM-dependent targets.
if (args.constraint or args.material or args.androidx or args.gmaven) \
and which('pom2bp') is None:
parser.error('Cannot find pom2bp in path; please run lunch to set up build environment. '
'You may also need to run \'m pom2bp\' if it hasn\'t been built already.')
sys.exit(1)
# Validate the git status.
if has_uncommitted_changes():
if args.commit_first:
subprocess.check_call(f'cd {git_dir} && git add -u', shell=True)
subprocess.check_call(f'cd {git_dir} && git commit -m \'save working state\'',
shell=True)
if has_uncommitted_changes():
self_file = os.path.basename(__file__)
print_e(f'FAIL: There are uncommitted changes here. Please commit or stash before '
f'continuing, because {self_file} will run "git reset --hard" if execution fails')
sys.exit(1)
if args.bug:
commit_msg_suffix = '\n\nBug: {args.bug}'
else:
commit_msg_suffix = ''
# Are we fetching a build ID or using a local file?
build_id = None
file = None
if args.source:
build_id = parse_build_id(args.source)
if build_id is None:
file = args.source
try:
components = []
if args.constraint:
if update_constraint(file):
components.append('Constraint Layout')
else:
print_e('Failed to update Constraint Layout, aborting...')
sys.exit(1)
if args.material:
if update_material(file):
components.append('Material Design Components')
else:
print_e('Failed to update Material Design Components, aborting...')
sys.exit(1)
if args.gmaven:
updated_artifacts = update_gmaven(gmaven_artifacts)
if updated_artifacts:
components.append('\n'.join(updated_artifacts))
else:
print_e('Failed to update GMaven, aborting...')
sys.exit(1)
if args.androidx:
if update_androidx('androidx', build_id, file, args.include, args.exclude,
args.beyond_corp):
components.append('AndroidX')
else:
print_e('Failed to update AndroidX, aborting...')
sys.exit(1)
if args.jetifier:
if update_jetifier('androidx', build_id, args.beyond_corp):
components.append('Jetifier')
else:
print_e('Failed to update Jetifier, aborting...')
sys.exit(1)
if args.platform or args.finalize_sdk:
if update_framework_current(args.sdk_target, build_id, args.beyond_corp):
components.append('platform SDK')
else:
print_e('Failed to update platform SDK, aborting...')
sys.exit(1)
if args.finalize_sdk:
n = args.finalize_sdk
if not finalize_sdk(args.sdk_target, build_id, n, args.beyond_corp):
print_e('Failed to finalize SDK %d, aborting...' % n)
sys.exit(1)
# We commit the finalized dir separately from the current sdk update.
msg = f'Import final sdk version {n} from build {build_id.url_id}{commit_msg_suffix}'
subprocess.check_call(['git', 'add', '%d' % n])
subprocess.check_call(['git', 'add', 'Android.bp'])
subprocess.check_call(['git', 'commit', '-m', msg])
# Finalize extension sdk level
readme = (f'- {args.finalize_extension}: Finalized together with '
'Android {args.finalize_sdk} (all modules)')
cmd = extension_sdk_finalization_cmd.format(
readme=readme,
bug=args.bug,
extension_version=args.finalize_extension,
build_id=build_id.url_id)
subprocess.check_call(shlex.split(cmd), cwd=repo_root_dir.resolve())
if args.buildtools:
if update_buildtools('sdk_mac', 'darwin', build_id, args.beyond_corp) \
and update_buildtools('sdk', 'linux', build_id, args.beyond_corp) \
and update_buildtools('sdk', 'windows', build_id, args.beyond_corp):
components.append('build tools')
else:
print_e('Failed to update build tools, aborting...')
sys.exit(1)
# Build the git commit.
subprocess.check_call(['git', 'add', current_path, buildtools_dir])
# Build the commit message.
components_msg = ', '.join(components)
argv_msg = ' '.join(sys.argv)
if not args.source and args.gmaven:
src_msg = 'GMaven'
elif not args.source.isnumeric():
src_msg = 'local Maven ZIP'
else:
src_msg = f'build {build_id.url_id}'
msg = f'Import {components_msg} from {src_msg}\n\n{argv_msg}{commit_msg_suffix}'
# Create the git commit.
subprocess.check_call(['git', 'commit', '-q', '-m', msg])
if args.finalize_sdk:
print('NOTE: Created three commits:')
subprocess.check_call(['git', 'log', '-3', '--oneline'])
else:
print('Created commit:')
subprocess.check_call(['git', 'log', '-1', '--oneline'])
print('Remember to test this change before uploading it to Gerrit!')
finally:
# Revert all stray files, including the downloaded zip.
try:
with open(os.devnull, 'w') as bitbucket:
subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket)
subprocess.check_call(
['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!', '--allow-empty'],
stdout=bitbucket)
subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket)
except subprocess.CalledProcessError:
print_e('ERROR: Failed cleaning up, manual cleanup required!!!')
# Add automatic entries to maven_to_make.
populate_maven_to_make(maven_to_make)
if __name__ == '__main__':
main()