blob: bcab7292dbfb14370fd72378efbde46daa586d38 [file] [log] [blame]
#!/bin/bash
# Copyright 2018-2023 Fairphone B.V.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
set -u
SHELL_LIBS="$(dirname "$(readlink -f "$0")")/../shell-libs"
export SHELL_LIBS
. "${SHELL_LIBS}/utils.sh"
# Default target Gerrit setup
export TARGET_GERRIT_NAME="fairphone"
export TARGET_GERRIT_URL="review.fairphone.software"
export TARGET_GERRIT_PORT="29418"
export TARGET_GERRIT_PARENT_PROJECT="Acl/Android"
# Default values for global parameters
export DEBUG=
export DRY_RUN=false
export NEW_TAG_ANNOTATION_MESSAGE=
export PUSH_BRANCH_PREFIX=
export PUSH_SKIP_VALIDATION=false
export TRY_QUALIFYING_REFS=false
# Default values for internal parameters
CREATE_PROJECTS=
EXCLUDE_PROJECTS=
FETCH_TARGET_BEFORE_PUSH=false
INIT_AS_MIRROR=false
MANIFEST=
MANIFEST_BRANCH=
MANIFEST_GROUPS=
MANIFEST_URL=
NEW_BRANCH=
NEW_TAG=
PUSH_CURRENT_REMOTE_REVISIONS=
PUSH_CURRENT_UPSTREAM_REVISION=
REMOTE_BRANCH=
REMOTE_TAG=
REMOTE_TAG_WITH_NEW_TAG_FALLBACK=
REPO_DEPTH=
REPO_NUM_JOBS=
REPO_SYNC_FORCE_SYNC=
REPO_URL=
SKIP_INIT=
SOURCE_FETCH_ALL=false
TARGET_BRANCH=
_usage() {
cat <<heredoc
Fetch branches and tags from a source remote gerrit and push to projects in a
target gerrit. Missing projects will be created if needed.
Locally defined branches and tags based on the remote's manifest project
revisions can also be created and pushed to the target remote.
Usage:
$0 [-u|--url MANIFEST_URL] [-b|--manifest-branch] [-m|--manifest MANIFEST]
[-nb|--new-branch NEW_BRANCH] [-rb|--remote-branch REMOTE_BRANCH TARGET_BRANCH]
[-nt|--new-tag NEW_TAG] [-rt|--remote-tag REMOTE_TAG]
[--push-current-remote-rev|--push-current-upstream-rev]
[--push-with-branch-prefix PUSH_BRANCH_PREFIX]
[-g|--groups "<group1>,..."] [-p|--projects "project1 project2"] [-d|--debug]
[-n|--dry-run] [-s|--skip-init] [--push-skip-validation] [--repo-url REPO_URL]
[--depth REPO_DEPTH] [--force-sync] [-j|--jobs REPO_NUM_JOBS] [-h|--help]
Options:
--mirror : Initialize as mirror. Don't checkout working
directories of the projects. See 'repo init
--mirror'. Has no effect on an existing
workspace.
-u, --url MANIFEST_URL : Remote manifest URL to initialize from.
Default to the current one if unset. See also
--skip-init.
-b, --manifest-branch MANIFEST_BRANCH
: Remote manifest branch or revision to
initialize from. Default to the current one if
unset. See also --skip-init.
-m, --manifest MANIFEST : Remote manifest file to initialize the tree
from. See also --skip-init.
-gn, --target-gerrit-name NAME : Git remote name of the target gerrit.
Default: "${TARGET_GERRIT_NAME}"
-gu, --target-gerrit-url URL : URL of the target gerrit. Only access via ssh
is supported at the moment.
Default: "${TARGET_GERRIT_URL}"
-gp, --target-gerrit-port PORT : Port of the target gerrit.
Default: ${TARGET_GERRIT_PORT}
-pp, --target-gerrit-parent-project PARENT_PROJECT
Parent for project that need to be created on
the target gerrit.
Default: "${TARGET_GERRIT_PARENT_PROJECT}"
-nb, --new-branch NEW_BRANCH : New branch to create and push.
-rb, --remote-branch REMOTE_BRANCH TARGET_BRANCH
: Names of remote branch to fetch and target
branch to push to target gerrit.
-nt, --new-tag NEW_TAG : Name of a new tag to create and push.
-rt, --remote-tag REMOTE_TAG : Name of remote tag to fetch and import.
--remote-tag-fallback-new-tag NEW_TAG
: Workaround for CodeLinaro: After migration
from CodeAurora, some Qualcomm public release
tags are not availale (yet). In that case,
create and push a new tag locally instead. Use
together with --new-tag-annotation-message.
--new-tag-annotation-message MSG: When creating new tags (see options above),
annotate it with '--annotate -m MSG'
-pc, --push-current-remote-rev : Push the remote revisions (tags or branchs)
that the manifest is currently pointing on per
project. This allows to push the current state
when different tags or branches are used
across projects.
See also --push-with-branch-prefix.
--push-current-upstream-rev : Push revisions that are listed as 'upstream'
in the manifest xml attributes. This allows
syncing from a release manifest (with
'revision' attribute pointing to commit
sha1's) that also defines upstream branches in
the 'upstream' attributes.
See also --push-with-branch-prefix.
--push-with-branch-prefix PUSH_BRANCH_PREFIX
: Prepend "PUSH_BRANCH_PREFIX" to branch names
when importing upstream branches. Allows
grouping various upstream under one naming
scheme that's easy to match, e.g., for Gerrit
ACLs. Example: "clo/" for CodeLinaro branches.
--try-qualifying-refs : When using "--push-current-remote-rev" and
manifest (remote) revisions are unqualified,
query remote heads and tags to find the
matching one. If both, branch and tag. with
matching name exist, fail the sync job.
-g, --groups "<group>,..." : Quote enclosed, comma separated, manifest
groups.
Default: unset (all but "notdefault" projects)
E.g. "all" or "pdk,pdk-qcom"
-p, --projects "project ..." : Quote enclosed, space separated, list of
projects to work on. When undefined script
will run commands on all projects allowed by
MANIFEST_GROUPS.
E.g. -p "cts dalvik"
--exclude-projects EXCLUDE_PROJECTS
: Skip projects matching a regex or wildcard
expression. These projects will still be
synchronized via "repo sync", but not created
on or pushed to the target Gerrit.
-cp, --create-projects : Create missing projects on the target Gerrit.
By default, the script assumes that all
relevant projects exist already.
--dry-run : Run git and ssh commands but do not modify
remote.
-d, --debug : Show commands being run. I.e. set -x.
-s, --skip-init : Skip 'repo init' and 'repo sync' commands.
Only set this if you are sure all projects
have been synced and point to the correct
remote and revision.
--push-skip-validation : Add '-o skip-validation' to git push calls.
This can be necessary when pushing new
histories with a lot of commits that get
otherwise rejected by Gerrit. Requires
extended permissions, see the Gerrit docs.
--repo-url : Pass a custom repo tool URL to repo init. If
set, --no-repo-verify will be passed to repo
init as well to accept unsigned commits in the
repo tool git history.
--depth : Use a custom default depth for \`repo init\`.
This is most useful with our custom parameter
value \`--depth -1\` in repo to force
unshallow checkouts of all projects.
--force-sync : Pass "--force-sync" to repo sync, overwriting
local git directories that need to point to a
difference repo object directory.
--source-fetch-all : Fetch all branches from the source Gerrit. By
Default, --current-branch is set on repo sync.
--fetch-target-before-push : Fetch from the target Gerrit before push. Can
help speeding up pushing large projects.
-j, --jobs : Number of jobs for repo sync and forall. Use
this to speed up overall job execution. This
can make logs unusable, so it's on 1 by
default.
-h, --help : Print this help message
Examples:
$0 -m aosp/aosp-9.0.0_r8.xml -rb pie-r2-s1-release aosp/pie-r2-s1-release \\
-rt android-9.0.0_r8
$0 -m caf/default_LA.UM.7.6.2.r1-03400-89xx.0.xml -nb staging/arima/fp3/p/r0
Publishing sources for a Fairphone Open release:
$0 -m default.xml -gn code_fp -gu gerrit-public.fairphone.software -gp 29418 \\
-pc --create-projects -pp "All-Projects"
$0 -m rel/p/fp2/20.10.1-beta/20.10.1-beta.0-public.xml -gn code_fp \\
-gu gerrit-public.fairphone.software -gp 29418 \\
-rb rel/p/fp2/20.10.1-beta rel/p/fp2/20.10.1-beta -rt 20.10.1-beta.0
heredoc
}
# Perform a repo init and sync on a specified remote manifest
#
# Globals:
# REPO_URL
# REPO_DEPTH
# REPO_NUM_JOBS
# Arguments:
# manifest_url Remote manifest URL. If empty, the current one in local repo
# tree will be used.
# manifest Remote manifest to use.
# groups repo groups to use.
# platform repo platform to use.
# Returns:
# None
_init_projects()
{
local init_params=()
local sync_params=()
if [ "${INIT_AS_MIRROR}" = true ]; then
if ! [ -d .repo ]; then
# repo mirror can only be set up on a new workspace.
init_params+=(--mirror)
elif [ -d .repo/projects ]; then
log_w "--mirror is set but workspace is already initialized and not set up as mirror."
log_w "To transform it into a mirror, delete the workspace manually and restart."
fi
fi
local manifest_url="${1:-}"
if [ -n "${manifest_url}" ]; then
init_params+=(-u "${manifest_url}")
fi
local manifest_branch="${2:-}"
if [ -n "${manifest_branch}" ]; then
init_params+=(-b "${manifest_branch}")
fi
local manifest="${3:-}"
if [ -n "${manifest}" ]; then
init_params+=(--manifest-name "${manifest}")
fi
local groups="${4:-}"
if [ -n "${groups}" ]; then
init_params+=("--groups=${groups}")
fi
if [ -n "${REPO_URL}" ]; then
init_params+=(--repo-url "${REPO_URL}" --no-repo-verify)
# `repo sync` also does `repo selfupdate`, thus it also needs to know
# that this repo version might not be signed.
sync_params+=(--no-repo-verify)
fi
if [ -n "${REPO_DEPTH}" ]; then
init_params+=(--depth "${REPO_DEPTH}")
fi
if [ -n "${REPO_NUM_JOBS}" ]; then
sync_params+=(--jobs "${REPO_NUM_JOBS}")
fi
if [ "${REPO_SYNC_FORCE_SYNC}" = true ]; then
sync_params+=(--force-sync)
fi
if [ "${SOURCE_FETCH_ALL}" = true ]; then
sync_params+=(--no-current-branch)
else
sync_params+=(--current-branch --no-tags)
fi
log_bold "Repo init:"
repo init "${init_params[@]}"
log_bold "Repo sync:"
repo sync "${sync_params[@]}" --detach "${REPO_PROJECTS[@]}"
}
# Add target remote and create new remote projects if needed.
#
# Globals:
# TARGET_GERRIT_NAME
# TARGET_GERRIT_PORT
# TARGET_GERRIT_URL
# TARGET_GERRIT_PARENT_PROJECT
# Arguments:
# None
# Returns:
# None
_prepare_projects()
{
log_bold "Preparing projects:"
if [ "${CREATE_PROJECTS}" ]; then
create_projects_cmd="add_project_to_gerrit"
else
create_projects_cmd=""
fi
repo_forall "add_git_remote; ${create_projects_cmd}"
if [ "${FETCH_TARGET_BEFORE_PUSH}" = true ]; then
repo_forall "git fetch \"${TARGET_GERRIT_NAME}\""
fi
}
# Push the currently synced HEAD to a target branch.
#
# This will push the current HEAD to a target branch, creating or
# fast-forwarding it. If the target branch already exists on the target remote
# and histories have diverged, then this will fail.
#
# Globals:
# DRY_RUN
# PUSH_SKIP_VALIDATION
# REPO_PROJECT
# REPO_PROJECTS
# REPO_REMOTE
# REPO_RREV
# TARGET_GERRIT_NAME
# Arguments:
# target_branch Name of the target branch to create or update.
# Returns:
# None
_import_new_branch()
{
local target_branch="$1"
log_bold "BRANCH: Push HEAD to ${target_branch}:"
# Fetch remote branch and push to the target gerrit.
repo_forall \
"remove_shallow_clone;" \
"push_branch_to_gerrit HEAD \"${target_branch}\""
}
# Fetch a remote branch and push to the target gerrit.
#
# Globals:
# DRY_RUN
# PUSH_SKIP_VALIDATION
# REPO_REMOTE
# TARGET_GERRIT_NAME
# Arguments:
# remote_branch Name of the remote branch to fetch.
# target_branch Name of the target branch to push to.
# Returns:
# None
_import_remote_branch()
{
local remote_branch="$1"
local target_branch="$2"
log_bold "BRANCH: Fetch ${remote_branch} and push ${target_branch}:"
# Fetch remote branch and push to the target gerrit.
repo_forall \
"fetch_remote_branch \"${remote_branch}\";" \
"push_branch_to_gerrit \"${remote_branch}\" \"${target_branch}\""
}
# Create a new tag at the manifest revisions and push to the target gerrit.
#
# Globals:
# DRY_RUN
# NEW_TAG_ANNOTATION_MESSAGE
# PUSH_SKIP_VALIDATION
# REPO_PROJECT
# REPO_REMOTE
# REPO_RREV
# TARGET_GERRIT_NAME
# Arguments:
# new_tag Name of the tag to create and push
# Returns:
# None
_import_new_tag()
{
local new_tag="$1"
log_bold "TAG: Create and push ${new_tag}:"
repo_forall \
"create_new_tag \"${new_tag}\";" \
"remove_shallow_clone;" \
"push_tag_to_gerrit \"${new_tag}\""
}
# Fetch a remote tag and push to the target gerrit.
#
# Globals:
# DRY_RUN
# PUSH_SKIP_VALIDATION
# TARGET_GERRIT_NAME
# Arguments:
# remote_tag - Name of the remote tag to fetch and push
# Returns:
# None
_import_remote_tag()
{
local remote_tag="$1"
log_bold "TAG: Fetch and push ${remote_tag}:"
repo_forall \
"fetch_remote_tag \"${remote_tag}\";" \
"push_tag_to_gerrit \"${remote_tag}\""
}
# Fetch a remote tag and push to the target gerrit; fall back to locally created
# tag if the source remote doesn't have the tag.
#
# Globals:
# NEW_TAG_ANNOTATION_MESSAGE
# PUSH_SKIP_VALIDATION
# REPO_PROJECT
# REPO_REMOTE
# TARGET_GERRIT_NAME
# VERBOSE
# Arguments:
# new_tag Name of the tag to create and push
# Returns:
# None
_import_remote_tag_with_new_tag_fallback()
{
local tag="$1"
log_bold "TAG: Fetch, create if needed, and push ${tag}:"
repo_forall \
"fetch_and_push_tag_to_gerrit_fall_back_to_new_tag \"${tag}\""
}
# Push current remote revisions to the target gerrit
#
# Globals:
# DRY_RUN
# PUSH_BRANCH_PREFIX
# PUSH_SKIP_VALIDATION
# REPO_PROJECT
# REPO_REMOTE
# REPO_RREV
# TARGET_GERRIT_NAME
# TRY_QUALIFYING_REFS
# Arguments:
# None
# Returns:
# None
_push_current_remote_revisions()
{
log_bold "REVISIONS: Pushing current remote revisions:"
repo_forall \
"remove_shallow_clone;" \
"push_current_remote_revision"
}
# Push current upstream revisions to the target gerrit
#
# Globals:
# PUSH_BRANCH_PREFIX
# PUSH_SKIP_VALIDATION
# TARGET_GERRIT_NAME
# TRY_QUALIFYING_REFS
# Arguments:
# None
# Returns:
# None
_push_current_upstream_revision()
{
log_bold "REVISIONS: Pushing current upstream revisions:"
repo_forall "push_current_upstream_revision"
}
while [ $# -gt 0 ]; do
case "$1" in
--mirror)
INIT_AS_MIRROR=true
shift 1
;;
-u|--url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_URL="$2"
shift 2
;;
-b|--manifest-branch)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_BRANCH="$2"
shift 2
;;
-m|--manifest)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST="$2"
shift 2
;;
-gn|--target-gerrit-name)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
export TARGET_GERRIT_NAME="$2"
shift 2
;;
-gu|--target-gerrit-url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
export TARGET_GERRIT_URL="$2"
shift 2
;;
-gp|--target-gerrit-port)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
export TARGET_GERRIT_PORT="$2"
shift 2
;;
-pp|--target-gerrit-parent-project)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
export TARGET_GERRIT_PARENT_PROJECT="$2"
shift 2
;;
-nb|--new-branch)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
NEW_BRANCH="$2"
shift 2
;;
-rb|--remote-branch)
if [ $# -lt 3 ]; then
_usage >&2
exit 1
fi
REMOTE_BRANCH="$2"
TARGET_BRANCH="$3"
shift 3
;;
-nt|--new-tag)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
NEW_TAG="$2"
shift 2
;;
-rt|--remote-tag)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REMOTE_TAG="$2"
shift 2
;;
--remote-tag-fallback-new-tag)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REMOTE_TAG_WITH_NEW_TAG_FALLBACK="$2"
shift 2
;;
--new-tag-annotation-message)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
export NEW_TAG_ANNOTATION_MESSAGE="$2"
shift 2
;;
-pc|--push-current-remote-rev)
PUSH_CURRENT_REMOTE_REVISIONS=true
shift 1
;;
--push-current-upstream-rev)
PUSH_CURRENT_UPSTREAM_REVISION=true
shift 1
;;
--push-with-branch-prefix)
PUSH_BRANCH_PREFIX="$2"
shift 2
;;
--try-qualifying-refs)
export TRY_QUALIFYING_REFS=true
shift 1
;;
-g|--groups)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_GROUPS="$2"
shift 2
;;
-p|--projects)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
# Parse single-space-separated list into array, as repo requires a
# list of parameters for projects, rather than one parameter
# containing them all.
# NOTE: This WON'T work with spaces in project names. Doing this
# properly would really be a case for a python rewrite..
IFS=' ' read -r -a REPO_PROJECTS <<< "$2"
shift 2
;;
--exclude-projects)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
EXCLUDE_PROJECTS="$2"
shift 2
;;
-cp|--create-projects)
CREATE_PROJECTS=true
shift 1
;;
--dry-run)
export DRY_RUN=true
shift 1
;;
-d|--debug)
set -x
export DEBUG=true
shift 1
;;
-s|--skip-init)
SKIP_INIT=true
shift 1
;;
--push-skip-validation)
export PUSH_SKIP_VALIDATION=true
shift 1
;;
--repo-url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_URL="$2"
shift 2
;;
--depth)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_DEPTH="$2"
shift 2
;;
--force-sync)
REPO_SYNC_FORCE_SYNC=true
shift 1
;;
--source-fetch-all)
SOURCE_FETCH_ALL=true
shift 1
;;
--fetch-target-before-push)
FETCH_TARGET_BEFORE_PUSH=true
shift 1
;;
-j|--jobs)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_NUM_JOBS="$2"
shift 2
;;
-h|--help)
_usage
exit 0
;;
*)
echo "Unknown option $1" >&2
_usage >&2
exit 1
;;
esac
done
if [ "${INIT_AS_MIRROR}" = true ] && [ -n "${REPO_DEPTH}" ]; then
log_e "--mirror and --depth are mutually exclusive. See 'repo help init'."
fi
# Variables required for `repo_forall` wrapper function:
repo_forall_functions="${SHELL_LIBS}/repo-forall-functions.sh"
if [ ! -f "${repo_forall_functions}" ] ; then
log_e "Unable to find repo-forall-functions.sh"
fi
repo_forall_args=(--abort-on-errors --verbose -p "${REPO_PROJECTS[@]}")
if [ -n "${MANIFEST_GROUPS}" ]; then
repo_forall_args+=("--groups=${MANIFEST_GROUPS}")
fi
if [ -n "${REPO_NUM_JOBS}" ]; then
repo_forall_args+=(--jobs "${REPO_NUM_JOBS}")
fi
if [ -n "${EXCLUDE_PROJECTS}" ]; then
repo_forall_args+=(--inverse-regex "${EXCLUDE_PROJECTS}")
fi
# Run shell commands in pre-configured `repo forall` context
#
# Always use this wrapper to execute shell commands through `repo forall -c`.
# Commands will have access to code defined in repo-forall-functions.sh. Also,
# project selection, logging, job count etc is configured here.
#
# Globals:
# None
# Arguments:
# Arbitrary list of shell commands to execute per project.
# Returns:
# None
repo_forall() {
repo --no-pager forall "${repo_forall_args[@]}" --command bash -c \
"source \"${repo_forall_functions}\"; skip_repo_mirror_internal_project; $*"
}
# Initialize Android tree
if [ "${SKIP_INIT}" != true ] ; then
_init_projects "${MANIFEST_URL}" "${MANIFEST_BRANCH}" "${MANIFEST}" \
"${MANIFEST_GROUPS}"
fi
_prepare_projects
if [ -n "${NEW_BRANCH}" ] ; then
_import_new_branch "${NEW_BRANCH}"
fi
if [ -n "${REMOTE_BRANCH}" ] && [ -n "${TARGET_BRANCH}" ] ; then
_import_remote_branch "${REMOTE_BRANCH}" "${TARGET_BRANCH}"
fi
if [ "${PUSH_CURRENT_REMOTE_REVISIONS}" = true ] \
&& [ "${PUSH_CURRENT_UPSTREAM_REVISION}" = true ]; then
# Two expected valid cases for attributes in the manifest xml:
# * `revision` points to branches or tags
# -> import them via --push-current-remote-rev
# * `revision` points to a commit sha1, as typically done in release
# manifests. `upstream` points to branches that those sha1's are on.
# -> Use --push-current-upstream-rev, maybe do something else with the
# commit sha1's, like creating release tags manually.
log_e "Both --push-current-remote-rev and --push-current-upstream-rev" \
"are set. This is probably a mistake. Aborting."
fi
if [ "${PUSH_CURRENT_REMOTE_REVISIONS}" = true ]; then
_push_current_remote_revisions
fi
if [ "${PUSH_CURRENT_UPSTREAM_REVISION}" = true ]; then
_push_current_upstream_revision
fi
# Always import tags after importing branches. Gerrit defines permissions on
# branches (refs/heads/*). Therefore, first get Gerrit to know the new source
# history, then create tags on it. Otherwise tag pushing will fail for non-admin
# users.
if [ -n "${NEW_TAG}" ] ; then
_import_new_tag "${NEW_TAG}"
fi
if [ -n "${REMOTE_TAG}" ] ; then
_import_remote_tag "${REMOTE_TAG}"
fi
if [ -n "${REMOTE_TAG_WITH_NEW_TAG_FALLBACK}" ] ; then
_import_remote_tag_with_new_tag_fallback \
"${REMOTE_TAG_WITH_NEW_TAG_FALLBACK}"
fi