blob: 08b829ea4104fd5c702235750da5be42e06503eb [file] [log] [blame]
#!/bin/bash
# Copyright 2018-2021 Fairphone B.V.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Exported globals:
# DEBUG
# DRY_RUN
# SHELL_LIBS
# TARGET_GERRIT_NAME
# TARGET_GERRIT_URL
# TARGET_GERRIT_PORT
# TARGET_GERRIT_PARENT_PROJECT
# VERBOSE
set -e
SHELL_LIBS="$(dirname "$(readlink -f "$0")")/../shell-libs"
export SHELL_LIBS
. "${SHELL_LIBS}/utils.sh"
TARGET_GERRIT_NAME="fairphone"
TARGET_GERRIT_URL="review.fairphone.software"
TARGET_GERRIT_PORT="29418"
TARGET_GERRIT_PARENT_PROJECT="Acl/Android"
_usage() {
cat <<heredoc
Fetch branches and tags from a source remote gerrit and push to projects in a
target gerrit. Missing projects will be created if needed.
Locally defined branches and tags based on the remote's manifest project
revisions can also be created and pushed to the target remote.
Usage:
$0 [-u|--url MANIFEST_URL] [-b|--manifest-branch] [-m|--manifest MANIFEST]
[-nb|--new-branch NEW_BRANCH] [-rb|--remote-branch REMOTE_BRANCH TARGET_BRANCH]
[-nt|--new-tag NEW_TAG] [-rt|--remote-tag REMOTE_TAG]
[-g|--groups "<group1>,..."] [-p|--projects "project1 project2"] [-d|--debug]
[-v|--verbose] [-n|--dry-run] [-s|--skip-init] [--push-skip-validation]
[--repo-url REPO_URL] [--depth REPO_DEPTH] [-j|--jobs REPO_NUM_JOBS] [-h|--help]
Options:
-u, --url MANIFEST_URL : Remote manifest URL to initialize from.
Default to the current one if unset. See also
--skip-init.
-b, --manifest-branch MANIFEST_BRANCH
: Remote manifest branch or revision to
initialize from. Default to the current one if
unset. See also --skip-init.
-m, --manifest MANIFEST : Remote manifest file to initialize the tree
from. See also --skip-init.
-gn, --target-gerrit-name NAME : Git remote name of the target gerrit.
Default: "${TARGET_GERRIT_NAME}"
-gu, --target-gerrit-url URL : URL of the target gerrit. Only access via ssh
is supported at the moment.
Default: "${TARGET_GERRIT_URL}"
-gp, --target-gerrit-port PORT : Port of the target gerrit.
Default: ${TARGET_GERRIT_PORT}
-pp, --target-gerrit-parent-project PARENT_PROJECT
Parent for project that need to be created on
the target gerrit.
Default: "${TARGET_GERRIT_PARENT_PROJECT}"
-nb, --new-branch NEW_BRANCH : New branch to create and push.
-rb, --remote-branch REMOTE_BRANCH TARGET_BRANCH
: Names of remote branch to fetch and target
branch to push to target gerrit.
-nt, --new-tag NEW_TAG : Name of a new tag to create and push.
-rt, --remote-tag REMOTE_TAG : Name of remote tag to fetch and import.
-pc, --push-current-remote-rev : Push the remote revisions (tags or branchs)
that the manifest is currently pointing on per
project. This allows to push the current state
when different tags or branches are used
across projects.
--try-qualifying-refs : When using "--push-current-remote-rev" and
manifest (remote) revisions are unqualified,
query remote heads and tags to find the
matching one. If both, branch and tag. with
matching name exist, fail the sync job.
-g, --groups "<group>,..." : Quote enclosed, comma separated, manifest
groups.
Default: unset (all but "notdefault" projects)
E.g. "all" or "pdk,pdk-qcom"
-p, --projects "project ..." : Quote enclosed, space separated, list of
projects to work on. When undefined script
will run commands on all projects allowed by
MANIFEST_GROUPS.
E.g. -p "cts dalvik"
--exclude-projects EXCLUDE_PROJECTS
: Skip projects matching a regex or wildcard
expression. These projects will still be
synchronized via "repo sync", but not created
on or pushed to the target Gerrit.
-cp, --create-projects : Create missing projects on the target Gerrit.
By default, the script assumes that all
relevant projects exist already.
--dry-run : Run git and ssh commands but do not modify
remote.
-d, --debug : Show commands being run. I.e. set -x.
-v, --verbose : Show additional logging.
-s, --skip-init : Skip 'repo init' and 'repo sync' commands.
Only set this if you are sure all projects
have been synced and point to the correct
remote and revision.
--push-skip-validation : Add '-o skip-validation' to git push calls.
This can be necessary when pushing new
histories with a lot of commits that get
otherwise rejected by Gerrit. Requires
extended permissions, see the Gerrit docs.
--repo-url : Pass a custom repo tool URL to repo init. If
set, --no-repo-verify will be passed to repo
init as well to accept unsigned commits in the
repo tool git history.
--depth : Use a custom default depth for \`repo init\`.
This is most useful with our custom parameter
value \`--depth -1\` in repo to force
unshallow checkouts of all projects.
-j, --jobs : Number of jobs for repo sync and forall. Use
this to speed up overall job execution. This
can make logs unusable, so it's on 1 by
default.
-h, --help : Print this help message
Examples:
$0 -m aosp/aosp-9.0.0_r8.xml -rb pie-r2-s1-release aosp/pie-r2-s1-release -rt android-9.0.0_r8
$0 -m caf/default_LA.UM.7.6.2.r1-03400-89xx.0.xml -nb staging/arima/fp3/p/r0
Publishing sources for a Fairphone Open release:
$0 -m default.xml -gn code_fp -gu gerrit-public.fairphone.software -gp 29418 -pc --create-projects -pp "All-Projects"
$0 -m rel/p/fp2/20.10.1-beta/20.10.1-beta.0-public.xml -gn code_fp -gu gerrit-public.fairphone.software -gp 29418 -rb rel/p/fp2/20.10.1-beta rel/p/fp2/20.10.1-beta -rt 20.10.1-beta.0
heredoc
}
# Perform a repo init and sync on a specified remote manifest
#
# Globals:
# REPO_URL
# REPO_DEPTH
# REPO_NUM_JOBS
# Arguments:
# manifest_url Remote manifest URL. If empty, the current one in local repo
# tree will be used.
# manifest Remote manifest to use.
# groups repo groups to use.
# platform repo platform to use.
# Returns:
# None
_init_projects()
{
local manifest_url="${1:-}"
if [ -n "${manifest_url}" ]; then
manifest_url_params=(-u "${manifest_url}")
fi
local manifest_branch="${2:-}"
if [ -n "${manifest_branch}" ]; then
manifest_branch_params=(-b "${manifest_branch}")
fi
local manifest="${3:-}"
if [ -n "${manifest}" ]; then
manifest_params=(--manifest-name "${manifest}")
fi
local groups="${4:-}"
if [ -n "${groups}" ]; then
manifest_groups_params=("--groups=${groups}")
fi
if [ -n "${REPO_URL:-}" ]; then
repo_url_params=(--repo-url "${REPO_URL}" --no-repo-verify)
fi
if [ -n "${REPO_DEPTH:-}" ]; then
depth_params=(--depth "${REPO_DEPTH}")
fi
if [ -n "${REPO_NUM_JOBS:-}" ]; then
jobs_params=(--jobs "${REPO_NUM_JOBS}")
fi
log_bold "Repo init:"
repo init \
"${manifest_url_params[@]}" \
"${manifest_branch_params[@]}" \
"${manifest_params[@]}" \
"${manifest_groups_params[@]}" \
"${repo_url_params[@]}" \
"${depth_params[@]}"
log_bold "Repo sync:"
repo sync "${jobs_params[@]}" --current-branch --detach --no-tags "${REPO_PROJECTS[@]}"
}
# Add target remote and create new remote projects if needed.
#
# Globals:
# TARGET_GERRIT_NAME
# TARGET_GERRIT_PORT
# TARGET_GERRIT_URL
# TARGET_GERRIT_PARENT_PROJECT
# Arguments:
# None
# Returns:
# None
_prepare_projects()
{
log_bold "Preparing projects:"
if [ "${CREATE_PROJECTS}" ]; then
create_projects_cmd="add_project_to_gerrit"
else
create_projects_cmd=""
fi
repo_forall "add_git_remote; ${create_projects_cmd}"
}
# Create a local branch at the manifest revisions and push to the target gerrit.
#
# Globals:
# PUSH_SKIP_VALIDATION
# REPO_PROJECTS
# TARGET_GERRIT_NAME
# Arguments:
# new_branch Name of the branch to create and push.
# Returns:
# None
_import_new_branch()
{
local new_branch="$1"
log_bold "BRANCH: Create and push ${new_branch}:"
# Create the branch
repo start "${new_branch}" "${REPO_PROJECTS[@]}"
# Push branch to the project in the target gerrit.
repo_forall \
"remove_shallow_clone;" \
"push_branch_to_gerrit \"${new_branch}\" \"${new_branch}\""
}
# Fetch a remote branch and push to the target gerrit.
#
# Globals:
# PUSH_SKIP_VALIDATION
# TARGET_GERRIT_NAME
# Arguments:
# remote_branch Name of the remote branch to fetch.
# target_branch Name of the target branch to push to.
# Returns:
# None
_import_remote_branch()
{
local remote_branch="$1"
local target_branch="$2"
log_bold "BRANCH: Fetch ${remote_branch} and push ${target_branch}:"
# Fetch remote branch and push to the target gerrit.
repo_forall \
"fetch_remote_branch \"${remote_branch}\";" \
"push_branch_to_gerrit \"${remote_branch}\" \"${target_branch}\""
}
# Create a new tag at the manifest revisions and push to the target gerrit.
#
# Globals:
# PUSH_SKIP_VALIDATION
# TARGET_GERRIT_NAME
# Arguments:
# new_tag Name of the tag to create and push
# Returns:
# None
_import_new_tag()
{
local new_tag="$1"
log_bold "TAG: Create and push ${new_tag}:"
repo_forall \
"git tag \"${new_tag}\";" \
"remove_shallow_clone;" \
"push_tag_to_gerrit \"${new_tag}\""
}
# Fetch a remote tag and push to the target gerrit.
#
# Globals:
# PUSH_SKIP_VALIDATION
# Arguments:
# remote_tag - Name of the remote tag to fetch and push
# Returns:
# None
_import_remote_tag()
{
local remote_tag="$1"
log_bold "TAG: Fetch and push ${remote_tag}:"
repo_forall \
"fetch_remote_tag \"${remote_tag}\";" \
"push_tag_to_gerrit \"${remote_tag}\""
}
# Push current remote revisions to the target gerrit
#
# Globals:
# PUSH_SKIP_VALIDATION
# TARGET_GERRIT_NAME
# TRY_QUALIFYING_REFS
# Arguments:
# None
# Returns:
# None
_push_current_remote_revisions()
{
log_bold "REVISIONS: Pushing current remote revisions:"
repo_forall \
"remove_shallow_clone;" \
"push_current_remote_revision"
}
while [ $# -gt 0 ]; do
case "$1" in
-u|--url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_URL="$2"
shift 2
;;
-b|--manifest-branch)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_BRANCH="$2"
shift 2
;;
-m|--manifest)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST="$2"
shift 2
;;
-gn|--target-gerrit-name)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
TARGET_GERRIT_NAME="$2"
shift 2
;;
-gu|--target-gerrit-url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
TARGET_GERRIT_URL="$2"
shift 2
;;
-gp|--target-gerrit-port)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
TARGET_GERRIT_PORT="$2"
shift 2
;;
-pp|--target-gerrit-parent-project)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
TARGET_GERRIT_PARENT_PROJECT="$2"
shift 2
;;
-nb|--new-branch)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
NEW_BRANCH="$2"
shift 2
;;
-rb|--remote-branch)
if [ $# -lt 3 ]; then
_usage >&2
exit 1
fi
REMOTE_BRANCH="$2"
TARGET_BRANCH="$3"
shift 3
;;
-nt|--new-tag)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
NEW_TAG="$2"
shift 2
;;
-rt|--remote-tag)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REMOTE_TAG="$2"
shift 2
;;
-pc|--push-current-remote-rev)
export PUSH_CURRENT_REMOTE_REVISIONS=true
shift 1
;;
--try-qualifying-refs)
export TRY_QUALIFYING_REFS=true
shift 1
;;
-g|--groups)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
MANIFEST_GROUPS="$2"
shift 2
;;
-p|--projects)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
# Parse single-space-separated list into array, as repo requires a
# list of parameters for projects, rather than one parameter
# containing them all.
# NOTE: This WON'T work with spaces in project names. Doing this
# properly would really be a case for a python rewrite..
IFS=' ' read -r -a REPO_PROJECTS <<< "$2"
shift 2
;;
--exclude-projects)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
EXCLUDE_PROJECTS="$2"
shift 2
;;
-cp|--create-projects)
export CREATE_PROJECTS=true
shift 1
;;
--dry-run)
export DRY_RUN="--dry-run"
shift 1
;;
-v|--verbose)
export VERBOSE="--verbose"
shift 1
;;
-d|--debug)
set -x
export DEBUG="true"
shift 1
;;
-s|--skip-init)
SKIP_INIT=true
shift 1
;;
--push-skip-validation)
PUSH_SKIP_VALIDATION=true
shift 1
;;
--repo-url)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_URL="$2"
shift 2
;;
--depth)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_DEPTH="$2"
shift 2
;;
-j|--jobs)
if [ $# -lt 2 ]; then
_usage >&2
exit 1
fi
REPO_NUM_JOBS="$2"
shift 2
;;
-h|--help)
_usage
exit 0
;;
*)
echo "Unknown option $1" >&2
_usage >&2
exit 1
;;
esac
done
export PUSH_SKIP_VALIDATION
export TARGET_GERRIT_NAME
export TARGET_GERRIT_URL
export TARGET_GERRIT_PORT
export TARGET_GERRIT_PARENT_PROJECT
# Variables required for `repo_forall` wrapper function:
repo_forall_functions="${SHELL_LIBS}/repo-forall-functions.sh"
if [ ! -f "${repo_forall_functions}" ] ; then
log_e "Unable to find repo-forall-functions.sh" || _usage >&2
exit 1
fi
repo_forall_args=(--abort-on-errors --verbose "${REPO_PROJECTS[@]}")
if [ -n "${MANIFEST_GROUPS}" ]; then
repo_forall_args+=("--groups=${MANIFEST_GROUPS}")
fi
if [ -n "${REPO_NUM_JOBS:-}" ]; then
repo_forall_args+=(--jobs "${REPO_NUM_JOBS}")
fi
if [ -n "${EXCLUDE_PROJECTS:-}" ]; then
repo_forall_args+=(--inverse-regex "${EXCLUDE_PROJECTS}")
fi
# Run shell commands in pre-configured `repo forall` context
#
# Always use this wrapper to execute shell commands through `repo forall -c`.
# Commands will have access to code defined in repo-forall-functions.sh. Also,
# project selection, logging, job count etc is configured here.
#
# Globals:
# None
# Arguments:
# Arbitrary list of shell commands to execute per project.
# Returns:
# None
repo_forall() {
repo --no-pager forall "${repo_forall_args[@]}" --command bash -c \
"source \"${repo_forall_functions}\"; $*"
}
# Initialize Android tree
if [ -z "${SKIP_INIT}" ] ; then
_init_projects "${MANIFEST_URL}" "${MANIFEST_BRANCH}" "${MANIFEST}" "${MANIFEST_GROUPS}"
fi
_prepare_projects
if [ -n "${NEW_BRANCH}" ] ; then
_import_new_branch "${NEW_BRANCH}"
fi
if [ -n "${REMOTE_BRANCH}" ] && [ -n "${TARGET_BRANCH}" ] ; then
_import_remote_branch "${REMOTE_BRANCH}" "${TARGET_BRANCH}"
fi
if [ "${PUSH_CURRENT_REMOTE_REVISIONS}" = true ]; then
_push_current_remote_revisions
fi
# Always import tags after importing branches. Gerrit defines permissions on
# branches (refs/heads/*). Therefore, first get Gerrit to know the new source
# history, then create tags on it. Otherwise tag pushing will fail for non-admin
# users.
if [ -n "${NEW_TAG}" ] ; then
_import_new_tag "${NEW_TAG}"
fi
if [ -n "${REMOTE_TAG}" ] ; then
_import_remote_tag "${REMOTE_TAG}"
fi