bin/merge-partner-on-aosp

#!/bin/bash

# Copyright 2018-2021 Fairphone B.V.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -o nounset
set -o errexit

# Merge partner security branches or release tags on AOSP projects
#
# Use this script together with the repo tool in the root of the Android tree to
# merge partner branches or AOSP tags in supported projects.
#
# Following prerequisites need to be met:
#   * Run `fetch-and-push-partner` first to have up-to-date copies of related
#     tags or branches in the (internal) projects that are referenced in your
#     Android manifest.
#   * Make sure to `repo init` and `repo sync` with a manifest that points to
#     project states you want to merge on. The manifest must point to branches,
#     not tags or fixed commits; otherwise merges cannot be pushed (as there
#     would be no branch to push to).
#
# Finally, use the following command line as reference to use the tool. For
# merging release tags, set REF_IS_RELEASE_TAG=1 and AOSP_PARTNER_REF to the
# release tag. Then set REVIEW_TOPIC as needed and run first without
# PUSH_TO_REMOTE, to check the merge locally before pushing.
#
# Merging security branches:
# repo --no-pager forall -g aosp -vpc \
#     'PUSH_TO_REMOTE=1 REVIEW_TOPIC=... \
#     "$(pwd)/vendor/fairphone/tools/bin/merge-partner-on-aosp" 2>&1' | \
#     tee merge-partner-on-aosp-$(date +"%Y-%m-%d_%H-%M-%S").log
#
# Merging release tags:
# repo --no-pager forall -g aosp -vpc \
#     'PUSH_TO_REMOTE=1 REVIEW_TOPIC=... \
#     REF_IS_RELEASE_TAG=1 AOSP_PARTNER_REF=...
#     "$(pwd)/vendor/fairphone/tools/bin/merge-partner-on-aosp" 2>&1' | \
#     tee merge-partner-on-aosp-$(date +"%Y-%m-%d_%H-%M-%S").log
#
#
# Make sure to run this tool on projects with histories related to the ones
# you're merging (e.g., run it only on "aosp" repo groups, if the manifest is
# set up accordingly).

# The latest AOSP-Partner reference we want to merge
AOSP_PARTNER_REF=${AOSP_PARTNER_REF:-security-aosp-pi-release}
REF_IS_RELEASE_TAG=${REF_IS_RELEASE_TAG:-0}

PUSH_TO_REMOTE=${PUSH_TO_REMOTE:-0}
REVIEW_TOPIC=${REVIEW_TOPIC:-}
WIP_CHANGES=${WIP_CHANGES:-1}

ALLOW_FAST_FORWARD=${ALLOW_FAST_FORWARD:-0}
# Allow having commits in the local history that are not merged yet on the
# remote.
ALLOW_LOCAL_COMMITS=${ALLOW_LOCAL_COMMITS:-0}


# Check bash version requirements
#
# Required from version 4.4:
#   * Initialize empty array correctly with `set -u` enabled. When resolving the
#     variable as parameter (`some_command "${some_array[@]}"`), don't add an
#     empty parameter if the array is empty. This breaks in bash < 4.4.
check_min_bash_version() {
    local required_version="4.4"
    local res
    res=$(echo "${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]} >= ${required_version}" | bc -l)
    if [ "${res}" -eq 1 ]; then
        return 0
    else
        return 1
    fi
}

if ! check_min_bash_version; then
    echo "ERROR: Bash version 4.4 or higher is required." >&2
fi

fast_forward_to_aosp_partner() {
    local head_commit
    head_commit="$1"
    local on_branch
    on_branch="$2"
    echo "INFO: Fast-forwarding to the AOSP-Partner ref.…"
    if ! git merge --quiet --ff HEAD "${AOSP_PARTNER_REF}"; then
        echo "ERROR: Could not fast-forward to the AOSP-Partner ref. (${AOSP_PARTNER_REF})." >&2
        exit 1
    fi
    # Check for last part: Is there a branch to push to?
    if [ "${on_branch}" = false ]; then
        echo "ERROR: Fast-forward was successful but we are not on a branch. Investigate this \
manually!" >&2
        exit 1
    fi
    if [ 1 -eq "${PUSH_TO_REMOTE}" ]; then
        echo "INFO: Updating the remote…"
        if ! git push --quiet origin HEAD:"${target_branch}"; then
            echo "ERROR: Could not push the merged HEAD to the remote (${target_branch})." >&2
            exit 1
        fi
    else
        echo "INFO: NOT pushing to the remote, PUSH_TO_REMOTE is not enabled." >&2
    fi
    # If the fast-forward succeeds, there is nothing more to do.
    return 0
}

# Check for "Text file busy" failure during `git commit`
#
# Sometimes `git merge` (or `git commit`) in repo fails apparently with a timing
# issue:
#   > fatal: cannot exec '.git/hooks/commit-msg': Text file busy
#   > Not committing merge; use 'git commit' to complete the merge.
#
# Detect such cases based on the merge/commit command output, because the exit
# codes don't reveal anything about it.
_check_parallel_commit_failed_and_retry() {
    local output="$1"
    local failure_msg="fatal: cannot exec '.git/hooks/commit-msg': Text file busy"
    local num_tries=5
    for _ in $(seq "${num_tries}"); do
        if ! echo "${output}" | grep -qFe "${failure_msg}"; then
            # If getting here but but failure message doesn't match, something
            # else went wrong during the merge. A merge conflict for example.
            return 1
        fi
        echo "WARNING: Commit failed due to 'Text file busy'. Retrying..."
        local ret_val=0
        # core.editor=true: see comment in _merge_commit_safe
        output=$(git -c core.editor=true commit --quiet 2>&1) || ret_val=$?
        # Propagate error output, in case there is any.
        if [ -n "${output}" ]; then
            echo "${output}" >&2
        fi
        if [ "${ret_val}" -eq 0 ]; then
            # All good then, commit succeeded.
            echo "INFO: Resolved 'Text file busy' issue with additional 'git commit' attempt."
            return 0
        fi
        sleep 0.5s
    done
    echo "ERROR: 'Text file busy' error persists after ${num_tries} commit attempts." >&2
    return 1
}

# Perform a `git merge` and make sure it works
#
# We ran into various issues with scripting `git merge`; therefore it's moved to
# its very own function now.
_merge_commit_safe() {
    # Note: Commit message, and pre/post processing
    #   We're NOT using `-m ...` or `--no-edit` at the moment, because it breaks
    #   in various ways: Not dropping comments within the message body (e.g.,
    #   from GPG verification), not adding an empty line after the subject when
    #   merging annotated tags, not adding an empty line before the Change-Id.
    # Solution:
    #   Rely on git and its hooks to set the commit message. Also, set config
    #   `core.editor=true`: This has the same effect as `--no-edit`, only that
    #   it doesn't break in the ways listed above.

    # Checkout a local branch, to make the subject line more expressive than
    # "merge into HEAD". Cleanup the local branch first in case it's already
    # there.
    git branch --quiet -D "${target_branch}" 2>/dev/null || true
    git checkout -b "${target_branch}"
    local merge_output
    local merge_result=0
    merge_output=$(git -c core.editor=true merge \
        --quiet --edit --log --no-ff "${AOSP_PARTNER_REF}" 2>&1) \
        || merge_result=$?
    if [ -n "${merge_output}" ]; then
        echo "${merge_output}" >&2
    fi
    if [ 0 -ne ${merge_result} ]; then
        merge_result=0
        _check_parallel_commit_failed_and_retry "${merge_output}" || merge_result=$?
    fi
    # Cleanup again. But can't do is in case we're in a middle of a merge
    # conflict. The user needs to clean up manually then.
    if [ 0 -eq ${merge_result} ]; then
        git checkout --detach
        git branch --quiet -D "${target_branch}"
    fi
    return ${merge_result}
}

# Merge or fast-forward the AOSP-Partner reference on HEAD.
# Fast-forward is only done if allowed. It would be pushed directly to the
# origin. Merges are pushed for review. If we are not actually on a branch, the
# merge/fast-forward is done anyways and the repo is left as it is afterwards.
merge_aosp_partner_on_head() {
    local head_commit
    head_commit="${1:-}"
    local target_branch
    target_branch="${2:-}"
    if [ -z "${head_commit}" ] || [ -z "${target_branch}" ]; then
        echo "USAGE ERROR: merge_aosp_partner_on_head <head_commit> <target_branch>" >&2
        exit 1
    fi
    can_fast_forward=true
    on_branch=true
    git merge-base --is-ancestor HEAD "${AOSP_PARTNER_REF}" || can_fast_forward=false
    git show-ref --verify "refs/remotes/origin/${target_branch}" 1>/dev/null || on_branch=false

    # If enabled, check if we can fast-forward.
    if [ 1 -eq "${ALLOW_FAST_FORWARD}" ] && [ "${can_fast_forward}" = true ]; then
        fast_forward_to_aosp_partner "${head_commit}" "${on_branch}"
        return 0
    fi

    # Fast-forward is not enabled or is not possible. So try to merge!
    echo "INFO: Merging the latest AOSP-Partner ref. (${AOSP_PARTNER_REF})…"
    if [ ${can_fast_forward} = true ]; then
        echo "INFO: Could fast-forward to the AOSP-Partner ref., but will still create a merge \
commit."
    fi
    old_head=$(git rev-parse HEAD)
    # Perform the actual `git merge` -- including various workarounds.
    if ! _merge_commit_safe; then
        echo "ERROR: Could not merge the AOSP-Partner ref. (${AOSP_PARTNER_REF})." >&2
        exit 1
    fi
    new_head=$(git rev-parse HEAD)
    if [ "_${old_head}" = "_${new_head}" ]; then
        echo "INFO: AOSP-Partner ref. is already merged. Nothing to do."
        echo "DEBUG: This case should have been caught earlier (?)"
        return 0
    fi
    echo "INFO: Successfully merged the AOSP-Partner ref." >&2
    if [ "${on_branch}" = false ]; then
        if git diff HEAD^1..HEAD --exit-code >/dev/null; then
            echo "WARNING: Merge was successful but we are not on a branch! \
However, the merge is empty so can be ignored." >&2
        else
            echo "ERROR: Merge was successful but we are not on a branch. This \
merge needs to be pushed!" >&2
        fi
        exit 1
    fi
    if [ 1 -eq "${PUSH_TO_REMOTE}" ]; then
        echo "INFO: Pushing for review…"
        local topic_params=()
        if [ "${REVIEW_TOPIC}" ]; then
            topic_params=(-o "topic=${REVIEW_TOPIC}")
        fi
        local wip_option=""
        if [ "${WIP_CHANGES}" -eq 1 ]; then
            wip_option="%wip"
        fi
        local ret_val=0
        git push --quiet origin "HEAD:refs/for/${target_branch}${wip_option}" "${topic_params[@]}" \
            || ret_val=$?
        if [ "${ret_val}" -ne 0 ]; then
            echo "ERROR: Could not push the merged HEAD to the remote (${target_branch})." >&2
            exit 1
        fi
    else
        echo "INFO: NOT pushing to the remote, PUSH_TO_REMOTE is not enabled." >&2
    fi
}

# Check if a commit sha1 exists locally
#
# Return 0 if the supplied commit exists, 1 otherwise.
_git_commit_exists() {
    local commit=$1
    # Suppress error output of cat-file in case the commit does not exist.
    if git cat-file -e "${commit}" 2>/dev/null; then
        return 0;
    fi
    return 1
}

# Git doesn't allow to check if a specific revision is shallow, but "partially
# shallow" repositories are very common in a repo tree and "re-unshallowing" is
# expensive.
_is_shallow_revision() {
    local revision_to_check="$1"

    if [ "$(git rev-parse --is-shallow-repository)" = "false" ]; then
        return 1
    fi

    if ! _git_commit_exists "${revision_to_check}"; then
        # We don't have the commit at all in the local repository, so it's also
        # not shallow...
        return 0
    fi

    if [ ! -f .git/shallow ]; then
        echo "WARNING: .git/shallow missing on a shallow repo?!"
        # Unexpected case -- assume shallow, as reported by git rev-parse above.
        return 0
    fi

    for shallow_rev in $(cat .git/shallow); do
        if git merge-base --is-ancestor "${shallow_rev}" "${revision_to_check}"; then
            # We have a shallow revision in our history -- we're shallow.
            return 0
        fi
    done
    # Clearly not shallow
    return 1
}


# Gather some information about remote and local refs and revisions
if [ "${REF_IS_RELEASE_TAG}" -eq 1 ]; then
    # Get the actual commit object, not the annotated tag
    dereferenced_partner_ref="refs/tags/${AOSP_PARTNER_REF}^{}"
else
    # Nothing to dereference on branches
    dereferenced_partner_ref="${AOSP_PARTNER_REF}"
fi
aosp_partner_ref_list=$(git ls-remote --exit-code origin "${dereferenced_partner_ref}" 2>/dev/null)
case $? in
    0)
        # Success
        ;;
    2)
        echo "ERROR: The AOSP-Partner ref does not exist on remote (${AOSP_PARTNER_REF})." >&2
        exit 1
        ;;
    *)
        echo "ERROR: ls-remote on origin failed." >&2
        exit 1
        ;;
esac
aosp_partner_ref_commit=$(echo "${aosp_partner_ref_list}" | awk '{ print $1 }')
head_commit=$(git rev-parse HEAD)


# Check if the current HEAD and the target reference already point to the same
# commit. In that case, we can skip all further fetch and merge steps.
if [ "_${head_commit}" = "_${aosp_partner_ref_commit}" ]; then
    echo "INFO: HEAD and AOSP-Partner reference point to the same commit. Nothing to do."
    exit 0
fi

target_branch="${REPO_RREV}"
# Drop the refs/heads/ prefix if it exists.
target_branch="${REPO_RREV#refs/heads/}"
ls_remote_ret_val=0
target_branch_commit=$(git ls-remote origin "refs/heads/${target_branch}" 2>/dev/null) \
    || ls_remote_ret_val=$?
target_branch_commit=$(echo "${target_branch_commit}" | awk '{ print $1 }')
if [ "${ls_remote_ret_val}" -ne 0 ]; then
    echo "ERROR: ls-remote on origin failed." >&2
    exit 1
fi
if [ -z "${target_branch_commit}" ]; then
    echo "ERROR: Could not determine the remote revision of the target branch. Is this project " \
"pointing to a branch?" >&2
    exit 1
fi
if [ "${target_branch_commit}" != "${head_commit}" ]; then
    if [ "${ALLOW_LOCAL_COMMITS}" -eq 1 ]; then
        echo "INFO: Continuing on top of local commits."
    else
        echo "ERROR: Aborting as we are not on the target branch and ALLOW_LOCAL_COMMITS is not " \
"enabled." >&2
        exit 1
    fi
fi

# Make sure we have the target branch history locally
# We need it, because merging and conflict resolution without the (shared)
# histories would not work well.
if _is_shallow_revision "${target_branch_commit}"; then
    if ! git fetch --no-tags --quiet --unshallow origin "${target_branch}"; then
        echo "ERROR: Could not unshallow the target branch." >&2
        exit 1
    fi
fi

# Now that we're unshallow on the current branch, check if the new revision is
# already part of our history. With checking that early on we can skip all
# further logic.
if _git_commit_exists "${aosp_partner_ref_commit}" \
        && git merge-base --is-ancestor "${aosp_partner_ref_commit}" HEAD; then
    echo "INFO: AOSP-Partner ref. is already merged. Nothing to do."
    exit 0
fi

# Make sure we have the AOSP-Partner reference locally.
if [ "${REF_IS_RELEASE_TAG}" -eq 1 ]; then
    echo "INFO: Fetching AOSP-Partner tag from origin (${AOSP_PARTNER_REF})…"
    _fetch_ref_spec="refs/tags/${AOSP_PARTNER_REF}"
else
    echo "INFO: Creating local branch for AOSP-Partner ref. fetched from origin " \
"(${AOSP_PARTNER_REF})…"
    _fetch_ref_spec="${AOSP_PARTNER_REF}"
fi
# Need to check if again if it's shallow. According to the checks above, the
# AOSP Partner ref is for sure not part of the current history, so it might
# still be shallow.
unshallow_param=()
if _is_shallow_revision "${aosp_partner_ref_commit}"; then
    unshallow_param=("--unshallow")
fi
ret_val=0
git fetch --no-tags --quiet "${unshallow_param[@]}" origin "${_fetch_ref_spec}:${_fetch_ref_spec}" \
    || ret_val=$?
if [ "${ret_val}" -ne 0 ]; then
    echo "ERROR: Could not fetch the AOSP-Partner ref." >&2
    exit 1
fi

merge_aosp_partner_on_head "${head_commit}" "${target_branch}"