automated/lib/android-multinode-test-lib

#!/bin/sh

# Configure adb to accept connections via tcpip and make sure that the device as actually
# accessible. This includes assumes that the device has a network address. Guards around `adb tcpip`
# and test connections setups using `adb connect` guarantee so some degree that the device will be
# reachable after this call.
# Output: variable dutAddress set to ipAddress:adb_port
open_adb_tcpip_on_local_device() {
    [ "$#" -lt 2 -o "$#" -gt 3 ] && \
        error_fatal "Usage: open_adb_tcpip_on_local_device adb_tcpip_retries timeout_in_seconds [adb_port]"
    local adb_tcpip_retries="$1"
    local timeout_in_seconds="$2"
    local adb_port="$3"
    if [ -z "${adb_port}" ]; then
        local adb_port=5555    # default port assumed by adb connect
    fi

    local end=$(( $(date +%s) + timeout_in_seconds ))

    local retVal=0
    local ipAddress="$(get_ip_address ${timeout_in_seconds} || retVal=$?)"
    if [ "${retVal}" -ne 0 ]; then
        error_msg "get_ip_address failed unexpectedly."
        exit 1
    fi
    if [ -z "${ipAddress}" ]; then
        error_msg "Device has no ip address (network not connected?)"
        exit 1
    fi
    dutAddress="${ipAddress}:${adb_port}"

    # adb tcpip may fail with different reasons
    # (e.g., "error: protocol fault (couldn't read status): Connection reset by peer").
    # Just hope that it works after a few retries.
    local retry=0
    while [ "${retry}" -lt "${adb_tcpip_retries}" -a "$(date +%s)" -lt "$end" ]; do
      local retVal=0
      adb tcpip "${adb_port}" || retVal=$?
      if [ "${retVal}" -eq 0 ]; then break; fi
      info_msg "adb tcpip apparently failed. Retrying in a moment..."
      sleep 5
      adb usb || true # In between, make sure to have some default state.
    done
    if [ "${retVal}" -ne 0 ]; then
        error_msg "Could not prepare the device for adb tcpip connections: adb tcpip failed."
    fi
    # adb tcpip sometimes takes some time
    # (on FP2 Android 7 test builds, up to 10 seconds were observed)
    local success=false
    while [ "$(date +%s)" -lt "$end" ]; do
        if [ $(adb connect "${dutAddress}" | grep -c '^connected to ') -eq 1 ]; then
            local success=true
            break
        fi
        sleep 1
    done

    # Make sure the device is not reserved to the local adb server.
    adb disconnect "${dutAddress}" 1>/dev/null 2>/dev/null || true

    if [ "${success}" = false ]; then
        error_msg "Could not prepare the device for adb tcpip connections: device is not reachable via network."
    fi
}

# Make this device accessible via adb tcpip and send its address via handshake to a waiting role.
# NOTE: This function must only be called once per role per test submission, as LAVA does currently
# not allow to use the same MultiNode message ID multiple times.
# One job instance must call connect_to_remote_adb_tcpip_devices to receive the send addresses and
# complete the handshake.
# See open_adb_tcpip_on_local_device and connect_to_remote_adb_tcpip_devices
share_local_device_over_adb_tcpip() {
    [ "$#" -lt 2 -o "$#" -gt 3 ] && \
        error_fatal "Usage: share_local_device_over_adb_tcpip adb_tcpip_retries timeout_in_seconds [adb_port]"
    local adb_tcpip_retries="$1"
    local timeout_in_seconds="$2"
    local adb_port="$3"

    open_adb_tcpip_on_local_device "${adb_tcpip_retries}" "${timeout_in_seconds}" "${adb_port}"

    lava-sync startHandover
    lava-send dutAddress dutAddress="${dutAddress}"
    lava-sync finishHandover
}

# Counterpart to share_local_device_over_adb_tcpip
# Wait for other job instances to send their device address, guarded by a handshake for
# synchronization.
connect_to_remote_adb_tcpip_devices() {
    [ "$#" -lt 1 -o "$#" -gt 2 ] && \
        error_fatal "Usage: connect_to_remote_adb_tcpip_devices adb_connect_timeout [device_worker_mapping_file]"

    adb_connect_timeout="$1"
    device_worker_mapping_file="$2"

    lava-sync startHandover
    # For lava-wait-all, all involved nodes must invoke lava-send with the same message id,
    # otherwise lava-wait-all would lead do a dead lock.
    # However, only the nodes that make their device accessible (workers) add the value
    # dutAddress="address:port".
    lava-send dutAddress
    lava-wait-all dutAddress

    local cacheLines="$(cat "/tmp/lava_multi_node_cache.txt" | grep "dutAddress" | grep -v '^$')"
    local deviceWorkerMapping=""

    for line in $cacheLines; do
        # <workerJobId>:dutAddress=<dutAddress>
        local dutAddress="$(echo "$line"| sed 's/.*dutAddress=//')"
        local workerHost="$(echo "$line" | cut -d: -f1)"
        local deviceWorkerMapping="${deviceWorkerMapping}${dutAddress};${workerHost}\n"
    done
    local deviceWorkerMapping="$(echo "${deviceWorkerMapping}" | grep -v '^$')"

    lava-sync finishHandover
    # adb is not super reliable, it too often sees connected and authorized devices as "offline"
    adb kill-server || true

    # Connect to remote devices and wait until they appear online

    for deviceToWorker in ${deviceWorkerMapping}; do
        local device="$(echo ${deviceToWorker} | cut -d';' -f1)"
        for i in `seq 5`; do
            local retVal=0
            adb connect "${device}" || retVal="$?"
            if [ "${retVal}" -eq 0 ]; then
                break
            fi
            adb disconnect "${device}" || true
            warn_msg "adb connect failed. Retrying in a minute..."
            sleep 1m
        done
    done

    for deviceToWorker in ${deviceWorkerMapping}; do
        local device="$(echo ${deviceToWorker} | cut -d';' -f1)"
        local retVal=0
        if ! timeout "${adb_connect_timeout}" adb -s "${device}" wait-for-device; then
            error_msg "adb wait-for-device for ${device} timed out after ${adb_connect_timeout} seconds."
        fi
    done

    local numRemoteDevices="$(echo "${deviceWorkerMapping}" | wc -l)"
    info_msg "All ${numRemoteDevices} remote devices are connected and online."

    info_msg "Now adding devices locally connected via USB."
    local connectedDevices="$(adb devices | grep -E '^([:\.a-z0-9]+)\s+device$' | cut -f1)"
    local remoteOnlyMapping="${deviceWorkerMapping}"
    local deviceWorkerMapping="${deviceWorkerMapping}\n"
    for device in ${connectedDevices}; do
        if [ "$(echo "${remoteOnlyMapping}" | cut -d';' -f1 | grep -xc "${device}")" -eq 0 ]; then
            local deviceWorkerMapping="${deviceWorkerMapping}${device};\n"
            info_msg "Local device: ${device}"
        fi
    done
    local deviceWorkerMapping="$(echo "${deviceWorkerMapping}" | grep -v '^$')"

    if [ "${device_worker_mapping_file}" ]; then
        # Make mapping between attached DUTs and worker job ids and accessible to subsequent tests:
        echo "${deviceWorkerMapping}" > "${device_worker_mapping_file}"
        info_msg "Mapping between devices and to worker job ids stored in ${device_worker_mapping_file}:"
        info_msg "${deviceWorkerMapping}"
    else
        info_msg "NOT storing device to worker job id mapping, empty filename specified."
    fi
}