Snap for 9268991 from 671c3cd0d8681393fa2e4f9d4c07c768447c660a to tm-qpr2-release
Change-Id: I56cb599809616b58ddab8ca6c9f60a863bd3c330
diff --git a/atest/atest.py b/atest/atest.py
index a6e9a3d..794be87 100755
--- a/atest/atest.py
+++ b/atest/atest.py
@@ -233,6 +233,7 @@
'collect_tests_only': constants.COLLECT_TESTS_ONLY,
'custom_args': constants.CUSTOM_ARGS,
'disable_teardown': constants.DISABLE_TEARDOWN,
+ 'disable_upload_result': constants.DISABLE_UPLOAD_RESULT,
'dry_run': constants.DRY_RUN,
'enable_device_preparer': constants.ENABLE_DEVICE_PREPARER,
'flakes_info': constants.FLAKES_INFO,
diff --git a/atest/atest_arg_parser.py b/atest/atest_arg_parser.py
index c7caaa0..a285653 100644
--- a/atest/atest_arg_parser.py
+++ b/atest/atest_arg_parser.py
@@ -73,7 +73,14 @@
REBUILD_MODULE_INFO = ('Forces a rebuild of the module-info.json file. '
'This may be necessary following a repo sync or '
'when writing a new test.')
-REQUEST_UPLOAD_RESULT = 'Request permission to upload test result.'
+
+REQUEST_UPLOAD_RESULT = ('Request permission to upload test result. This option '
+ 'only needs to set once and takes effect until '
+ '--disable-upload-result is set.')
+DISABLE_UPLOAD_RESULT = ('Turn off the upload of test result. This option '
+ 'only needs to set once and takes effect until '
+ '--request-upload-result is set')
+
RERUN_UNTIL_FAILURE = ('Rerun all tests until a failure occurs or the max '
'iteration is reached. (default: forever!)')
# For Integer.MAX_VALUE == (2**31 - 1) and not possible to give a larger integer
@@ -175,8 +182,13 @@
action='store_true')
self.add_argument('-w', '--wait-for-debugger', action='store_true',
help=WAIT_FOR_DEBUGGER)
- self.add_argument('--request-upload-result', action='store_true',
+ # Options for request/disable upload results. They are mutually
+ # exclusive in a command line.
+ ugroup = self.add_mutually_exclusive_group()
+ ugroup.add_argument('--request-upload-result', action='store_true',
help=REQUEST_UPLOAD_RESULT)
+ ugroup.add_argument('--disable-upload-result', action='store_true',
+ help=DISABLE_UPLOAD_RESULT)
# Options related to Test Mapping
self.add_argument('-p', '--test-mapping', action='store_true',
@@ -338,6 +350,7 @@
CLEAR_CACHE=CLEAR_CACHE,
COLLECT_TESTS_ONLY=COLLECT_TESTS_ONLY,
DISABLE_TEARDOWN=DISABLE_TEARDOWN,
+ DISABLE_UPLOAD_RESULT=DISABLE_UPLOAD_RESULT,
DRY_RUN=DRY_RUN,
ENABLE_DEVICE_PREPARER=ENABLE_DEVICE_PREPARER,
ENABLE_FILE_PATTERNS=ENABLE_FILE_PATTERNS,
diff --git a/atest/constants_default.py b/atest/constants_default.py
index 4b370e0..e3122a0 100644
--- a/atest/constants_default.py
+++ b/atest/constants_default.py
@@ -61,6 +61,7 @@
TF_EARLY_DEVICE_RELEASE = 'TF_EARLY_DEVICE_RELEASE'
BAZEL_MODE_FEATURES = 'BAZEL_MODE_FEATURES'
REQUEST_UPLOAD_RESULT = 'REQUEST_UPLOAD_RESULT'
+DISABLE_UPLOAD_RESULT = 'DISABLE_UPLOAD_RESULT'
MODULES_IN = 'MODULES-IN-'
NO_ENABLE_ROOT = 'NO_ENABLE_ROOT'
VERIFY_ENV_VARIABLE = 'VERIFY_ENV_VARIABLE'
@@ -329,6 +330,11 @@
DISCOVERY_SERVICE = ''
STORAGE2_TEST_URI = ''
+# SSO constants.
+TOKEN_EXCHANGE_COMMAND = ''
+TOKEN_EXCHANGE_REQUEST = ''
+SCOPE = ''
+
# Example arguments used in ~/.atest/config
ATEST_EXAMPLE_ARGS = ('## Specify only one option per line; any test name/path will be ignored automatically.\n'
'## Option that follows a "#" will be ignored.\n'
diff --git a/atest/logstorage/atest_gcp_utils.py b/atest/logstorage/atest_gcp_utils.py
index 4d95577..a1cc684 100644
--- a/atest/logstorage/atest_gcp_utils.py
+++ b/atest/logstorage/atest_gcp_utils.py
@@ -28,8 +28,10 @@
"""
from __future__ import print_function
-import os
+import getpass
import logging
+import os
+import subprocess
import uuid
try:
import httplib2
@@ -37,9 +39,8 @@
logging.debug('Import error due to %s', e)
from pathlib import Path
-import constants
+from socket import socket
-from logstorage import logstorage_utils
try:
# pylint: disable=import-error
from oauth2client import client as oauth2_client
@@ -48,7 +49,9 @@
except ModuleNotFoundError as e:
logging.debug('Import error due to %s', e)
+from logstorage import logstorage_utils
import atest_utils
+import constants
class RunFlowFlags():
"""Flags for oauth2client.tools.run_flow."""
@@ -117,6 +120,18 @@
Returns:
An oauth2client.OAuth2Credentials instance.
"""
+ credentials = None
+ # SSO auth
+ try:
+ token = self._get_sso_access_token()
+ credentials = oauth2_client.AccessTokenCredentials(
+ token , 'atest')
+ if credentials:
+ return credentials
+ # pylint: disable=broad-except
+ except Exception as e:
+ logging.debug('Exception:%s', e)
+ # GCP auth flow
credentials = self.get_refreshed_credential_from_file(creds_file_path)
if not credentials:
storage = multistore_file.get_credential_storage(
@@ -130,21 +145,53 @@
def _run_auth_flow(self, storage):
"""Get user oauth2 credentials.
+ Using the loopback IP address flow for desktop clients.
+
Args:
storage: GCP storage object.
Returns:
An oauth2client.OAuth2Credentials instance.
"""
- flags = RunFlowFlags(browser_auth=False)
+ flags = RunFlowFlags(browser_auth=True)
+
+ # Get a free port on demand.
+ port = None
+ while not port or port < 10000:
+ with socket() as local_socket:
+ local_socket.bind(('',0))
+ _, port = local_socket.getsockname()
+ _localhost_port = port
+ _direct_uri = f'http://localhost:{_localhost_port}'
flow = oauth2_client.OAuth2WebServerFlow(
client_id=self.client_id,
client_secret=self.client_secret,
scope=self.scope,
- user_agent=self.user_agent)
+ user_agent=self.user_agent,
+ redirect_uri=f'{_direct_uri}')
credentials = oauth2_tools.run_flow(
flow=flow, storage=storage, flags=flags)
return credentials
+ def _get_sso_access_token(self):
+ """Use stubby command line to exchange corp sso to a scoped oauth
+ token.
+
+ Returns:
+ A token string.
+ """
+ if not constants.TOKEN_EXCHANGE_COMMAND:
+ return None
+
+ request = constants.TOKEN_EXCHANGE_REQUEST.format(
+ user=getpass.getuser(), scope=constants.SCOPE)
+ # The output format is: oauth2_token: "<TOKEN>"
+ return subprocess.run(constants.TOKEN_EXCHANGE_COMMAND,
+ input=request,
+ check=True,
+ text=True,
+ shell=True,
+ stdout=subprocess.PIPE).stdout.split('"')[1]
+
def do_upload_flow(extra_args):
"""Run upload flow.
@@ -157,9 +204,7 @@
tuple(invocation, workunit)
"""
config_folder = os.path.join(atest_utils.get_misc_dir(), '.atest')
- creds = request_consent_of_upload_test_result(
- config_folder,
- extra_args.get(constants.REQUEST_UPLOAD_RESULT, None))
+ creds = fetch_credential(config_folder, extra_args)
if creds:
inv, workunit, local_build_id, build_target = _prepare_data(creds)
extra_args[constants.INVOCATION_ID] = inv['invocationId']
@@ -169,56 +214,54 @@
if not os.path.exists(os.path.dirname(constants.TOKEN_FILE_PATH)):
os.makedirs(os.path.dirname(constants.TOKEN_FILE_PATH))
with open(constants.TOKEN_FILE_PATH, 'w') as token_file:
- token_file.write(creds.token_response['access_token'])
+ if creds.token_response:
+ token_file.write(creds.token_response['access_token'])
+ else:
+ token_file.write(creds.access_token)
return creds, inv
return None, None
-def request_consent_of_upload_test_result(config_folder,
- request_to_upload_result):
- """Request the consent of upload test results at the first time.
+def fetch_credential(config_folder, extra_args):
+ """Fetch the credential whenever --request-upload-result is specified.
Args:
- config_folder: The directory path to put config file.
- request_to_upload_result: Prompt message for user determine.
+ config_folder: The directory path to put config file. The default path
+ is ~/.atest.
+ extra_args: Dict of extra args to add to test run.
Return:
The credential object.
"""
if not os.path.exists(config_folder):
os.makedirs(config_folder)
- not_upload_file = os.path.join(config_folder,
- constants.DO_NOT_UPLOAD)
+ not_upload_file = os.path.join(config_folder, constants.DO_NOT_UPLOAD)
# Do nothing if there are no related config or DO_NOT_UPLOAD exists.
if (not constants.CREDENTIAL_FILE_NAME or
not constants.TOKEN_FILE_PATH):
return None
creds_f = os.path.join(config_folder, constants.CREDENTIAL_FILE_NAME)
- yn_result = False
- if request_to_upload_result:
- yn_result = atest_utils.prompt_with_yn_result(
- constants.UPLOAD_TEST_RESULT_MSG, False)
- if yn_result:
- if os.path.exists(not_upload_file):
- os.remove(not_upload_file)
- else:
+ if extra_args.get(constants.REQUEST_UPLOAD_RESULT):
+ if os.path.exists(not_upload_file):
+ os.remove(not_upload_file)
+ else:
+ if extra_args.get(constants.DISABLE_UPLOAD_RESULT):
if os.path.exists(creds_f):
os.remove(creds_f)
+ Path(not_upload_file).touch()
- # If the credential file exists or the user says “Yes”, ATest will
- # try to get the credential from the file, else will create a
- # DO_NOT_UPLOAD to keep the user's decision.
+ # If DO_NOT_UPLOAD not exist, ATest will try to get the credential
+ # from the file.
if not os.path.exists(not_upload_file):
- if os.path.exists(creds_f) or yn_result:
- return GCPHelper(
- client_id=constants.CLIENT_ID,
- client_secret=constants.CLIENT_SECRET,
- user_agent='atest').get_credential_with_auth_flow(creds_f)
+ return GCPHelper(
+ client_id=constants.CLIENT_ID,
+ client_secret=constants.CLIENT_SECRET,
+ user_agent='atest').get_credential_with_auth_flow(creds_f)
- Path(not_upload_file).touch()
atest_utils.colorful_print(
- 'WARNING: In order to allow upload local test results to AnTS, it '
- 'is recommended you add the option --request-upload-result.',
- constants.YELLOW)
+ 'WARNING: In order to allow uploading local test results to AnTS, it '
+ 'is recommended you add the option --request-upload-result. This option'
+ ' only needs to set once and takes effect until --disable-upload-result'
+ ' is set.', constants.YELLOW)
return None
def _prepare_data(creds):
@@ -256,9 +299,9 @@
"""
default_branch = ('git_master'
if constants.CREDENTIAL_FILE_NAME else 'aosp-master')
- local_branch = atest_utils.get_manifest_branch()
- branches = [b['name'] for b in build_client.list_branch()['branches']]
- return local_branch if local_branch in branches else default_branch
+ local_branch = "git_%s" % atest_utils.get_manifest_branch()
+ branch = build_client.get_branch(local_branch)
+ return local_branch if branch else default_branch
def _get_target(branch, build_client):
"""Get local build selected target.
diff --git a/atest/logstorage/atest_gcp_utils_unittest.py b/atest/logstorage/atest_gcp_utils_unittest.py
index 3a2ea05..89975bd 100644
--- a/atest/logstorage/atest_gcp_utils_unittest.py
+++ b/atest/logstorage/atest_gcp_utils_unittest.py
@@ -20,6 +20,7 @@
import tempfile
import unittest
+from pathlib import Path
from unittest import mock
import constants
@@ -30,7 +31,7 @@
"""Unit tests for atest_gcp_utils.py"""
@mock.patch.object(atest_gcp_utils, '_prepare_data')
- @mock.patch.object(atest_gcp_utils, 'request_consent_of_upload_test_result')
+ @mock.patch.object(atest_gcp_utils, 'fetch_credential')
def test_do_upload_flow(self, mock_request, mock_prepare):
"""test do_upload_flow method."""
fake_extra_args = {}
@@ -62,41 +63,77 @@
self.assertEqual(None, inv)
@mock.patch.object(atest_gcp_utils.GCPHelper,
- 'get_credential_with_auth_flow')
- @mock.patch('builtins.input')
- def test_request_consent_of_upload_test_result_yes(
- self, mock_input, mock_get_credential_with_auth_flow):
- """test request_consent_of_upload_test_result method."""
+ 'get_credential_with_auth_flow')
+ def test_fetch_credential_with_request_option(
+ self, mock_get_credential_with_auth_flow):
+ """test fetch_credential method."""
constants.CREDENTIAL_FILE_NAME = 'cred_file'
constants.GCP_ACCESS_TOKEN = 'access_token'
tmp_folder = tempfile.mkdtemp()
- mock_input.return_value = 'Y'
not_upload_file = os.path.join(tmp_folder,
constants.DO_NOT_UPLOAD)
- atest_gcp_utils.request_consent_of_upload_test_result(tmp_folder, True)
+ atest_gcp_utils.fetch_credential(tmp_folder,
+ {constants.REQUEST_UPLOAD_RESULT:True})
self.assertEqual(1, mock_get_credential_with_auth_flow.call_count)
self.assertFalse(os.path.exists(not_upload_file))
- atest_gcp_utils.request_consent_of_upload_test_result(tmp_folder, True)
+ atest_gcp_utils.fetch_credential(tmp_folder,
+ {constants.REQUEST_UPLOAD_RESULT:True})
self.assertEqual(2, mock_get_credential_with_auth_flow.call_count)
self.assertFalse(os.path.exists(not_upload_file))
@mock.patch.object(atest_gcp_utils.GCPHelper,
'get_credential_with_auth_flow')
- @mock.patch('builtins.input')
- def test_request_consent_of_upload_test_result_no(
- self, mock_input, mock_get_credential_with_auth_flow):
- """test request_consent_of_upload_test_result method."""
- mock_input.return_value = 'N'
+ def test_fetch_credential_with_disable_option(
+ self, mock_get_credential_with_auth_flow):
+ """test fetch_credential method."""
constants.CREDENTIAL_FILE_NAME = 'cred_file'
constants.GCP_ACCESS_TOKEN = 'access_token'
tmp_folder = tempfile.mkdtemp()
not_upload_file = os.path.join(tmp_folder,
constants.DO_NOT_UPLOAD)
- atest_gcp_utils.request_consent_of_upload_test_result(tmp_folder, True)
+ atest_gcp_utils.fetch_credential(tmp_folder,
+ {constants.DISABLE_UPLOAD_RESULT:True})
self.assertTrue(os.path.exists(not_upload_file))
self.assertEqual(0, mock_get_credential_with_auth_flow.call_count)
- atest_gcp_utils.request_consent_of_upload_test_result(tmp_folder, True)
+
+ atest_gcp_utils.fetch_credential(tmp_folder,
+ {constants.DISABLE_UPLOAD_RESULT:True})
self.assertEqual(0, mock_get_credential_with_auth_flow.call_count)
+
+ @mock.patch.object(atest_gcp_utils.GCPHelper,
+ 'get_credential_with_auth_flow')
+ def test_fetch_credential_no_upload_option(
+ self, mock_get_credential_with_auth_flow):
+ """test fetch_credential method."""
+ constants.CREDENTIAL_FILE_NAME = 'cred_file'
+ constants.GCP_ACCESS_TOKEN = 'access_token'
+ tmp_folder = tempfile.mkdtemp()
+ not_upload_file = os.path.join(tmp_folder,
+ constants.DO_NOT_UPLOAD)
+ fake_cred_file = os.path.join(tmp_folder,
+ constants.CREDENTIAL_FILE_NAME)
+
+ # mock cred file not exist, and not_upload file exists.
+ if os.path.exists(fake_cred_file):
+ os.remove(fake_cred_file)
+ if os.path.exists(not_upload_file):
+ os.remove(not_upload_file)
+ Path(not_upload_file).touch()
+
+ atest_gcp_utils.fetch_credential(tmp_folder, dict())
+ self.assertEqual(0, mock_get_credential_with_auth_flow.call_count)
+ self.assertTrue(os.path.exists(not_upload_file))
+
+ # mock cred file exists, and not_upload_file not exist.
+ if os.path.exists(fake_cred_file):
+ os.remove(fake_cred_file)
+ Path(fake_cred_file).touch()
+ if os.path.exists(not_upload_file):
+ os.remove(not_upload_file)
+
+ atest_gcp_utils.fetch_credential(tmp_folder, dict())
+ self.assertEqual(1, mock_get_credential_with_auth_flow.call_count)
+ self.assertFalse(os.path.exists(not_upload_file))
diff --git a/atest/logstorage/logstorage_utils.py b/atest/logstorage/logstorage_utils.py
index a18e7bb..5f056f2 100644
--- a/atest/logstorage/logstorage_utils.py
+++ b/atest/logstorage/logstorage_utils.py
@@ -52,6 +52,7 @@
logging.debug('Import error due to: %s', e)
import constants
+import metrics
class BuildClient:
@@ -79,6 +80,19 @@
return self.client.target().list(branch=branch,
maxResults=10000).execute()
+ def get_branch(self, branch):
+ """Get BuildInfo for specific branch.
+ Args:
+ branch: A string of branch name to query.
+ """
+ query_branch = ''
+ try:
+ query_branch = self.client.branch().get(resourceId=branch).execute()
+ # pylint: disable=broad-except
+ except Exception:
+ return ''
+ return query_branch
+
def insert_local_build(self, external_id, target, branch):
"""Insert a build record.
Args:
@@ -129,6 +143,7 @@
A build invocation object.
"""
sponge_invocation_id = str(uuid.uuid4())
+ user_email = metrics.metrics_base.get_user_email()
invocation = {
"primaryBuild": {
"buildId": build_record['buildId'],
@@ -138,6 +153,7 @@
"schedulerState": "running",
"runner": "atest",
"scheduler": "atest",
+ "users": [user_email],
"properties": [{
'name': 'sponge_invocation_id',
'value': sponge_invocation_id,
diff --git a/atest/metrics/metrics_base.py b/atest/metrics/metrics_base.py
index b2cabc8..a058c8c 100644
--- a/atest/metrics/metrics_base.py
+++ b/atest/metrics/metrics_base.py
@@ -47,6 +47,25 @@
EXTERNAL_USER: 934
}
+def get_user_email():
+ """Get user mail in git config.
+
+ Returns:
+ user's email.
+ """
+ try:
+ output = subprocess.check_output(
+ ['git', 'config', '--get', 'user.email'], universal_newlines=True)
+ return output.strip() if output else ''
+ except OSError:
+ # OSError can be raised when running atest_unittests on a host
+ # without git being set up.
+ logging.debug('Unable to determine if this is an external run, git is '
+ 'not found.')
+ except subprocess.CalledProcessError:
+ logging.debug('Unable to determine if this is an external run, email '
+ 'is not found in git config.')
+ return ''
def get_user_type():
"""Get user type.
diff --git a/atest/test_runners/atest_tf_test_runner.py b/atest/test_runners/atest_tf_test_runner.py
index f352db5..ed15b73 100644
--- a/atest/test_runners/atest_tf_test_runner.py
+++ b/atest/test_runners/atest_tf_test_runner.py
@@ -1064,7 +1064,8 @@
constants.ENABLE_DEVICE_PREPARER,
constants.DRY_RUN,
constants.VERIFY_ENV_VARIABLE,
- constants.FLAKES_INFO):
+ constants.FLAKES_INFO,
+ constants.DISABLE_UPLOAD_RESULT):
continue
unsupported_args.append(arg)
return supported_args, unsupported_args