Nox system test refactor (#60)
* Add basic noxfile to orchestrate system tests
* Move explicit system tests to nox
* Add cloud sdk environment automation
diff --git a/system_tests/conftest.py b/system_tests/conftest.py
index 5969bee..afa7854 100644
--- a/system_tests/conftest.py
+++ b/system_tests/conftest.py
@@ -23,20 +23,22 @@
HERE = os.path.dirname(__file__)
DATA_DIR = os.path.join(HERE, 'data')
-HTTP = urllib3.PoolManager()
+SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json')
+AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json')
+HTTP = urllib3.PoolManager(retries=False)
TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo'
@pytest.fixture
def service_account_file():
"""The full path to a valid service account key file."""
- yield os.path.join(DATA_DIR, 'service_account.json')
+ yield SERVICE_ACCOUNT_FILE
@pytest.fixture
def authorized_user_file():
"""The full path to a valid authorized user file."""
- yield os.path.join(DATA_DIR, 'authorized_user.json')
+ yield AUTHORIZED_USER_FILE
@pytest.fixture
@@ -67,6 +69,21 @@
yield _token_info
+@pytest.fixture
+def verify_refresh(http_request):
+ """Returns a function that verifies that credentials can be refreshed."""
+ def _verify_refresh(credentials):
+ if credentials.requires_scopes:
+ credentials = credentials.with_scopes(['email', 'profile'])
+
+ credentials.refresh(http_request)
+
+ assert credentials.token
+ assert credentials.valid
+
+ yield _verify_refresh
+
+
def verify_environment():
"""Checks to make sure that requisite data files are available."""
if not os.path.isdir(DATA_DIR):
diff --git a/system_tests/nox.py b/system_tests/nox.py
new file mode 100644
index 0000000..67f9900
--- /dev/null
+++ b/system_tests/nox.py
@@ -0,0 +1,211 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Noxfile for automating system tests.
+
+This file handles setting up environments needed by the system tests. This
+separates the tests from their environment configuration.
+
+See the `nox docs`_ for details on how this file works:
+
+.. _nox docs: http://nox.readthedocs.io/en/latest/
+"""
+
+import os
+
+from nox.command import which
+import py.path
+
+
+HERE = os.path.dirname(__file__)
+DATA_DIR = os.path.join(HERE, 'data')
+SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json')
+AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json')
+EXPLICIT_CREDENTIALS_ENV = 'GOOGLE_APPLICATION_CREDENTIALS'
+EXPLICIT_PROJECT_ENV = 'GOOGLE_CLOUD_PROJECT'
+EXPECT_PROJECT_ENV = 'EXPECT_PROJECT_ID'
+
+# The download location for the Cloud SDK
+CLOUD_SDK_DIST_FILENAME = 'google-cloud-sdk.tar.gz'
+CLOUD_SDK_DOWNLOAD_URL = (
+ 'https://dl.google.com/dl/cloudsdk/release/{}'.format(
+ CLOUD_SDK_DIST_FILENAME))
+
+# This environment variable is recognized by the Cloud SDK and overrides
+# the location of the SDK's configuration files (which is usually at
+# ${HOME}/.config).
+CLOUD_SDK_CONFIG_ENV = 'CLOUDSDK_CONFIG'
+
+# If set, this is where the environment setup will install the Cloud SDK.
+# If unset, it will download the SDK to a temporary directory.
+CLOUD_SDK_ROOT = os.environ.get('CLOUD_SDK_ROOT')
+
+if CLOUD_SDK_ROOT is not None:
+ CLOUD_SDK_ROOT = py.path.local(CLOUD_SDK_ROOT)
+ CLOUD_SDK_ROOT.ensure(dir=True) # Makes sure the directory exists.
+else:
+ CLOUD_SDK_ROOT = py.path.local.mkdtemp()
+
+# The full path the cloud sdk install directory
+CLOUD_SDK_INSTALL_DIR = CLOUD_SDK_ROOT.join('google-cloud-sdk')
+
+# The full path to the gcloud cli executable.
+GCLOUD = str(CLOUD_SDK_INSTALL_DIR.join('bin', 'gcloud'))
+
+# gcloud requires Python 2 and doesn't work on 3, so we need to tell it
+# where to find 2 when we're running in a 3 environment.
+CLOUD_SDK_PYTHON_ENV = 'CLOUDSDK_PYTHON'
+CLOUD_SDK_PYTHON = which('python2', None)
+
+# Cloud SDK helpers
+
+
+def install_cloud_sdk(session):
+ """Downloads and installs the Google Cloud SDK."""
+ # Configure environment variables needed by the SDK.
+ # This sets the config root to the tests' config root. This prevents
+ # our tests from clobbering a developer's configuration when running
+ # these tests locally.
+ session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT)
+ # This tells gcloud which Python interpreter to use (always use 2.7)
+ session.env[CLOUD_SDK_PYTHON_ENV] = CLOUD_SDK_PYTHON
+
+ # If the glcoud already exists, we don't need to do anything else.
+ # Note that because of this we do not attempt to update the sdk -
+ # if the CLOUD_SDK_ROOT is cached, it will need to be periodically cleared.
+ if py.path.local(GCLOUD).exists():
+ return
+
+ tar_path = CLOUD_SDK_ROOT.join(CLOUD_SDK_DIST_FILENAME)
+
+ # Download the release.
+ session.run(
+ 'wget', CLOUD_SDK_DOWNLOAD_URL, '-O', str(tar_path), silent=True)
+
+ # Extract the release.
+ session.run(
+ 'tar', 'xzf', str(tar_path), '-C', str(CLOUD_SDK_ROOT))
+ session.run(tar_path.remove)
+
+ # Run the install script.
+ session.run(
+ str(CLOUD_SDK_INSTALL_DIR.join('install.sh')),
+ '--usage-reporting', 'false',
+ '--path-update', 'false',
+ '--command-completion', 'false',
+ silent=True)
+
+
+def copy_credentials(credentials_path):
+ """Copies credentials into the SDK root as the application default
+ credentials."""
+ dest = CLOUD_SDK_ROOT.join('application_default_credentials.json')
+ if dest.exists():
+ dest.remove()
+ py.path.local(credentials_path).copy(dest)
+
+
+def configure_cloud_sdk(
+ session, application_default_credentials, project=False):
+ """Installs and configures the Cloud SDK with the given application default
+ credentials.
+
+ If project is True, then a project will be set in the active config.
+ If it is false, this will ensure no project is set.
+ """
+ install_cloud_sdk(session)
+
+ if project:
+ session.run(GCLOUD, 'config', 'set', 'project', 'example-project')
+ else:
+ session.run(GCLOUD, 'config', 'unset', 'project')
+
+ # Copy the credentials file to the config root. This is needed because
+ # unfortunately gcloud doesn't provide a clean way to tell it to use
+ # a particular set of credentials. However, this does verify that gcloud
+ # also considers the credentials valid by calling application-default
+ # print-access-token
+ session.run(copy_credentials, application_default_credentials)
+
+ # Calling this forces the Cloud SDK to read the credentials we just wrote
+ # and obtain a new access token with those credentials. This validates
+ # that our credentials matches the format expected by gcloud.
+ # Silent is set to True to prevent leaking secrets in test logs.
+ session.run(
+ GCLOUD, 'auth', 'application-default', 'print-access-token',
+ silent=True)
+
+
+# Test sesssions
+
+
+def session_service_account(session):
+ session.virtualenv = False
+ session.run('pytest', 'test_service_account.py')
+
+
+def session_oauth2_credentials(session):
+ session.virtualenv = False
+ session.run('pytest', 'test_oauth2_credentials.py')
+
+
+def session_default_explicit_service_account(session):
+ session.virtualenv = False
+ session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
+ session.env[EXPECT_PROJECT_ENV] = '1'
+ session.run('pytest', 'test_default.py')
+
+
+def session_default_explicit_authorized_user(session):
+ session.virtualenv = False
+ session.env[EXPLICIT_CREDENTIALS_ENV] = AUTHORIZED_USER_FILE
+ session.run('pytest', 'test_default.py')
+
+
+def session_default_explicit_authorized_user_explicit_project(session):
+ session.virtualenv = False
+ session.env[EXPLICIT_CREDENTIALS_ENV] = AUTHORIZED_USER_FILE
+ session.env[EXPLICIT_PROJECT_ENV] = 'example-project'
+ session.env[EXPECT_PROJECT_ENV] = '1'
+ session.run('pytest', 'test_default.py')
+
+
+def session_default_cloud_sdk_service_account(session):
+ session.virtualenv = False
+ configure_cloud_sdk(session, SERVICE_ACCOUNT_FILE)
+ session.env[EXPECT_PROJECT_ENV] = '1'
+ session.run('pytest', 'test_default.py')
+
+
+def session_default_cloud_sdk_authorized_user(session):
+ session.virtualenv = False
+ configure_cloud_sdk(session, AUTHORIZED_USER_FILE)
+ session.run('pytest', 'test_default.py')
+
+
+def session_default_cloud_sdk_authorized_user_configured_project(session):
+ session.virtualenv = False
+ configure_cloud_sdk(session, AUTHORIZED_USER_FILE, project=True)
+ session.env[EXPECT_PROJECT_ENV] = '1'
+ session.run('pytest', 'test_default.py')
+
+
+def session_compute_engine(session):
+ session.virtualenv = False
+ session.run('pytest', 'test_compute_engine.py')
+
+
+def session_app_engine(session):
+ session.virtualenv = False
+ session.run('pytest', 'app_engine/test_app_engine.py')
diff --git a/system_tests/test_compute_engine.py b/system_tests/test_compute_engine.py
index 7fd31b4..ceba0f7 100644
--- a/system_tests/test_compute_engine.py
+++ b/system_tests/test_compute_engine.py
@@ -21,7 +21,7 @@
@pytest.fixture(autouse=True)
def check_gce_environment(http_request):
- if not _metadata.ping(http_request):
+ if not _metadata.ping(http_request, timeout=1):
pytest.skip('Compute Engine metadata service is not available.')
diff --git a/system_tests/test_default.py b/system_tests/test_default.py
index 02242e9..0e0bcc6 100644
--- a/system_tests/test_default.py
+++ b/system_tests/test_default.py
@@ -14,108 +14,17 @@
import os
-import py
-
import google.auth
-from google.auth import environment_vars
-import google.oauth2.credentials
-from google.oauth2 import service_account
+
+EXPECT_PROJECT_ID = os.environ.get('EXPECT_PROJECT_ID')
-def validate_refresh(credentials, http_request):
- if credentials.requires_scopes:
- credentials = credentials.with_scopes(['email', 'profile'])
-
- credentials.refresh(http_request)
-
- assert credentials.token
- assert credentials.valid
-
-
-def test_explicit_credentials_service_account(
- monkeypatch, service_account_file, http_request):
- monkeypatch.setitem(
- os.environ, environment_vars.CREDENTIALS, service_account_file)
-
+def test_explicit_credentials(verify_refresh):
credentials, project_id = google.auth.default()
- assert isinstance(credentials, service_account.Credentials)
- assert project_id is not None
+ if EXPECT_PROJECT_ID is not None:
+ assert project_id is not None
+ else:
+ assert project_id is None
- validate_refresh(credentials, http_request)
-
-
-def test_explicit_credentials_authorized_user(
- monkeypatch, authorized_user_file, http_request):
- monkeypatch.setitem(
- os.environ, environment_vars.CREDENTIALS, authorized_user_file)
-
- credentials, project_id = google.auth.default()
-
- assert isinstance(credentials, google.oauth2.credentials.Credentials)
- assert project_id is None
-
- validate_refresh(credentials, http_request)
-
-
-def test_explicit_credentials_explicit_project_id(
- monkeypatch, service_account_file, http_request):
- project = 'system-test-project'
- monkeypatch.setitem(
- os.environ, environment_vars.CREDENTIALS, service_account_file)
- monkeypatch.setitem(
- os.environ, environment_vars.PROJECT, project)
-
- _, project_id = google.auth.default()
-
- assert project_id == project
-
-
-def generate_cloud_sdk_config(
- tmpdir, credentials_file, active_config='default', project=None):
- tmpdir.join('active_config').write(
- '{}\n'.format(active_config), ensure=True)
-
- if project is not None:
- config_file = tmpdir.join(
- 'configurations', 'config_{}'.format(active_config))
- config_file.write(
- '[core]\nproject = {}'.format(project), ensure=True)
-
- py.path.local(credentials_file).copy(
- tmpdir.join('application_default_credentials.json'))
-
-
-def test_cloud_sdk_credentials_service_account(
- tmpdir, monkeypatch, service_account_file, http_request):
- # Create the Cloud SDK configuration tree
- project = 'system-test-project'
- generate_cloud_sdk_config(tmpdir, service_account_file, project=project)
- monkeypatch.setitem(
- os.environ, environment_vars.CLOUD_SDK_CONFIG_DIR, str(tmpdir))
-
- credentials, project_id = google.auth.default()
-
- assert isinstance(credentials, service_account.Credentials)
- assert project_id is not None
- # The project ID should be the project ID specified in the the service
- # account file, not the project in the config.
- assert project_id is not project
-
- validate_refresh(credentials, http_request)
-
-
-def test_cloud_sdk_credentials_authorized_user(
- tmpdir, monkeypatch, authorized_user_file, http_request):
- # Create the Cloud SDK configuration tree
- project = 'system-test-project'
- generate_cloud_sdk_config(tmpdir, authorized_user_file, project=project)
- monkeypatch.setitem(
- os.environ, environment_vars.CLOUD_SDK_CONFIG_DIR, str(tmpdir))
-
- credentials, project_id = google.auth.default()
-
- assert isinstance(credentials, google.oauth2.credentials.Credentials)
- assert project_id == project
-
- validate_refresh(credentials, http_request)
+ verify_refresh(credentials)