[autotest] Move database configuration to separate file.
As we're about to add more databases the logic for parsing the
database configurations gets more complicated.
This centralizes the configuration parsing in one dedicated file.
BUG=chromium:419435
DEPLOY=apache,scheduler,host_scheduler,shard_client
TEST=Ran suites, tried syncdb and apache restart
Change-Id: I486b72f30ccd0f1e44316927b4fe3d5c93eacbe2
Reviewed-on: https://chromium-review.googlesource.com/223377
Reviewed-by: Dan Shi <dshi@chromium.org>
Commit-Queue: Jakob Jülich <jakobjuelich@chromium.org>
Tested-by: Jakob Jülich <jakobjuelich@chromium.org>
diff --git a/frontend/database_settings_helper.py b/frontend/database_settings_helper.py
new file mode 100644
index 0000000..a68bd60
--- /dev/null
+++ b/frontend/database_settings_helper.py
@@ -0,0 +1,150 @@
+#pylint: disable-msg=C0111
+
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helpers to load database settings.
+
+Three databases are used with django (a default and one for tko tables,
+which always must be the global database, plus a readonly connection to the
+global database).
+
+In order to save configuration overhead, settings that aren't set for the
+desired database type, should be obtained from the setting with the next lower
+priority. The order is:
+readonly -> global -> local.
+I.e. this means if `readonly_host` is not set, `global_db_host` will be used. If
+that is also not set, `host` (the local one) will be used.
+
+In case an instance is running on a shard, a global database must explicitly
+be set. Instead of failing over from global to local, an exception will be
+raised in that case.
+
+The complexity to do this, is combined in this file.
+"""
+
+
+# Don't import anything that needs django here: Django may not be configured
+# on the builders, and this is also used by tko/db.py so failures like this
+# may occur: http://crbug.com/421565
+import common
+from autotest_lib.client.common_lib import global_config
+
+config = global_config.global_config
+SHARD_HOSTNAME = config.get_config_value('SHARD', 'shard_hostname',
+ default=None)
+
+
+def _get_config(config_key, **kwargs):
+ """Retrieves a config value for the specified key.
+
+ @param config_key: The string key associated with the desired config value.
+ @param **kwargs: Additional arguments to be passed to
+ global_config.get_config_value.
+
+ @return: The config value, as returned by
+ global_config.global_config.get_config_value().
+ """
+ return config.get_config_value('AUTOTEST_WEB', config_key, **kwargs)
+
+
+def _get_global_config(config_key, default=config._NO_DEFAULT_SPECIFIED, **kwargs):
+ """Retrieves a global config value for the specified key.
+
+ If the value can't be found, this will happen:
+ - if no default value was specified, and this is run on a shard instance,
+ a ConfigError will be raised.
+ - if a default value is set or this is run on a non-shard instancee, the
+ non-global value is returned
+
+ @param config_key: The string key associated with the desired config value.
+ @param default: The default value to return if the value couldn't be looked
+ up; neither with global_db_ nor no prefix.
+ @param **kwargs: Additional arguments to be passed to
+ global_config.get_config_value.
+
+ @return: The config value, as returned by
+ global_config.global_config.get_config_value().
+ """
+ try:
+ return _get_config('global_db_' + config_key, **kwargs)
+ except global_config.ConfigError:
+ if SHARD_HOSTNAME and default == config._NO_DEFAULT_SPECIFIED:
+ # When running on a shard, fail loudly if the global_db_ prefixed
+ # settings aren't present.
+ raise
+ return _get_config(config_key, default=default, **kwargs)
+
+
+def _get_readonly_config(config_key, default=config._NO_DEFAULT_SPECIFIED,
+ **kwargs):
+ """Retrieves a readonly config value for the specified key.
+
+ If no value can be found, the value of non readonly but global value
+ is returned instead.
+
+ @param config_key: The string key associated with the desired config value.
+ @param default: The default value to return if the value couldn't be looked
+ up; neither with readonly_, global_db_ nor no prefix.
+ @param **kwargs: Additional arguments to be passed to
+ global_config.get_config_value.
+
+ @return: The config value, as returned by
+ global_config.global_config.get_config_value().
+ """
+ try:
+ return _get_config('readonly_' + config_key, **kwargs)
+ except global_config.ConfigError:
+ return _get_global_config(config_key, default=default, **kwargs)
+
+
+def _get_database_config(getter):
+ """Create a configuration dictionary that can be passed to Django.
+
+ @param config_prefix: If specified, this function will try to prefix lookup
+ keys from global_config with this. If those values
+ don't exist, the normal key without the prefix will
+ be used.
+
+ @return A dictionary that can be used in the Django DATABASES setting.
+ """
+ config = {
+ 'ENGINE': 'autotest_lib.frontend.db.backends.afe',
+ 'PORT': '',
+ 'HOST': getter('host'),
+ 'NAME': getter('database'),
+ 'USER': getter('user'),
+ 'PASSWORD': getter('password', default=''),
+ 'READONLY_HOST': getter('readonly_host', default=getter('host')),
+ 'READONLY_USER': getter('readonly_user', default=getter('user')),
+ }
+ if config['READONLY_USER'] != config['USER']:
+ config['READONLY_PASSWORD'] = getter('readonly_password', default='')
+ else:
+ config['READONLY_PASSWORD'] = config['PASSWORD']
+ return config
+
+
+def get_global_db_config():
+ """Returns settings for the global database as required by django.
+
+ @return: A dictionary that can be used in the Django DATABASES setting.
+ """
+ return _get_database_config(getter=_get_global_config)
+
+
+def get_default_db_config():
+ """Returns settings for the default/local database as required by django.
+
+ @return: A dictionary that can be used in the Django DATABASES setting.
+ """
+ return _get_database_config(getter=_get_config)
+
+
+def get_readonly_db_config():
+ """Returns settings for the readonly database as required by django.
+
+ @return: A dictionary that can be used in the Django DATABASES setting.
+ """
+ return _get_database_config(getter=_get_readonly_config)
diff --git a/frontend/settings.py b/frontend/settings.py
index df70915..0e8f502 100644
--- a/frontend/settings.py
+++ b/frontend/settings.py
@@ -4,6 +4,7 @@
import os
import common
from autotest_lib.client.common_lib import global_config
+from autotest_lib.frontend import database_settings_helper
c = global_config.global_config
_section = 'AUTOTEST_WEB'
@@ -20,38 +21,17 @@
MANAGERS = ADMINS
-def _get_config(config_key, default=None):
- """Retrieves a global config value for the specified key.
-
- @param config_key: The string key associated with the desired config value.
- @param default: The default value to return if an existing one cannot be
- found.
-
- @return The config value, as returned by
- global_config.global_config.get_config_value().
- """
- return c.get_config_value(_section, config_key, default=default)
-
-AUTOTEST_DEFAULT = {
- 'ENGINE': 'autotest_lib.frontend.db.backends.afe',
- 'PORT': '',
- 'HOST': _get_config("host"),
- 'NAME': _get_config("database"),
- 'USER': _get_config("user"),
- 'PASSWORD': _get_config("password", default=''),
- 'READONLY_HOST': _get_config("readonly_host", default=_get_config("host")),
- 'READONLY_USER': _get_config("readonly_user", default=_get_config("user")),
-}
+AUTOTEST_DEFAULT = database_settings_helper.get_default_db_config()
ALLOWED_HOSTS = '*'
-if AUTOTEST_DEFAULT['READONLY_USER'] != AUTOTEST_DEFAULT['USER']:
- AUTOTEST_DEFAULT['READONLY_PASSWORD'] = _get_config("readonly_password",
- default='')
-else:
- AUTOTEST_DEFAULT['READONLY_PASSWORD'] = AUTOTEST_DEFAULT['PASSWORD']
+DATABASES = {'default': AUTOTEST_DEFAULT,}
-DATABASES = {'default': AUTOTEST_DEFAULT}
+# Have to set SECRET_KEY before importing connections because of this bug:
+# https://code.djangoproject.com/ticket/20704
+# TODO: Order this again after an upgrade to Django 1.6 or higher.
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = 'pn-t15u(epetamdflb%dqaaxw+5u&2#0u-jah70w1l*_9*)=n7'
# prefix applied to all URLs - useful if requests are coming through apache,
# and you need this app to coexist with others
@@ -93,9 +73,6 @@
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'
-# Make this unique, and don't share it with anybody.
-SECRET_KEY = 'pn-t15u(epetamdflb%dqaaxw+5u&2#0u-jah70w1l*_9*)=n7'
-
# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
diff --git a/global_config.ini b/global_config.ini
index c360eb0..8cf008b 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -34,14 +34,14 @@
# The tko parser will use these database settings.
# This is for sharding: Even when sharding, the results (tko tables) should
# still be written to the master database.
-global_db_host: 172.18.72.10
-global_db_database: chromeos_autotest_db
-global_db_type: mysql
-global_db_user: chromeosqa-admin
-global_db_password: USE SHADOW PASSWORD
-global_db_query_timeout: 3600
-global_db_min_retry_delay: 20
-global_db_max_retry_delay: 60
+global_db_host:
+global_db_database:
+global_db_type:
+global_db_user:
+global_db_password:
+global_db_query_timeout:
+global_db_min_retry_delay:
+global_db_max_retry_delay:
[SHARD]
# If this is not None, the instance is considered a shard.
diff --git a/tko/db.py b/tko/db.py
index bdce8c7..35efbc1 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -1,7 +1,12 @@
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
import re, os, sys, types, time, random
import common
from autotest_lib.client.common_lib import global_config
+from autotest_lib.frontend import database_settings_helper
from autotest_lib.tko import utils
@@ -37,37 +42,44 @@
def _load_config(self, host, database, user, password):
- # grab the global config
- get_value = global_config.global_config.get_config_value
+ """Loads configuration settings required to connect to the database.
+
+ This will try to connect to use the settings prefixed with global_db_.
+ If they do not exist, they un-prefixed settings will be used.
+
+ If parameters are supplied, these will be taken instead of the values
+ in global_config.
+
+ @param host: If set, this host will be used, if not, the host will be
+ retrieved from global_config.
+ @param database: If set, this database will be used, if not, the
+ database will be retrieved from global_config.
+ @param user: If set, this user will be used, if not, the
+ user will be retrieved from global_config.
+ @param password: If set, this password will be used, if not, the
+ password will be retrieved from global_config.
+ """
+ database_settings = database_settings_helper.get_global_db_config()
# grab the host, database
- if host:
- self.host = host
- else:
- self.host = get_value("AUTOTEST_WEB", "global_db_host")
- if database:
- self.database = database
- else:
- self.database = get_value("AUTOTEST_WEB", "global_db_database")
+ self.host = host or database_settings['HOST']
+ self.database = database or database_settings['NAME']
# grab the user and password
- if user:
- self.user = user
- else:
- self.user = get_value("AUTOTEST_WEB", "global_db_user")
- if password is not None:
- self.password = password
- else:
- self.password = get_value("AUTOTEST_WEB", "global_db_password")
+ self.user = user or database_settings['USER']
+ self.password = password or database_settings['PASSWORD']
# grab the timeout configuration
- self.query_timeout = get_value("AUTOTEST_WEB",
- "global_db_query_timeout",
- type=int, default=3600)
+ self.query_timeout =(
+ database_settings.get('OPTIONS', {}).get('timeout', 3600))
+
+ # Using fallback to non-global in order to work without configuration
+ # overhead on non-shard instances.
+ get_value = global_config.global_config.get_config_value_with_fallback
self.min_delay = get_value("AUTOTEST_WEB", "global_db_min_retry_delay",
- type=int, default=20)
+ "min_retry_delay", type=int, default=20)
self.max_delay = get_value("AUTOTEST_WEB", "global_db_max_retry_delay",
- type=int, default=60)
+ "max_retry_delay", type=int, default=60)
def _init_db(self):
@@ -552,8 +564,9 @@
def _get_db_type():
"""Get the database type name to use from the global config."""
- get_value = global_config.global_config.get_config_value
- return "db_" + get_value("AUTOTEST_WEB", "global_db_type", default="mysql")
+ get_value = global_config.global_config.get_config_value_with_fallback
+ return "db_" + get_value("AUTOTEST_WEB", "global_db_type", "db_type",
+ default="mysql")
def _get_error_class(class_name):
diff --git a/tko/site_parse_unittest.py b/tko/site_parse_unittest.py
index f54b772..82925a6 100755
--- a/tko/site_parse_unittest.py
+++ b/tko/site_parse_unittest.py
@@ -11,6 +11,7 @@
import common
from autotest_lib.client.common_lib import global_config
+from autotest_lib.frontend import database_settings_helper
from autotest_lib.frontend import setup_django_environment
from autotest_lib.frontend import setup_test_environment
from autotest_lib.tko.site_parse import StackTrace
@@ -106,5 +107,25 @@
self.assertEqual(version, '1166.0.0')
+ def testRunOnShardWithoutGlobalConfigsFails(self):
+ global_config.global_config.override_config_value(
+ 'SHARD', 'shard_hostname', 'host1')
+ # settings module was already loaded during the imports of this file,
+ # so before the configuration setting was made, therefore reload it:
+ reload(database_settings_helper)
+ self.assertRaises(global_config.ConfigError,
+ database_settings_helper.get_global_db_config)
+
+
+ def testRunOnMasterWithoutGlobalConfigsWorks(self):
+ global_config.global_config.override_config_value(
+ 'SHARD', 'shard_hostname', '')
+ from autotest_lib.frontend import database_settings_helper
+ # settings module was already loaded during the imports of this file,
+ # so before the configuration setting was made, therefore reload it:
+ reload(database_settings_helper)
+ database_settings_helper.get_global_db_config()
+
+
if __name__ == "__main__":
unittest.main()