Get the scheduler unittest to run against SQLite!
* get rid of monitor_db.DatabaseConn, and make monitor_db use the new DatabaseConnection
* modify some queries in monitor_db that weren't SQLite-compatible (SQLite doesn't support TRUE and FALSE literals)
* add frontend/django_test_utils.py, which contains utilities to
* setup a django environment (something manage.py normally does for you)
* replace the configured DB with a SQLite one, either in-memory or on disk
* run syncdb on the test DB
* backup and restore the test DB, handy because then we can syncdb once, save the fresh DB, and quickly restore it between unittests without having to run syncdb again (syncdb is terribly slow for whatever reason)
* modify monitor_db_unittest to use these methods to set up a temporary SQLite DB, run syncdb on it, and test against it
* replace much of the data modification code in monitor_db_unittest with use of the django models. The INSERTs were very problematic with SQLite because syncdb doesn't set database defaults, but using the models solves that (django inserts the defaults itself). using the models is much cleaner anyway as you can see. it was just difficult to do before, but now that we've got the infrastructure to setup the environment anyway, it's easy. this is a good model for how we can make the scheduler use the django models eventually.
* reorder fields of Label model to match actual DB ordering; this is necessary since monitor_db depends on field ordering
* add defaults to some fields in AFE models that should've had them
* make DatabaseConnection.get_test_database support SQLite in files, which gives us persistence that is necessary and handy in the scheduler unittest
* add a fix to _SqliteBackend for pysqlite2 crappiness
The following are extras that weren't strictly necessary to get things working:
* add a debug feature to DatabaseConnection to print all queries
* add an execute_script method to DatabaseConnection (it was duplicated in migrate and monitor_db_unittest)
* rename "arguments" to "parameters" in _GenericBackend.execute, to match the DB-API names
* get rid of some debug code that was left in monitor_db, and one unnecessary statement
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2252 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/scheduler/monitor_db.py b/scheduler/monitor_db.py
index cb237ad..10cac39 100644
--- a/scheduler/monitor_db.py
+++ b/scheduler/monitor_db.py
@@ -10,10 +10,12 @@
import common
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import host_protections, utils
+from autotest_lib.database import database_connection
RESULTS_DIR = '.'
AUTOSERV_NICE_LEVEL = 10
+CONFIG_SECTION = 'AUTOTEST_WEB'
AUTOTEST_PATH = os.path.join(os.path.dirname(__file__), '..')
@@ -78,7 +80,7 @@
# read in base url
global _base_url
- val = c.get_config_value("AUTOTEST_WEB", "base_url")
+ val = c.get_config_value(CONFIG_SECTION, "base_url")
if val:
_base_url = val
else:
@@ -111,9 +113,13 @@
print "%s> dispatcher starting" % time.strftime("%X %x")
print "My PID is %d" % os.getpid()
+ if _testing_mode:
+ global_config.global_config.override_config_value(
+ CONFIG_SECTION, 'database', 'stresstest_autotest_web')
+
os.environ['PATH'] = AUTOTEST_SERVER_DIR + ':' + os.environ['PATH']
global _db
- _db = DatabaseConn()
+ _db = database_connection.DatabaseConnection(CONFIG_SECTION)
_db.connect()
print "Setting signal handler"
@@ -152,73 +158,6 @@
os.remove(path)
-class DatabaseConn:
- def __init__(self):
- self.reconnect_wait = 20
- self.conn = None
- self.cur = None
-
- import MySQLdb.converters
- self.convert_dict = MySQLdb.converters.conversions
- self.convert_dict.setdefault(bool, self.convert_boolean)
-
-
- @staticmethod
- def convert_boolean(boolean, conversion_dict):
- 'Convert booleans to integer strings'
- return str(int(boolean))
-
-
- def connect(self, db_name=None):
- self.disconnect()
-
- # get global config and parse for info
- c = global_config.global_config
- dbase = "AUTOTEST_WEB"
- db_host = c.get_config_value(dbase, "host")
- if db_name is None:
- db_name = c.get_config_value(dbase, "database")
-
- if _testing_mode:
- db_name = 'stresstest_autotest_web'
-
- db_user = c.get_config_value(dbase, "user")
- db_pass = c.get_config_value(dbase, "password")
-
- while not self.conn:
- try:
- self.conn = MySQLdb.connect(
- host=db_host, user=db_user, passwd=db_pass,
- db=db_name, conv=self.convert_dict)
-
- self.conn.autocommit(True)
- self.cur = self.conn.cursor()
- except MySQLdb.OperationalError:
- traceback.print_exc()
- print "Can't connect to MYSQL; reconnecting"
- time.sleep(self.reconnect_wait)
- self.disconnect()
-
-
- def disconnect(self):
- if self.conn:
- self.conn.close()
- self.conn = None
- self.cur = None
-
-
- def execute(self, *args, **dargs):
- while (True):
- try:
- self.cur.execute(*args, **dargs)
- return self.cur.fetchall()
- except MySQLdb.OperationalError:
- traceback.print_exc()
- print "MYSQL connection died; reconnecting"
- time.sleep(self.reconnect_wait)
- self.connect()
-
-
def generate_parse_command(results_dir, flags=""):
parse = os.path.abspath(os.path.join(AUTOTEST_TKO_DIR, 'parse'))
output = os.path.abspath(os.path.join(results_dir, '.parse.log'))
@@ -320,9 +259,9 @@
hosts = Host.fetch(
joins='LEFT JOIN host_queue_entries AS active_hqe '
'ON (hosts.id = active_hqe.host_id AND '
- 'active_hqe.active = TRUE)',
+ 'active_hqe.active)',
where="active_hqe.host_id IS NULL "
- "AND hosts.locked = FALSE "
+ "AND NOT hosts.locked "
"AND (hosts.status IS NULL OR hosts.status = 'Ready')")
return dict((host.id, host) for host in hosts)
@@ -763,14 +702,14 @@
_db.execute(update + """
SET host_queue_entries.status = 'Abort'
- WHERE host_queue_entries.active IS TRUE""" + timed_out)
+ WHERE host_queue_entries.active""" + timed_out)
_db.execute(update + """
SET host_queue_entries.status = 'Aborted',
- host_queue_entries.active = FALSE,
- host_queue_entries.complete = TRUE
- WHERE host_queue_entries.active IS FALSE
- AND host_queue_entries.complete IS FALSE""" + timed_out)
+ host_queue_entries.active = 0,
+ host_queue_entries.complete = 1
+ WHERE NOT host_queue_entries.active
+ AND NOT host_queue_entries.complete""" + timed_out)
def _clear_inactive_blocks(self):
@@ -1660,7 +1599,6 @@
@classmethod
def fetch(cls, where='', params=(), joins='', order_by=''):
- table = cls._get_table()
order_by = cls._prefix_with(order_by, 'ORDER BY ')
where = cls._prefix_with(where, 'WHERE ')
query = ('SELECT %(table)s.* FROM %(table)s %(joins)s '