Adds extra fields to tko proto buffer definition.
Also fixed all the docs to pass pylint.
BUG=chromium:660218
TEST=unitest and local test.
Change-Id: Ibcf110be84a71353947443672f0ea3b795f33042
Reviewed-on: https://chromium-review.googlesource.com/405001
Commit-Ready: Michael Tang <ntang@chromium.org>
Tested-by: Michael Tang <ntang@chromium.org>
Reviewed-by: Moises Osorio <moisesosorio@chromium.org>
Reviewed-by: Michael Tang <ntang@chromium.org>
diff --git a/tko/db.py b/tko/db.py
index 362feae..a70774d 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -13,10 +13,13 @@
class MySQLTooManyRows(Exception):
+ """Too many records."""
pass
class db_sql(object):
+ """Data access."""
+
def __init__(self, debug=False, autocommit=True, host=None,
database=None, user=None, password=None):
self.debug = debug
@@ -129,7 +132,12 @@
It can be safely used with transactions, but the
transaction start & end must be completely contained
- within the call to 'function'."""
+ within the call to 'function'.
+
+ @param function: The function to run with retry.
+ @param args: The arguments
+ @param dargs: The named arguments.
+ """
OperationalError = _get_error_class("OperationalError")
success = False
@@ -163,6 +171,10 @@
def dprint(self, value):
+ """Print out debug value.
+
+ @param value: The value to print out.
+ """
if self.debug:
sys.stdout.write('SQL: ' + str(value) + '\n')
@@ -174,6 +186,7 @@
def commit(self):
+ """Commit the sql transaction."""
if self.autocommit:
return self.run_with_retry(self._commit)
else:
@@ -181,10 +194,15 @@
def rollback(self):
+ """Rollback the sql transaction."""
self.con.rollback()
def get_last_autonumber_value(self):
+ """Gets the last auto number.
+
+ @return: The last auto number.
+ """
self.cur.execute('SELECT LAST_INSERT_ID()', [])
return self.cur.fetchall()[0][0]
@@ -240,6 +258,13 @@
where = ("a = %s AND b = %s", ['val', 'val'])
is better than
where = "a = 'val' AND b = 'val'"
+
+ @param fields: The list of selected fields string.
+ @param table: The name of the database table.
+ @param where: The where clause string.
+ @param distinct: If select distinct values.
+ @param group_by: Group by clause.
+ @param max_rows: unused.
"""
cmd = ['select']
if distinct:
@@ -256,6 +281,7 @@
# create a re-runable function for executing the query
def exec_sql():
+ """Exeuctes an the sql command."""
sql = ' '.join(cmd)
numRec = self.cur.execute(sql, values)
if max_rows is not None and numRec > max_rows:
@@ -273,29 +299,34 @@
def select_sql(self, fields, table, sql, values):
"""\
select fields from table "sql"
+
+ @param fields: The list of selected fields string.
+ @param table: The name of the database table.
+ @param sql: The sql string.
+ @param values: The sql string parameter values.
"""
cmd = 'select %s from %s %s' % (fields, table, sql)
self.dprint(cmd)
# create a -re-runable function for executing the query
- def exec_sql():
+ def _exec_sql():
self.cur.execute(cmd, values)
return self.cur.fetchall()
# run the query, re-trying after operational errors
if self.autocommit:
- return self.run_with_retry(exec_sql)
+ return self.run_with_retry(_exec_sql)
else:
- return exec_sql()
+ return _exec_sql()
def _exec_sql_with_commit(self, sql, values, commit):
if self.autocommit:
# re-run the query until it succeeds
- def exec_sql():
+ def _exec_sql():
self.cur.execute(sql, values)
self.con.commit()
- self.run_with_retry(exec_sql)
+ self.run_with_retry(_exec_sql)
else:
# take one shot at running the query
self.cur.execute(sql, values)
@@ -309,6 +340,10 @@
data:
dictionary of fields and data
+
+ @param table: The name of the table.
+ @param data: The insert data.
+ @param commit: If commit the transaction .
"""
fields = data.keys()
refs = ['%s' for field in fields]
@@ -322,6 +357,12 @@
def delete(self, table, where, commit = None):
+ """Delete entries.
+
+ @param table: The name of the table.
+ @param where: The where clause.
+ @param commit: If commit the transaction .
+ """
cmd = ['delete from', table]
if commit is None:
commit = self.autocommit
@@ -339,6 +380,11 @@
data:
dictionary of fields and data
+
+ @param table: The name of the table.
+ @param data: The sql parameter values.
+ @param where: The where clause.
+ @param commit: If commit the transaction .
"""
if commit is None:
commit = self.autocommit
@@ -358,6 +404,11 @@
def delete_job(self, tag, commit = None):
+ """Delete a tko job.
+
+ @param tag: The job tag.
+ @param commit: If commit the transaction .
+ """
job_idx = self.find_job(tag)
for test_idx in self.find_tests(job_idx):
where = {'test_idx' : test_idx}
@@ -372,6 +423,13 @@
def insert_job(self, tag, job, parent_job_id=None, commit=None):
+ """Insert a tko job.
+
+ @param tag: The job tag.
+ @param job: The job object.
+ @param parent_job_id: The parent job id.
+ @param commit: If commit the transaction .
+ """
job.machine_idx = self.lookup_machine(job.machine)
if not job.machine_idx:
job.machine_idx = self.insert_machine(job, commit=commit)
@@ -391,13 +449,19 @@
'finished_time': job.finished_time,
'afe_job_id': afe_job_id,
'afe_parent_job_id': parent_job_id}
+ job.afe_job_id = afe_job_id
+ if parent_job_id:
+ job.afe_parent_job_id = str(parent_job_id)
if job.label:
label_info = site_utils.parse_job_name(job.label)
if label_info:
data['build'] = label_info.get('build', None)
- data['build_version'] = label_info.get('build_version', None)
- data['board'] = label_info.get('board', None)
- data['suite'] = label_info.get('suite', None)
+ job.build_version = data['build_version'] = label_info.get(
+ 'build_version', None)
+ job.board = data['board'] = label_info.get('board', None)
+ job.suite = data['suite'] = label_info.get('suite', None)
+
+ # TODO(ntang): check job.index directly.
is_update = hasattr(job, 'index')
if is_update:
self.update('tko_jobs', data, {'job_idx': job.index}, commit=commit)
@@ -410,6 +474,11 @@
def update_job_keyvals(self, job, commit=None):
+ """Updates the job key values.
+
+ @param job: The job object.
+ @param commit: If commit the transaction .
+ """
for key, value in job.keyval_dict.iteritems():
where = {'job_id': job.index, 'key': key}
data = dict(where, value=value)
@@ -422,6 +491,12 @@
def insert_test(self, job, test, commit = None):
+ """Inserts a job test.
+
+ @param job: The job object.
+ @param test: The test object.
+ @param commit: If commit the transaction .
+ """
kver = self.insert_kernel(test.kernel, commit=commit)
data = {'job_idx':job.index, 'test':test.testname,
'subdir':test.subdir, 'kernel_idx':kver,
@@ -472,6 +547,7 @@
def read_machine_map(self):
+ """Reads the machine map."""
if self.machine_group or not self.machine_map:
return
for line in open(self.machine_map, 'r').readlines():
@@ -480,6 +556,12 @@
def machine_info_dict(self, job):
+ """Reads the machine information of a job.
+
+ @param job: The job object.
+
+ @return: The machine info dictionary.
+ """
hostname = job.machine
group = job.machine_group
owner = job.machine_owner
@@ -494,12 +576,22 @@
def insert_machine(self, job, commit = None):
+ """Inserts the job machine.
+
+ @param job: The job object.
+ @param commit: If commit the transaction .
+ """
machine_info = self.machine_info_dict(job)
self.insert('tko_machines', machine_info, commit=commit)
return self.get_last_autonumber_value()
def update_machine_information(self, job, commit = None):
+ """Updates the job machine information.
+
+ @param job: The job object.
+ @param commit: If commit the transaction .
+ """
machine_info = self.machine_info_dict(job)
self.update('tko_machines', machine_info,
where={'hostname': machine_info['hostname']},
@@ -507,6 +599,10 @@
def lookup_machine(self, hostname):
+ """Look up the machine information.
+
+ @param hostname: The hostname as string.
+ """
where = { 'hostname' : hostname }
rows = self.select('machine_idx', 'tko_machines', where)
if rows:
@@ -516,6 +612,10 @@
def lookup_kernel(self, kernel):
+ """Look up the kernel.
+
+ @param kernel: The kernel object.
+ """
rows = self.select('kernel_idx', 'tko_kernels',
{'kernel_hash':kernel.kernel_hash})
if rows:
@@ -525,6 +625,11 @@
def insert_kernel(self, kernel, commit = None):
+ """Insert a kernel.
+
+ @param kernel: The kernel object.
+ @param commit: If commit the transaction .
+ """
kver = self.lookup_kernel(kernel)
if kver:
return kver
@@ -558,6 +663,12 @@
def insert_patch(self, kver, patch, commit = None):
+ """Insert a kernel patch.
+
+ @param kver: The kernel version.
+ @param patch: The kernel patch object.
+ @param commit: If commit the transaction .
+ """
print patch.reference
name = os.path.basename(patch.reference)[:80]
self.insert('tko_patches',
@@ -569,6 +680,12 @@
def find_test(self, job_idx, testname, subdir):
+ """Find a test by name.
+
+ @param job_idx: The job index.
+ @param testname: The test name.
+ @param subdir: The test sub directory under the job directory.
+ """
where = {'job_idx': job_idx , 'test': testname, 'subdir': subdir}
rows = self.select('test_idx', 'tko_tests', where)
if rows:
@@ -578,6 +695,11 @@
def find_tests(self, job_idx):
+ """Find all tests by job index.
+
+ @param job_idx: The job index.
+ @return: A list of tests.
+ """
where = { 'job_idx':job_idx }
rows = self.select('test_idx', 'tko_tests', where)
if rows:
@@ -587,6 +709,11 @@
def find_job(self, tag):
+ """Find a job by tag.
+
+ @param tag: The job tag name.
+ @return: The job object or None.
+ """
rows = self.select('job_idx', 'tko_jobs', {'tag': tag})
if rows:
return rows[0][0]
@@ -612,7 +739,13 @@
def db(*args, **dargs):
"""Creates an instance of the database class with the arguments
provided in args and dargs, using the database type specified by
- the global configuration (defaulting to mysql)."""
+ the global configuration (defaulting to mysql).
+
+ @param args: The db_type arguments.
+ @param dargs: The db_type named arguments.
+
+ @return: An db object.
+ """
db_type = _get_db_type()
db_module = __import__("autotest_lib.tko." + db_type, globals(),
locals(), [db_type])
diff --git a/tko/job_serializer.py b/tko/job_serializer.py
index 3999e9d..92bc923 100755
--- a/tko/job_serializer.py
+++ b/tko/job_serializer.py
@@ -10,11 +10,8 @@
"""
# import python libraries
-import os
import datetime
import time
-import random
-import re
# import autotest libraries
from autotest_lib.tko import models
@@ -45,7 +42,11 @@
'machine_owner':str,
'machine_group':str, 'aborted_by':str,
'aborted_on':datetime,
- 'keyval_dict':dict}
+ 'keyval_dict':dict,
+ 'afe_parent_job_id':str,
+ 'build_version':str,
+ 'suite':str,
+ 'board':str}
self.test_type_dict = {'subdir':str, 'testname':str,
'status':str, 'reason':str,
@@ -68,10 +69,10 @@
job object and then converts the job object into a tko job
object.
- @param
- infile: the name of the binary file that will be deserialized.
- @return a tko job that is represented by the binary file will
+ @param infile: the name of the binary file that will be deserialized.
+
+ @return: a tko job that is represented by the binary file will
be returned.
"""
@@ -99,12 +100,13 @@
is already in the job object. Any fields that is None will be
provided a default value.
- @param
- the_job: the tko job object that will be serialized.
+ @param the_job: the tko job object that will be serialized.
tag: contains the job name and the afe_job_id
binaryfilename: the name of the file that will be written to
+ @param tag: The job tag string.
+ @param binaryfilename: The output filename.
- @return the filename of the file that contains the
+ @return: the filename of the file that contains the
binary of the serialized object.
"""
@@ -184,6 +186,8 @@
self.set_trivial_attr(tko_job, pb_job, self.job_type_dict)
self.set_afe_job_id_and_tag(pb_job, tag)
+ if hasattr(tko_job, 'index'):
+ pb_job.job_idx = tko_job.index
for test in tko_job.tests:
newtest = pb_job.tests.add()
@@ -257,6 +261,8 @@
self.set_trivial_attr(tko_test, pb_test, self.test_type_dict)
self.set_pb_kernel(tko_test.kernel, pb_test.kernel)
+ if hasattr(tko_test, 'test_idx'):
+ pb_test.test_idx = tko_test.test_idx
for current_iteration in tko_test.iterations:
pb_iteration = pb_test.iterations.add()
diff --git a/tko/job_serializer_unittest.py b/tko/job_serializer_unittest.py
index f57f91f..4387413 100755
--- a/tko/job_serializer_unittest.py
+++ b/tko/job_serializer_unittest.py
@@ -7,8 +7,6 @@
"""
import datetime
-import os
-import re
import tempfile
import time
import unittest
@@ -36,6 +34,11 @@
tko_job = models.job('/tmp/', 'autotest', 'test', 'My Computer',
tko_time, tko_time, tko_time, 'root',
'www', 'No one', tko_time, {'1+1':2})
+ tko_job.afe_parent_job_id = '111'
+ tko_job.build_version = 'R1-1.0.0'
+ tko_job.suite = 'bvt'
+ tko_job.board = 'alex'
+ tko_job.index = 2
tko_iteration = models.iteration(0, {'2+2':4, '3+3':6},
{'4+4':8, '5+5':10, '6+6':12})
@@ -49,7 +52,7 @@
tko_time, [tko_iteration,
tko_iteration, tko_iteration],
{'abc':'def'}, [], tko_labels)
-
+ tko_test.test_idx = 3
self.tko_job = tko_job
self.tko_job.tests = [tko_test, tko_test, tko_test]
@@ -62,10 +65,12 @@
def test_tag(self):
+ """Test serializing tag field."""
self.assertEqual(self.tag, self.pb_job.tag)
def test_afe_job_id(self):
+ """Test serializing afe_job_id field."""
self.assertEqual(self.expected_afe_job_id,
self.pb_job.afe_job_id)
@@ -137,6 +142,7 @@
def test_aborted_on(self):
+ """Test serializing aborted_on field."""
self.check_time(self.tko_job.aborted_on,
self.pb_job.aborted_on)
@@ -151,6 +157,36 @@
'keyval_dict'))
+ def test_job_idx(self):
+ """Test serializing job_idx field."""
+ self.assertEqual(self.tko_job.index,
+ self.pb_job.job_idx)
+
+
+ def test_afe_parent_job_id(self):
+ """Test serializing afe_parent_job_id field."""
+ self.assertEqual(self.tko_job.afe_parent_job_id,
+ self.pb_job.afe_parent_job_id)
+
+
+ def test_build_version(self):
+ """Test serializing build_version field."""
+ self.assertEqual(self.tko_job.build_version,
+ self.pb_job.build_version)
+
+
+ def test_suite(self):
+ """Test serializing suite field."""
+ self.assertEqual(self.tko_job.suite,
+ self.pb_job.suite)
+
+
+ def test_board(self):
+ """Test serializing board field."""
+ self.assertEqual(self.tko_job.board,
+ self.pb_job.board)
+
+
def test_tests(self):
"""Check if all the test are the same.
"""
@@ -164,6 +200,7 @@
self.assertEqual(test.reason, newtest.reason)
self.assertEqual(test.machine, newtest.machine)
self.assertEqual(test.labels, newtest.labels)
+ self.assertEqual(test.test_idx, newtest.test_idx)
self.check_time(test.started_time, newtest.started_time)
self.check_time(test.finished_time, newtest.finished_time)
@@ -180,6 +217,9 @@
def check_time(self, dTime, stime):
"""Check if the datetime object contains the same time value
in microseconds.
+
+ @param dTime: The datetime.
+ @param stime: The original time.
"""
t = mktime(dTime.timetuple()) + 1e-6 * dTime.microsecond
self.assertEqual(long(t), stime/1000)
@@ -187,6 +227,9 @@
def check_iteration(self, tko_iterations, pb_iterations):
"""Check if the iteration objects are the same.
+
+ @param tko_iterations: The list of iterations.
+ @param pb_iterations: The proto iterations.
"""
for tko_iteration, pb_iteration in zip(tko_iterations,
pb_iterations):
@@ -205,6 +248,9 @@
def convert_keyval_to_dict(self, var, attr):
"""Convert a protocol buffer repeated keyval object into a
python dict.
+
+ @param var: The variable name.
+ @param attr: The attribute name.
"""
return dict((keyval.name, keyval.value) for keyval in
@@ -214,6 +260,9 @@
def check_dict(self, dictionary, keyval):
"""Check if the contents of the dictionary are the same as a
repeated keyval pair.
+
+ @param dictionary: The dict object.
+ @param keyval: The keyval object.
"""
for key, value in dictionary.iteritems():
self.assertTrue(key in keyval);
@@ -222,6 +271,9 @@
def check_kernel(self, kernel, newkernel):
"""Check if the kernels are the same.
+
+ @param kernel: The kernel object.
+ @param newkernel: The proto kernel object.
"""
self.assertEqual(kernel.base, newkernel.base)
self.assertEqual(kernel.kernel_hash, newkernel.kernel_hash)
@@ -233,6 +285,7 @@
"""
def setUp(self):
+ """Setup the test."""
super(ReadBackTest, self).setUp()
out_binary = NamedTemporaryFile(mode='wb')
diff --git a/tko/models.py b/tko/models.py
index 1e2446c..b92a878 100644
--- a/tko/models.py
+++ b/tko/models.py
@@ -24,6 +24,10 @@
self.aborted_by = aborted_by
self.aborted_on = aborted_on
self.keyval_dict = keyval_dict
+ self.afe_parent_job_id = None
+ self.build_version = None
+ self.suite = None
+ self.board = None
@staticmethod
diff --git a/tko/tko.proto b/tko/tko.proto
index 8d8939f..4a769b9 100644
--- a/tko/tko.proto
+++ b/tko/tko.proto
@@ -30,6 +30,8 @@
repeated Iteration iterations = 9;
repeated KeyVal attributes = 10;
repeated string labels = 11;
+ // Could be none. Don't depend it as the key for a test.
+ optional int64 test_idx = 12;
}
required string dir = 1;
@@ -47,4 +49,10 @@
required int64 aborted_on = 13;
required string afe_job_id = 14;
repeated KeyVal keyval_dict = 15;
+ optional string afe_parent_job_id = 16; // If none, assuming parent job.
+ // Could be none, use a generated id as foreign key to tests.
+ optional int64 job_idx = 17;
+ optional string build_version = 18;
+ optional string suite = 19;
+ optional string board = 20;
}