[autotest] Update HQE state in job_reporter
BUG=chromium:748234
TEST=Unittests
Change-Id: I4ea8d4985f4deeb0e7d5c7a78e145efbb376e2ea
Reviewed-on: https://chromium-review.googlesource.com/666140
Commit-Ready: Allen Li <ayatane@chromium.org>
Tested-by: Allen Li <ayatane@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>
diff --git a/venv/lucifer/autotest.py b/venv/lucifer/autotest.py
new file mode 100644
index 0000000..7d89731
--- /dev/null
+++ b/venv/lucifer/autotest.py
@@ -0,0 +1,99 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Kludges to support legacy Autotest code.
+
+Autotest imports should be done by calling monkeypatch() first and then
+calling load(). monkeypatch() should only be called once from a
+script's main function.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import imp
+import importlib
+import logging
+import os
+import site
+import sys
+
+import autotest_lib
+
+_AUTOTEST_DIR = autotest_lib.__path__[0]
+_SITEPKG_DIR = os.path.join(_AUTOTEST_DIR, 'site-packages')
+
+_setup_done = False
+
+logger = logging.getLogger(__name__)
+
+
+def monkeypatch():
+ """Do necessary Autotest monkeypatching.
+
+ This should be called before any autotest_lib imports in the main
+ function in scripts. Thus, only the main function in scripts can
+ import autotest_lib.
+
+ Library code should rely on dependency injection, falling back to
+ load().
+
+ This should be called no more than once.
+
+ This adds Autotest's site-packages and modifies sys.meta_path so
+ that all common.py imports are no-ops.
+ """
+ global _setup_done
+ assert not _setup_done
+ site.addsitedir(_SITEPKG_DIR)
+ sys.meta_path.insert(0, _CommonRemovingFinder())
+ _setup_done = True
+
+
+class _CommonRemovingFinder(object):
+ """Python import finder that neuters Autotest's common.py
+
+ The common module is replaced with an empty module everywhere it is
+ imported. common.py should have only been imported for side
+ effects, so nothing should actually use the imported module.
+
+ See also https://www.python.org/dev/peps/pep-0302/
+ """
+
+ def find_module(self, fullname, path=None):
+ """Find module."""
+ del path # unused
+ if not self._is_autotest_common(fullname):
+ return None
+ logger.debug('Dummying out %s import', fullname)
+ return self
+
+ def _is_autotest_common(self, fullname):
+ return (fullname.partition('.')[0] == 'autotest_lib'
+ and fullname.rpartition('.')[-1] == 'common')
+
+ def load_module(self, fullname):
+ """Load module."""
+ if fullname in sys.modules:
+ return sys.modules[fullname]
+ mod = imp.new_module(fullname)
+ mod.__file__ = '<removed>'
+ mod.__loader__ = self
+ mod.__package__ = fullname.rpartition('.')[0]
+ sys.modules[fullname] = mod
+ return mod
+
+
+def load(name):
+ """Import module from autotest.
+
+ This enforces that monkeypatch() is called first. Otherwise,
+ autotest imports may or may not work. When they do work, they may
+ screw up global state.
+ """
+ if not _setup_done:
+ raise ImportError('cannot load Autotest modules before monkeypatching')
+ relpath = name.lstrip('.')
+ return importlib.import_module('.%s' % relpath, package='autotest_lib')
diff --git a/venv/lucifer/autotest_unittest.py b/venv/lucifer/autotest_unittest.py
new file mode 100644
index 0000000..4571e44
--- /dev/null
+++ b/venv/lucifer/autotest_unittest.py
@@ -0,0 +1,56 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for autotest.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+import mock
+import pytest
+import subprocess32
+
+from lucifer import autotest
+
+
+def test_monkeypatch():
+ """Test monkeypatch()."""
+ common_file = subprocess32.check_output(
+ [sys.executable, '-m', 'lucifer.scripts.autotest_monkeypatcher'])
+ assert common_file.rstrip() == '<removed>'
+
+
+@pytest.mark.parametrize('fullname,expected', [
+ ('autotest_lib.common', True),
+ ('autotest_lib.server.common', True),
+ ('autotest_lib.server', False),
+ ('some_lib.common', False),
+])
+def test__CommonRemovingFinder__is_autotest_common(fullname, expected):
+ """Test _CommonRemovingFinder._is_common()."""
+ finder = autotest._CommonRemovingFinder()
+ assert finder._is_autotest_common(fullname) == expected
+
+
+@pytest.mark.parametrize('name,expected', [
+ ('scheduler.models', '.scheduler.models'),
+ ('...scheduler.models', '.scheduler.models'),
+])
+def test_load(name, expected):
+ """Test load()."""
+ with mock.patch('importlib.import_module', autospec=True) \
+ as import_module, \
+ mock.patch.object(autotest, '_setup_done', True):
+ autotest.load(name)
+ import_module.assert_called_once_with(expected, package='autotest_lib')
+
+
+def test_load_without_patch_fails():
+ """Test load() without patch."""
+ with mock.patch.object(autotest, '_setup_done', False):
+ with pytest.raises(ImportError):
+ autotest.load('asdf')
diff --git a/venv/lucifer/eventlib.py b/venv/lucifer/eventlib.py
index b719c79..d2934e3 100644
--- a/venv/lucifer/eventlib.py
+++ b/venv/lucifer/eventlib.py
@@ -2,13 +2,18 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""Status change events module.
+"""Event subprocess module.
-This is used to standardize communication of events between processes
-through a pipe, generally through stdout.
+Event subprocesses are subprocesses that print event changes to stdout
+and reads command from stdin.
-run_event_command() starts a process that sends such events to stdout
-and handles them through a callback.
+Each event and command is a UNIX line, with a terminating newline
+character.
+
+Only the abort command is supported. The main process aborts the event
+subprocess when SIGUSR1 is received.
+
+run_event_command() starts such a process.
"""
from __future__ import absolute_import
@@ -42,6 +47,7 @@
job_shepherd, which lives in the infra/lucifer repository.
"""
STARTING = 'starting'
+ PARSING = 'parsing'
COMPLETED = 'completed'
@@ -65,9 +71,12 @@
def run_event_command(event_handler, args):
"""Run a command that emits events.
- Events printed by the command will be handled by event_handler.
- While the process for the command is running, trapped signals will
- be passed on to it so it can abort gracefully.
+ Events printed by the command will be handled by event_handler. All
+ exceptions raised by event_handler will be caught and logged;
+ however, event_handler should not let any exceptions escape.
+
+ While the event command is running, SIGUSR1 is interpreted as an
+ abort command and sent to the subprocess via stdin.
@param event_handler: callable that takes an Event instance.
@param args: passed to subprocess.Popen.
@@ -106,10 +115,9 @@
while True:
logger.debug('Reading subprocess stdout')
line = proc.stdout.readline()
- if line:
- _handle_output_line(event_handler, line)
- else:
+ if not line:
break
+ _handle_output_line(event_handler, line)
def _handle_output_line(event_handler, line):
@@ -122,5 +130,5 @@
event = Event(line.rstrip())
except ValueError:
logger.warning('Invalid output %r received', line)
- else:
- event_handler(event)
+ return
+ event_handler(event)
diff --git a/venv/lucifer/scripts/autotest_monkeypatcher.py b/venv/lucifer/scripts/autotest_monkeypatcher.py
new file mode 100644
index 0000000..0becef9
--- /dev/null
+++ b/venv/lucifer/scripts/autotest_monkeypatcher.py
@@ -0,0 +1,36 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Monkeypatch autotest
+
+This is used for testing Autotest monkeypatching.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+import lucifer.autotest
+from lucifer import loglib
+
+
+def main(args):
+ """Main function
+
+ @param args: list of command line args
+ """
+ del args
+ loglib.configure_logging(name='autotest_monkeypatcher')
+
+ lucifer.autotest.monkeypatch()
+ from autotest_lib import common
+
+ print(common.__file__)
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/venv/lucifer/scripts/job_reporter.py b/venv/lucifer/scripts/job_reporter.py
index 997e10c..73b8568 100644
--- a/venv/lucifer/scripts/job_reporter.py
+++ b/venv/lucifer/scripts/job_reporter.py
@@ -2,6 +2,11 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""Run a job against Autotest.
+
+See http://goto.google.com/monitor_db_per_job_refactor
+"""
+
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
@@ -10,6 +15,7 @@
import logging
import sys
+from lucifer import autotest
from lucifer import eventlib
from lucifer import loglib
@@ -25,27 +31,106 @@
@param args: list of command line args
"""
- parser = argparse.ArgumentParser(prog='job_reporter')
+
+ parser = argparse.ArgumentParser(prog='job_reporter', description=__doc__)
loglib.add_logging_options(parser)
+ parser.add_argument('--job-id', type=int, default=None)
+ parser.add_argument('--autoserv-exit', type=int, default=None,
+ help='autoserv exit status')
args = parser.parse_args(args)
loglib.configure_logging_with_args(parser, args)
- return _run_shepherd(_handle_event)
+
+ autotest.monkeypatch()
+ autotest.load('frontend.setup_django_environment')
+ scheduler_models = autotest.load('scheduler.scheduler_models')
+
+ if args.job_id is not None:
+ if args.autoserv_exit is None:
+ # TODO(crbug.com/748234): autoserv not implemented yet.
+ raise NotImplementedError('not implemented yet (crbug.com/748234)')
+ job = scheduler_models.Job.objects.get(id=args.job_id)
+ hqes = list(scheduler_models.HostQueueEntry.objects
+ .filter(job_id=args.job_id))
+ else:
+ # TODO(crbug.com/748234): Full jobs not implemented yet.
+ raise NotImplementedError('not implemented yet')
+ handler = _EventHandler(job, hqes, autoserv_exit=args.autoserv_exit)
+ return _run_shepherd(handler, job_id=args.job_id)
-def _run_shepherd(event_handler):
+def _run_shepherd(event_handler, job_id):
"""Run job_shepherd.
Events issues by the job_shepherd will be handled by event_handler.
- @param event_handler: callable that takes an Event.
+ @param event_handler: callable that takes an Event
+ @param job_id: job ID
"""
- return eventlib.run_event_command(
- event_handler=event_handler,
- args=[_JOB_SHEPHERD_PROGRAM])
+ args = [_JOB_SHEPHERD_PROGRAM]
+ args.append('%d' % job_id)
+ return eventlib.run_event_command(event_handler=event_handler, args=args)
-def _handle_event(event):
- logger.debug('Received event %r', event.name)
+class _EventHandler(object):
+ """Event handling dispatcher.
+
+ Event handlers are implemented as methods named _handle_<event value>.
+
+ Each handler method must handle its exceptions accordingly. If an
+ exception escapes, the job dies on the spot.
+ """
+
+ def __init__(self, job, hqes, autoserv_exit):
+ """Initialize instance.
+
+ @param job: Job instance to own
+ @param hqes: list of HostQueueEntry instances for the job
+ @param autoserv_exit: autoserv exit status
+ """
+ self._job = job
+ self._hqes = hqes
+ # TODO(crbug.com/748234): autoserv not implemented yet.
+ self._autoserv_exit = autoserv_exit
+
+ def __call__(self, event):
+ logger.debug('Received event %r', event.name)
+ method_name = '_handle_%s' % event.value
+ try:
+ handler = getattr(self, method_name)
+ except AttributeError:
+ raise NotImplementedError('%s is not implemented for handling %s',
+ method_name, event.name)
+ handler(event)
+
+ def _handle_starting(self):
+ # TODO(crbug.com/748234): No event update needed yet.
+ pass
+
+ def _handle_parsing(self):
+ # TODO(crbug.com/748234): monitor_db leaves the HQEs in parsing
+ pass
+
+ def _handle_completed(self):
+ final_status = self._final_status()
+ for hqe in self._hqes:
+ hqe.set_status(final_status)
+
+ def _final_status(self):
+ afe_models = autotest.load('frontend.afe.models')
+ Status = afe_models.HostQueueEntry.Status
+ if self._job_was_aborted():
+ return Status.ABORTED
+
+ if self._autoserv_exit == 0:
+ return Status.COMPLETED
+ return Status.FAILED
+
+ def _job_was_aborted(self):
+ for hqe in self._hqes:
+ hqe.update_from_database()
+ if hqe.aborted:
+ return True
+ return False
if __name__ == '__main__':
diff --git a/venv/pytest.ini b/venv/pytest.ini
index c21e1a3..c36bbe4 100644
--- a/venv/pytest.ini
+++ b/venv/pytest.ini
@@ -5,3 +5,4 @@
--durations=5
--ignore setup.py
--cov lucifer
+ -r xX
diff --git a/venv/requirements.txt b/venv/requirements.txt
index dacf608..ef16180 100644
--- a/venv/requirements.txt
+++ b/venv/requirements.txt
@@ -1,8 +1,15 @@
contextlib2==0.5.4
coverage==4.4.1
+Django==1.5.1
enum34==1.1.6
+funcsigs==1.0.2
mock==2.0.0
-subprocess32==3.2.7
+pbr==1.10.0
+pip==8.1.2
psutil==4.3.1
-pytest-cov==2.5.1
+py==1.4.34
pytest==3.1.3
+pytest-cov==2.5.1
+setuptools==28.2.0
+six==1.10.0
+subprocess32==3.2.7