autotest: Encapsulate dark_resume_utils, add DR counter

Currently, tests have no way of telling the difference between
full resumes and dark resumes, and dark resumes may be short enough
that wait_up isn't going to catch them. This adds a listener on the
DUT which counts DarkSuspendImminent DBus signals, which are sent
as soon as powerd notices we are in dark resume, and adds a function
which allows us to wait for some period of time and then ask the
listener how many signals it noticed.

dark_resume_utils now takes the form of a class instead of a bunch
of functions you call during intialization and teardown because there
is now state which needs to persist across those parts of the test.
It also exposes a better way to suspend for dark resume tests because
we can't rely on the RTC timer waking us up (since that will cause a
dark resume and resuspend), and makes sure that this doesn't put us
to sleep forever.

BUG=chromium:466731
TEST=convert WakeOnDisconnect/WoWLAN to use this, make sure
  tests pass

Change-Id: Icb0d7b555685dad7673cbc81840e40eccd282f38
Reviewed-on: https://chromium-review.googlesource.com/260038
Trybot-Ready: Samuel Tan <samueltan@chromium.org>
Tested-by: Samuel Tan <samueltan@chromium.org>
Reviewed-by: Christopher Wiley <wiley@chromium.org>
Reviewed-by: Samuel Tan <samueltan@chromium.org>
Commit-Queue: Samuel Tan <samueltan@chromium.org>
diff --git a/client/common_lib/cros/dark_resume_xmlrpc_server.py b/client/common_lib/cros/dark_resume_xmlrpc_server.py
new file mode 100755
index 0000000..35678b3
--- /dev/null
+++ b/client/common_lib/cros/dark_resume_xmlrpc_server.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import logging.handlers
+
+import common
+from autotest_lib.client.common_lib.cros import xmlrpc_server
+from autotest_lib.client.cros import dark_resume_listener
+from autotest_lib.client.cros import sys_power
+
+
+SERVER_PORT = 9993
+SERVER_COMMAND = ('cd /usr/local/autotest/common_lib/cros; '
+           './dark_resume_xmlrpc_server.py')
+CLEANUP_PATTERN = 'dark_resume_xmlrpc_server'
+READY_METHOD = 'ready'
+
+
+class DarkResumeXmlRpcDelegate(xmlrpc_server.XmlRpcDelegate):
+    """Exposes methods called remotely during dark resume autotests.
+
+    All instance methods of this object without a preceding '_' are exposed via
+    an XMLRPC server.  This is not a stateless handler object, which means that
+    if you store state inside the delegate, that state will remain around for
+    future calls.
+
+    """
+
+    def __init__(self):
+        super(DarkResumeXmlRpcDelegate, self).__init__()
+        self._listener = dark_resume_listener.DarkResumeListener()
+
+
+    @xmlrpc_server.dbus_safe(None)
+    def suspend_bg_for_dark_resume(self):
+        """Suspends this system indefinitely for dark resume."""
+        sys_power.suspend_bg_for_dark_resume()
+
+
+    @xmlrpc_server.dbus_safe(0)
+    def get_dark_resume_count(self):
+        """Gets the number of dark resumes that have occurred since
+        this listener was created."""
+        return self._listener.count
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    handler = logging.handlers.SysLogHandler(address = '/dev/log')
+    formatter = logging.Formatter(
+            'dark_resume_xmlrpc_server: [%(levelname)s] %(message)s')
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.debug('dark_resume_xmlrpc_server main...')
+    server = xmlrpc_server.XmlRpcServer(
+            'localhost', SERVER_PORT)
+    server.register_delegate(DarkResumeXmlRpcDelegate())
+    server.run()
diff --git a/client/cros/dark_resume_listener.py b/client/cros/dark_resume_listener.py
new file mode 100644
index 0000000..fc585e8
--- /dev/null
+++ b/client/cros/dark_resume_listener.py
@@ -0,0 +1,55 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import threading
+
+import dbus
+import dbus.mainloop.glib
+import gobject
+
+
+class DarkResumeListener(object):
+    """Server which listens for dark resume-related DBus signals to count how
+    many dark resumes we have seen since instantiation."""
+
+    SIGNAL_NAME = 'DarkSuspendImminent'
+
+
+    def __init__(self):
+        dbus.mainloop.glib.threads_init()
+        gobject.threads_init()
+
+        dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+        self._bus = dbus.SystemBus()
+        self._count = 0
+
+        self._bus.add_signal_receiver(handler_function=self._saw_dark_resume,
+                                      signal_name=self.SIGNAL_NAME)
+
+        def loop_runner():
+            """Handles DBus events on the system bus using the mainloop."""
+            # If we just call run on this loop, the listener will hang and the test
+            # will never finish. Instead, we process events as they come in. This
+            # thread is set to daemon below, which means that the program will exit
+            # when the main thread exits.
+            loop = gobject.MainLoop()
+            context = loop.get_context()
+            while True:
+                context.iteration(True)
+        thread = threading.Thread(None, loop_runner)
+        thread.daemon = True
+        thread.start()
+        logging.debug('Dark resume listener started')
+
+
+    @property
+    def count(self):
+        """Number of DarkSuspendImminent events this listener has seen since its
+        creation."""
+        return self._count
+
+
+    def _saw_dark_resume(self, unused):
+        self._count += 1
diff --git a/client/cros/sys_power.py b/client/cros/sys_power.py
index 188afc0..0f5a490 100644
--- a/client/cros/sys_power.py
+++ b/client/cros/sys_power.py
@@ -4,7 +4,13 @@
 
 """Provides utility methods for controlling powerd in ChromiumOS."""
 
-import errno, logging, os, rtc, time, upstart
+import errno
+import logging
+import multiprocessing
+import os
+import rtc
+import time
+import upstart
 
 SYSFS_POWER_STATE = '/sys/power/state'
 SYSFS_WAKEUP_COUNT = '/sys/power/wakeup_count'
@@ -114,6 +120,26 @@
     return alarm
 
 
+def suspend_bg_for_dark_resume(delay_seconds=0):
+    """Do a non-blocking indefinite suspend using power manager. ONLY USE THIS
+    IF YOU ARE ABSOLUTELY CERTAIN YOU NEED TO.
+
+    Wait for |delay_seconds|, then suspend to RAM (S3). This does not set an RTC
+    alarm and does not pass an external wakeup count. It is meant to be used for
+    dark resume testing, where the server-side API exposes it in such a fashion
+    that the DUT will be woken by the server no matter how the test is exited.
+
+    @param delay_seconds: Number of seconds wait before suspending the DUT.
+
+    """
+    upstart.ensure_running(['powerd'])
+    command = ('/usr/bin/powerd_dbus_suspend --delay=%d '
+               '--timeout=30') % delay_seconds
+    logging.info("Running '%s'", command)
+    process = multiprocessing.Process(target=os.system, args=(command,))
+    process.start()
+
+
 def kernel_suspend(seconds):
     """Do a kernel suspend.
 
diff --git a/server/cros/dark_resume_utils.py b/server/cros/dark_resume_utils.py
index 8e030c2..634db1b 100644
--- a/server/cros/dark_resume_utils.py
+++ b/server/cros/dark_resume_utils.py
@@ -3,57 +3,185 @@
 # found in the LICENSE file.
 
 import logging
+import time
+
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib.cros import dark_resume_xmlrpc_server
+from autotest_lib.server import autotest
 
 POWER_DIR = '/var/lib/power_manager'
 TMP_POWER_DIR = '/tmp/power_manager'
 POWER_DEFAULTS = '/usr/share/power_manager/board_specific'
 
+RESUME_CTRL_RETRIES = 3
+RESUME_GRACE_PERIOD = 10
+XMLRPC_BRINGUP_TIMEOUT_SECONDS = 60
 
-def dark_resume_setup(host):
-    """Set up powerd preferences so we will properly go into dark resume,
-    and still be able to communicate with the DUT.
 
-    @param host: the DUT to set up dark resume for
+class DarkResumeSuspend(object):
+    """Context manager which exposes the dark resume-specific suspend
+    functionality.
 
+    This is required because using the RTC for a dark resume test will
+    cause the system to wake up in dark resume and resuspend, which is
+    not what we want. Instead, we suspend indefinitely, but make sure we
+    don't leave the DUT asleep by always running code to wake it up via
+    servo.
     """
-    logging.info('Setting up dark resume preferences')
-
-    # Make temporary directory, which will be used to hold
-    # temporary preferences. We want to avoid writing into
-    # /var/lib so we don't have to save any state.
-    logging.debug('Creating temporary powerd prefs at %s', TMP_POWER_DIR)
-    host.run('mkdir -p %s' % TMP_POWER_DIR)
-
-    logging.debug('Enabling dark resume')
-    host.run('echo 0 > %s/disable_dark_resume' % TMP_POWER_DIR)
-
-    logging.debug('Enabling USB ports in dark resume')
-
-    dev_contents = host.run('cat %s/dark_resume_devices' % POWER_DEFAULTS,
-                            ignore_status=True).stdout
-    dev_list = dev_contents.split('\n')
-    new_dev_list = filter(lambda dev: dev.find('usb') == -1, dev_list)
-    new_dev_contents = '\n'.join(new_dev_list)
-    host.run('echo -e \'%s\' > %s/dark_resume_devices' %
-             (new_dev_contents, TMP_POWER_DIR))
-
-    # bind the tmp directory to the power preference directory
-    host.run('mount --bind %s %s' % (TMP_POWER_DIR, POWER_DIR))
-
-    logging.debug('Restarting powerd with new settings')
-    host.run('restart powerd')
 
 
-def dark_resume_teardown(host):
-    """Clean up changes made by dark_resume_setup.
+    def __init__(self, proxy, host):
+        """Set up for a dark-resume-ready suspend to be carried out using
+        |proxy| and for the subsequent wakeup to be carried out using
+        |servo|.
 
-    @param host: the DUT to remove dark resume prefs for
+        @param proxy: a dark resume xmlrpc server proxy object for the DUT
+        @param servo: a servo host connected to the DUT
 
+        """
+        self._client_proxy = proxy
+        self._host = host
+
+
+    def __enter__(self):
+        """Suspend the DUT."""
+        logging.info('Suspending DUT (in background)...')
+        self._client_proxy.suspend_bg_for_dark_resume()
+
+
+    def __exit__(self, exception, value, traceback):
+        """Wake up the DUT."""
+        logging.info('Waking DUT from server.')
+        _wake_dut(self._host)
+
+
+class DarkResumeUtils(object):
+    """Class containing common functionality for tests which exercise dark
+    resume pathways. We set up powerd to allow dark resume and also configure
+    the suspended devices so that the backchannel can stay up. We can also
+    check for the number of dark resumes that have happened in a particular
+    suspend request.
     """
-    logging.info('Tearing down dark resume preferences')
 
-    logging.debug('Cleaning up temporary powerd bind mounts')
-    host.run('umount %s' % POWER_DIR, ignore_status=True)
 
-    logging.debug('Restarting powerd to revert to old settings')
-    host.run('restart powerd')
+    def __init__(self, host):
+        """Set up powerd preferences so we will properly go into dark resume,
+        and still be able to communicate with the DUT.
+
+        @param host: the DUT to set up dark resume for
+
+        """
+        self._host = host
+        logging.info('Setting up dark resume preferences')
+
+        # Make temporary directory, which will be used to hold
+        # temporary preferences. We want to avoid writing into
+        # /var/lib so we don't have to save any state.
+        logging.debug('Creating temporary powerd prefs at %s', TMP_POWER_DIR)
+        host.run('mkdir -p %s' % TMP_POWER_DIR)
+
+        logging.debug('Enabling dark resume')
+        host.run('echo 0 > %s/disable_dark_resume' % TMP_POWER_DIR)
+
+        logging.debug('Enabling USB ports in dark resume')
+
+        dev_contents = host.run('cat %s/dark_resume_devices' % POWER_DEFAULTS,
+                                ignore_status=True).stdout
+        dev_list = dev_contents.split('\n')
+        new_dev_list = filter(lambda dev: dev.find('usb') == -1, dev_list)
+        new_dev_contents = '\n'.join(new_dev_list)
+        host.run('echo -e \'%s\' > %s/dark_resume_devices' %
+                 (new_dev_contents, TMP_POWER_DIR))
+
+        # bind the tmp directory to the power preference directory
+        host.run('mount --bind %s %s' % (TMP_POWER_DIR, POWER_DIR))
+
+        logging.debug('Restarting powerd with new settings')
+        host.run('restart powerd')
+
+        logging.debug('Starting XMLRPC session to watch for dark resumes')
+        self._client_proxy = self._get_xmlrpc_proxy()
+
+
+    def teardown(self):
+        """Clean up changes made by DarkResumeUtils."""
+
+        logging.info('Tearing down dark resume preferences')
+
+        logging.debug('Cleaning up temporary powerd bind mounts')
+        self._host.run('umount %s' % POWER_DIR, ignore_status=True)
+
+        logging.debug('Restarting powerd to revert to old settings')
+        self._host.run('restart powerd')
+
+
+    def suspend(self):
+        """Returns a DarkResumeSuspend context manager that allows safe suspending
+        of the DUT."""
+        return DarkResumeSuspend(self._client_proxy, self._host)
+
+
+    def count_dark_resumes(self):
+        """Return the number of dark resumes that have occurred since the beginning
+        of the test. This will wake up the DUT, so make sure to put it back to
+        sleep if you need to keep it suspended for some reason.
+
+        This method will raise an error if the DUT does not wake up.
+
+        @return the number of dark resumes counted by this DarkResumeUtils
+
+        """
+        _wake_dut(self._host)
+
+        return self._client_proxy.get_dark_resume_count()
+
+
+    def _get_xmlrpc_proxy(self):
+        """Get a dark resume XMLRPC proxy for the host this DarkResumeUtils is
+        attached to.
+
+        The returned object has no particular type.  Instead, when you call
+        a method on the object, it marshalls the objects passed as arguments
+        and uses them to make RPCs on the remote server.  Thus, you should
+        read dark_resume_xmlrpc_server.py to find out what methods are supported.
+
+        @return proxy object for remote XMLRPC server.
+
+        """
+        # Make sure the client library is on the device so that the proxy
+        # code is there when we try to call it.
+        client_at = autotest.Autotest(self._host)
+        client_at.install()
+        # Start up the XMLRPC proxy on the client
+        proxy = self._host.xmlrpc_connect(
+                dark_resume_xmlrpc_server.SERVER_COMMAND,
+                dark_resume_xmlrpc_server.SERVER_PORT,
+                command_name=dark_resume_xmlrpc_server.CLEANUP_PATTERN,
+                ready_test_name=dark_resume_xmlrpc_server.READY_METHOD,
+                timeout_seconds=XMLRPC_BRINGUP_TIMEOUT_SECONDS)
+        return proxy
+
+
+def _wake_dut(host):
+    """Make sure |host| is up. If we can't wake it with normal keys, hit the
+    power button."""
+
+    woken = False
+    for i in range(RESUME_CTRL_RETRIES):
+        # Double tap servo key to make sure we signal user activity to Chrome.
+        # The first key might occur during the kernel suspend pathway, which
+        # causes the suspend to abort, but might put us in dark resume since
+        # the keypress is not forwarded to Chrome.
+        host.servo.ctrl_key()
+        time.sleep(0.5)
+        host.servo.ctrl_key()
+
+        if host.wait_up(timeout=RESUME_GRACE_PERIOD):
+            woken = True
+            break
+        logging.debug('Wake attempt #%d failed', i+1)
+
+    if not woken:
+        logging.warning('DUT did not wake -- trouble ahead')
+        host.servo.power_key()
+        raise error.TestFail('DUT did not wake')
diff --git a/server/site_tests/network_WiFi_WakeOnDisconnect/network_WiFi_WakeOnDisconnect.py b/server/site_tests/network_WiFi_WakeOnDisconnect/network_WiFi_WakeOnDisconnect.py
index bdd285d..6ccbd0d 100644
--- a/server/site_tests/network_WiFi_WakeOnDisconnect/network_WiFi_WakeOnDisconnect.py
+++ b/server/site_tests/network_WiFi_WakeOnDisconnect/network_WiFi_WakeOnDisconnect.py
@@ -13,7 +13,7 @@
 from autotest_lib.server.cros.network import wifi_client
 
 SUSPEND_WAIT_TIME=10
-RESUME_WAIT_TIME=10
+RESUME_WAIT_TIME=25
 
 
 class network_WiFi_WakeOnDisconnect(wifi_cell_test_base.WiFiCellTestBase):
@@ -23,7 +23,7 @@
 
     def initialize(self, host):
         """Set up for dark resume."""
-        dark_resume_utils.dark_resume_setup(host)
+        self._dr_utils = dark_resume_utils.DarkResumeUtils(host)
 
 
     def run_once(self):
@@ -40,24 +40,22 @@
         with client.wake_on_wifi_features(wifi_client.WAKE_ON_WIFI_SSID):
             logging.info('Set up WoWLAN')
 
-            client.do_suspend_bg(SUSPEND_WAIT_TIME + RESUME_WAIT_TIME + 10)
-            time.sleep(SUSPEND_WAIT_TIME)
+            with self._dr_utils.suspend():
+                time.sleep(SUSPEND_WAIT_TIME)
 
-            # kick over the router
-            router.deconfig_aps(silent=True)
+                # kick over the router
+                router.deconfig_aps(silent=True)
 
-            # The DUT should wake up soon, but we'll give it a bit of a
-            # grace period.
-            if not client.host.wait_up(timeout=RESUME_WAIT_TIME):
-                raise error.TestFail('Client failed to wake up.')
+                # The DUT should wake up soon, but we'll give it a bit of a
+                # grace period.
+                time.sleep(RESUME_WAIT_TIME)
+                if self._dr_utils.count_dark_resumes() < 1:
+                    raise error.TestFail('Client failed to wake up.')
 
-            logging.info('Client woke up successfully.')
+                logging.info('Client woke up successfully.')
 
 
     def cleanup(self):
-        # make sure the DUT is up on the way out
-        self.context.client.host.servo.ctrl_key()
-
-        dark_resume_utils.dark_resume_teardown(self.context.client.host)
+        self._dr_utils.teardown()
         # make sure we clean up everything
         super(network_WiFi_WakeOnDisconnect, self).cleanup()
diff --git a/server/site_tests/network_WiFi_WoWLAN/network_WiFi_WoWLAN.py b/server/site_tests/network_WiFi_WoWLAN/network_WiFi_WoWLAN.py
index b20ca7c..2bcb72f 100644
--- a/server/site_tests/network_WiFi_WoWLAN/network_WiFi_WoWLAN.py
+++ b/server/site_tests/network_WiFi_WoWLAN/network_WiFi_WoWLAN.py
@@ -13,7 +13,7 @@
 from autotest_lib.server.cros.network import wifi_client
 
 SUSPEND_WAIT_TIME=10
-RESUME_WAIT_TIME=10
+RESUME_WAIT_TIME=25
 
 
 class network_WiFi_WoWLAN(wifi_cell_test_base.WiFiCellTestBase):
@@ -23,7 +23,7 @@
 
     def initialize(self, host):
         """Set up for dark resume."""
-        dark_resume_utils.dark_resume_setup(host)
+        self._dr_utils = dark_resume_utils.DarkResumeUtils(host)
 
 
     def run_once(self):
@@ -46,25 +46,23 @@
             client.add_wake_packet_source(router.wifi_ip)
             logging.info('Set up WoWLAN')
 
-            client.do_suspend_bg(SUSPEND_WAIT_TIME + RESUME_WAIT_TIME + 10)
-            time.sleep(SUSPEND_WAIT_TIME)
+            with self._dr_utils.suspend():
+                time.sleep(SUSPEND_WAIT_TIME)
 
-            router.send_magic_packet(dut_ip, dut_mac)
+                router.send_magic_packet(dut_ip, dut_mac)
 
-            # The DUT should wake up soon, but we'll give it a bit of a
-            # grace period.
-            if not client.host.wait_up(timeout=RESUME_WAIT_TIME):
-                raise error.TestFail('Client failed to wake up.')
+                # The DUT should wake up soon, but we'll give it a bit of a
+                # grace period.
+                time.sleep(RESUME_WAIT_TIME)
+                if self._dr_utils.count_dark_resumes() < 1:
+                    raise error.TestFail('Client failed to wake up.')
 
-            logging.info('Client woke up successfully.')
+                logging.info('Client woke up successfully.')
 
 
     def cleanup(self):
-        # make sure the DUT is up on the way out
-        self.context.client.host.servo.ctrl_key()
+        self._dr_utils.teardown()
         # clean up packet wake sources
         self.context.client.remove_all_wake_packet_sources()
-
-        dark_resume_utils.dark_resume_teardown(self.context.client.host)
         # make sure we clean up everything
         super(network_WiFi_WoWLAN, self).cleanup()