Autotest: Add chromeos2 lab support to RPM Infrastructure.
Added support for devices in chromeos2 lab to the RPM Controller
codebase. It will determine if the device is behind a hydra device and
if so complete the proper login/logout procedures required.
So it turns out that theres a known pycurl/libcurl issue that can occur
sometimes when used with multiple threads and cause the python thread to
crash. Therefore I replaced the geturl call to utilize urllib instead.
BUG=chromium-os:30955
TEST=Expanded the functional tests and ensured the unittests still work.
Stress tested web calls in a while loop to ensure that no threads crash.
Change-Id: I646abe2f09cf00304892bad217e7f1a9561e1297
Reviewed-on: https://gerrit.chromium.org/gerrit/29822
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Ready: Simran Basi <sbasi@chromium.org>
Tested-by: Simran Basi <sbasi@chromium.org>
diff --git a/site_utils/rpm_control_system/dli_urllib.py b/site_utils/rpm_control_system/dli_urllib.py
new file mode 100644
index 0000000..f14aeaa
--- /dev/null
+++ b/site_utils/rpm_control_system/dli_urllib.py
@@ -0,0 +1,23 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import urllib
+
+import dli
+
+
+class Powerswitch(dli.powerswitch):
+ """
+ This class will utilize urllib instead of pycurl to get the web page info.
+ """
+
+
+ def geturl(self,url='index.htm') :
+ self.contents=''
+ path = 'http://%s:%s@%s:80/%s' % (self.userid,self.password,
+ self.hostname,url)
+ web_file = urllib.urlopen(path)
+ if web_file.getcode() != 200:
+ return None
+ self.contents = web_file.read()
+ return self.contents
diff --git a/site_utils/rpm_control_system/rpm_config.ini b/site_utils/rpm_control_system/rpm_config.ini
index c03c910..24a8dc2 100644
--- a/site_utils/rpm_control_system/rpm_config.ini
+++ b/site_utils/rpm_control_system/rpm_config.ini
@@ -32,3 +32,10 @@
# Access information for Web Powered Devices.
username: admin
password: 1234
+
+[hydra1]
+hostname: chromeos-197-hydra1.mtv
+username: autotest
+password: s3rialsw
+admin_username: root
+admin_password: tslinux
\ No newline at end of file
diff --git a/site_utils/rpm_control_system/rpm_controller.py b/site_utils/rpm_control_system/rpm_controller.py
index 736708f..87a7410 100644
--- a/site_utils/rpm_control_system/rpm_controller.py
+++ b/site_utils/rpm_control_system/rpm_controller.py
@@ -6,10 +6,12 @@
import logging
import pexpect
import Queue
+import re
import threading
import time
-import dli
+from config import rpm_config
+import dli_urllib
# Format Appears as: [Date] [Time] - [Msg Level] - [Message]
@@ -30,11 +32,16 @@
It assumes that you know the RPM hostname and that the DUT is on
the specified RPM.
+ This class also allows support for RPM devices that can be accessed
+ directly or through a hydra serial concentrator device.
+
Implementation details:
This is an abstract class, subclasses must implement the methods
listed here. You must not instantiate this class but should
instantiate one of those leaf subclasses.
+ @var behind_hydra: boolean value to represent whether or not this RPM is
+ behind a hydra device.
@var hostname: hostname for this rpm device.
@var is_running_lock: lock used to control access to _running.
@var request_queue: queue used to store requested outlet state changes.
@@ -44,18 +51,50 @@
"""
- def __init__(self, rpm_hostname):
+ SSH_LOGIN_CMD = 'ssh -l %s -o StrictHostKeyChecking=no ' \
+ '-o UserKnownHostsFile=/dev/null %s'
+ USERNAME_PROMPT = 'Username:'
+ HYRDA_RETRY_SLEEP_SECS = 10
+ HYDRA_MAX_CONNECT_RETRIES = 3
+ LOGOUT_CMD = 'logout'
+ CLI_CMD = 'CLI'
+ CLI_HELD = 'The administrator \[root\] has an active .* session.'
+ CLI_KILL_PREVIOUS = 'cancel'
+ CLI_PROMPT = 'cli>'
+ HYDRA_PROMPT = '#'
+ PORT_STATUS_CMD = 'portStatus'
+ QUIT_CMD = 'quit'
+ SESSION_KILL_CMD_FORMAT = 'administration sessions kill %s'
+ HYDRA_CONN_HELD_MSG_FORMAT = 'is being used'
+
+ # Global Variables that will likely be changed by subclasses.
+ DEVICE_PROMPT = '$'
+ PASSWORD_PROMPT = 'Password:'
+ # The state change command can be any string format but must accept 2 vars:
+ # state followed by DUT/Plug name.
+ STATE_CMD = '%s %s'
+ SUCCESS_MSG = None # Some RPM's may not return a success msg.
+
+
+ def __init__(self, rpm_hostname, hydra_name=None):
"""
RPMController Constructor.
To be called by subclasses.
@param rpm_hostname: hostname of rpm device to be controlled.
"""
- dns_zone = CONFIG.get('CROS', 'dns_zone')
- self.hostname = '.'.join([rpm_hostname, dns_zone])
+ self._dns_zone = CONFIG.get('CROS', 'dns_zone')
+ self.hostname = rpm_hostname
self.request_queue = Queue.Queue()
self._running = False
self.is_running_lock = threading.Lock()
+ self.behind_hydra = False
+ # If a hydra name is provided by the subclass then we know we are
+ # talking to an rpm behind a hydra device.
+ if hydra_name:
+ self.behind_hydra = True
+ self.hydra_name = hydra_name
+ self._hydra_hostname = CONFIG.get(hydra_name,'hostname')
def _start_processing_requests(self):
@@ -137,10 +176,194 @@
return result
+ def _kill_previous_connection(self):
+ """
+ In case the port to the RPM through the hydra serial concentrator is in
+ use, terminate the previous connection so we can log into the RPM.
+
+ It logs into the hydra serial concentrator over ssh, launches the CLI
+ command, gets the port number and then kills the current session.
+ """
+ ssh = self._authenticate_with_hydra(admin_override=True)
+ if not ssh:
+ return
+ ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60)
+ ssh.sendline(CONFIG.get(self.hydra_name, 'admin_password'))
+ ssh.expect(RPMController.HYDRA_PROMPT)
+ ssh.sendline(RPMController.CLI_CMD)
+ cli_prompt_re = re.compile(RPMController.CLI_PROMPT)
+ cli_held_re = re.compile(RPMController.CLI_HELD)
+ response = ssh.expect_list([cli_prompt_re, cli_held_re], timeout=60)
+ if response == 1:
+ # Need to kill the previous adminstator's session.
+ logging.error("Need to disconnect previous administrator's CLI "
+ "session to release the connection to RPM device %s.",
+ self.hostname)
+ ssh.sendline(RPMController.CLI_KILL_PREVIOUS)
+ ssh.expect(RPMController.CLI_PROMPT)
+ ssh.sendline(RPMController.PORT_STATUS_CMD)
+ ssh.expect(': %s' % self.hostname)
+ ports_status = ssh.before
+ port_number = ports_status.split(' ')[-1]
+ ssh.expect(RPMController.CLI_PROMPT)
+ ssh.sendline(RPMController.SESSION_KILL_CMD_FORMAT % port_number)
+ ssh.expect(RPMController.CLI_PROMPT)
+ self._logout(ssh, admin_logout=True)
+
+
+ def _hydra_login(self, ssh):
+ """
+ Perform the extra steps required to log into a hydra serial
+ concentrator.
+
+ @param ssh: pexpect.spawn object used to communicate with the hydra
+ serial concentrator.
+
+ @return: True if the login procedure is successful. False if an error
+ occurred. The most common case would be if another user is
+ logged into the device.
+ """
+ try:
+ ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60)
+ ssh.sendline(CONFIG.get(self.hydra_name,'password'))
+ ssh.sendline('')
+ response = ssh.expect_list(
+ [re.compile(RPMController.USERNAME_PROMPT),
+ re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)],
+ timeout=60)
+ except pexpect.EOF:
+ # Did not receive any of the expect responses, retry.
+ return False
+ except pexpect.TIMEOUT:
+ logging.debug('Timeout occurred logging in to hydra.')
+ return False
+ # Send the username that the subclass will have set in its
+ # construction.
+ if response == 1:
+ logging.debug('SSH Terminal most likely serving another'
+ ' connection, retrying.')
+ # Kill the connection for the next connection attempt.
+ try:
+ self._kill_previous_connection()
+ except pexpect.ExceptionPexpect:
+ logging.error('Failed to disconnect previous connection, '
+ 'retrying.')
+ raise
+ return False
+ logging.debug('Connected to rpm through hydra. Logging in.')
+ return True
+
+
+ def _authenticate_with_hydra(self, admin_override=False):
+ """
+ Some RPM's are behind a hydra serial concentrator and require their ssh
+ connection to be tunneled through this device. This can fail if another
+ user is logged in; therefore this will retry multiple times.
+
+ This function also allows us to authenticate directly to the
+ administrator interface of the hydra device.
+
+ @param admin_override: Set to True if we are trying to access the
+ administrator interface rather than tunnel
+ through to the RPM.
+
+ @return: The connected pexpect.spawn instance if the login procedure is
+ successful. None if an error occurred. The most common case
+ would be if another user is logged into the device.
+ """
+ if admin_override:
+ username = CONFIG.get(self.hydra_name, 'admin_username')
+ else:
+ username = '%s:%s' % (CONFIG.get(self.hydra_name,'username'),
+ self.hostname)
+ cmd = RPMController.SSH_LOGIN_CMD % (username, self._hydra_hostname)
+ num_attempts = 0
+ while num_attempts < RPMController.HYDRA_MAX_CONNECT_RETRIES:
+ try:
+ ssh = pexpect.spawn(cmd)
+ except pexpect.ExceptionPexpect:
+ return None
+ if admin_override:
+ return ssh
+ if self._hydra_login(ssh):
+ return ssh
+ # Authenticating with hydra failed. Sleep then retry.
+ time.sleep(RPMController.HYRDA_RETRY_SLEEP_SECS)
+ num_attempts += 1
+ logging.error('Failed to connect to the hydra serial concentrator after'
+ ' %d attempts.', RPMController.HYDRA_MAX_CONNECT_RETRIES)
+ return None
+
+
+ def _login(self):
+ """
+ Log in into the RPM Device.
+
+ The login process should be able to connect to the device whether or not
+ it is behind a hydra serial concentrator.
+
+ @return: ssh - a pexpect.spawn instance if the connection was successful
+ or None if it was not.
+ """
+ if self.behind_hydra:
+ # Tunnel the connection through the hydra.
+ ssh = self._authenticate_with_hydra()
+ if not ssh:
+ return None
+ ssh.sendline(self._username)
+ else:
+ # Connect directly to the RPM over SSH.
+ hostname = '%s.%s' % (self.hostname, self._dns_zone)
+ cmd = RPMController.SSH_LOGIN_CMD % (self._username, hostname)
+ try:
+ ssh = pexpect.spawn(cmd)
+ except pexpect.ExceptionPexpect:
+ return None
+ # Wait for the password prompt
+ try:
+ ssh.expect(self.PASSWORD_PROMPT, timeout=60)
+ ssh.sendline(self._password)
+ ssh.expect(self.DEVICE_PROMPT, timeout=60)
+ except pexpect.ExceptionPexpect:
+ return None
+ return ssh
+
+
+ def _logout(self, ssh, admin_logout=False):
+ """
+ Log out of the RPM device.
+
+ Send the device specific logout command and if the connection is through
+ a hydra serial concentrator, kill the ssh connection.
+
+ @param admin_logout: Set to True if we are trying to logout of the
+ administrator interface of a hydra serial
+ concentrator, rather than an RPM.
+ @param ssh: pexpect.spawn instance to use to send the logout command.
+ """
+ if admin_logout:
+ ssh.sendline(RPMController.QUIT_CMD)
+ ssh.expect(RPMController.HYDRA_PROMPT)
+ ssh.sendline(self.LOGOUT_CMD)
+ if self.behind_hydra and not admin_logout:
+ # Terminate the hydra session.
+ ssh.sendline('~.')
+ # Wait a bit so hydra disconnects completely. Launching another
+ # request immediately can cause a timeout.
+ time.sleep(5)
+
+
def set_power_state(self, dut_hostname, new_state):
"""
Set the state of the dut's outlet on this RPM.
- To be implemented by the subclasses.
+
+ For ssh based devices, this will create the connection either directly
+ or through a hydra tunnel and call the underlying _change_state function
+ to be implemented by the subclass device.
+
+ For non-ssh based devices, this method should be overloaded with the
+ proper connection and state change code. And the subclass will handle
+ accessing the RPM devices.
@param dut_hostname: hostname of DUT whose outlet we want to change.
@param new_state: ON/OFF/CYCLE - state or action we want to perform on
@@ -149,8 +372,51 @@
@return: True if the attempt to change power state was successful,
False otherwise.
"""
- raise NotImplementedError('Abstract class. Subclasses should implement '
- 'set_power_state().')
+ if dut_hostname.startswith('chromeos2'):
+ # Because the devices behind in chromeos2 lab all have long
+ # hostnames, we can't store their full names in the rpm, therefore
+ # for these devices we drop the 'chromeos2' part of their name.
+ # For example: chromeos2-rack2-row1-host1 is just stored as
+ # rack2-row1-host1 inside the RPM.
+ dut_hostname = dut_hostname.split('-', 1)[1]
+ ssh = self._login()
+ if not ssh:
+ return False
+ # Try to change the state of the DUT's power outlet.
+ result = self._change_state(dut_hostname, new_state, ssh)
+ # Terminate hydra connection if necessary.
+ self._logout(ssh)
+ return result
+
+
+ def _change_state(self, dut_hostname, new_state, ssh):
+ """
+ Perform the actual state change operation.
+
+ Once we have established communication with the RPM this method is
+ responsible for changing the state of the RPM outlet.
+
+ @param dut_hostname: hostname of DUT whose outlet we want to change.
+ @param new_state: ON/OFF/CYCLE - state or action we want to perform on
+ the outlet.
+ @param ssh: The ssh connection used to execute the state change commands
+ on the RPM device.
+
+ @return: True if the attempt to change power state was successful,
+ False otherwise.
+ """
+ ssh.sendline(self.SET_STATE_CMD % (new_state, dut_hostname))
+ if self.SUCCESS_MSG:
+ # If this RPM device returns a success message check for it before
+ # continuing.
+ try:
+ ssh.expect(self.SUCCESS_MSG, timeout=60)
+ except pexpect.ExceptionPexpect:
+ logging.error('Request to change outlet for DUT: %s to new '
+ 'state %s failed.', dut_hostname, new_state)
+ return False
+ logging.debug('Outlet for DUT: %s set to %s', dut_hostname, new_state)
+ return True
def type(self):
@@ -175,50 +441,20 @@
@var _username: username used to access device.
@var _password: password used to access device.
- @var _ssh_mock: mocked ssh interface used for testing.
"""
DEVICE_PROMPT = 'Switched CDU:'
- SSH_LOGIN_CMD = 'ssh -l %s -o StrictHostKeyChecking=no ' \
- '-o UserKnownHostsFile=/dev/null %s'
- PASSWORD_PROMPT = 'Password:'
SET_STATE_CMD = '%s %s'
SUCCESS_MSG = 'Command successful'
- LOGOUT_CMD = 'logout'
- def __init__(self, hostname, ssh_mock=None):
- super(SentryRPMController, self).__init__(hostname)
+ def __init__(self, hostname, hydra_name=None):
+ if hostname.startswith('chromeos2'):
+ hydra_name = 'hydra1'
+ super(SentryRPMController, self).__init__(hostname, hydra_name)
self._username = CONFIG.get('SENTRY', 'username')
self._password = CONFIG.get('SENTRY', 'password')
- self._ssh_mock = ssh_mock
-
-
- def set_power_state(self, dut_hostname, new_state):
- logging.debug("Setting outlet for DUT: %s to state: %s",
- dut_hostname, new_state)
- result = True
- cmd = SentryRPMController.SSH_LOGIN_CMD % (self._username,
- self.hostname)
- if not self._ssh_mock: # For testing purposes.
- ssh = pexpect.spawn(cmd)
- else:
- ssh = self._ssh_mock
- ssh.expect(SentryRPMController.PASSWORD_PROMPT, timeout=60)
- ssh.sendline(self._password)
- ssh.expect(SentryRPMController.DEVICE_PROMPT, timeout=60)
- ssh.sendline(SentryRPMController.SET_STATE_CMD % (new_state,
- dut_hostname))
- try:
- ssh.expect(SentryRPMController.SUCCESS_MSG, timeout=60)
- except pexpect.TIMEOUT:
- logging.error('Request to change outlet for DUT: %s to new '
- 'state %s timed out.', dut_hostname, new_state)
- result = False
- finally:
- ssh.sendline(SentryRPMController.LOGOUT_CMD)
- return result
def type(self):
@@ -230,7 +466,7 @@
This class implements RPMController for the Web Powered units
produced by Digital Loggers Inc.
- @var _rpm: dli.powerswitch instance used to interact with RPM.
+ @var _rpm: dli_urllib.Powerswitch instance used to interact with RPM.
"""
@@ -240,10 +476,15 @@
def __init__(self, hostname, powerswitch=None):
username = CONFIG.get('WEBPOWERED', 'username')
password = CONFIG.get('WEBPOWERED', 'password')
+ # Call the constructor in RPMController. However since this is a web
+ # accessible device, there should not be a need to tunnel through a
+ # hydra serial concentrator.
super(WebPoweredRPMController, self).__init__(hostname)
+ self.hostname = '%s.%s' % (self.hostname, self._dns_zone)
if not powerswitch:
- self._rpm = dli.powerswitch(hostname=self.hostname, userid=username,
- password=password)
+ self._rpm = dli_urllib.Powerswitch(hostname=self.hostname,
+ userid=username,
+ password=password)
else:
# Should only be used in unit_testing
self._rpm = powerswitch
@@ -265,6 +506,11 @@
def set_power_state(self, dut_hostname, new_state):
+ """
+ Since this does not utilize SSH in any manner, this will overload the
+ set_power_state in RPMController and completes all steps of changing
+ the DUT's outlet state.
+ """
outlet_and_state = self._get_outlet_value_and_state(dut_hostname)
if not outlet_and_state:
logging.error('DUT %s is not on rpm %s',
@@ -317,23 +563,27 @@
"""Simple integration testing."""
rpm = WebPoweredRPMController('chromeos-rack8e-rpm1')
threading.Thread(target=rpm.queue_request,
- args=('chromeos-rack8e-hostbs1', 'ON')).start()
+ args=('chromeos-rack8e-hostbs1', 'OFF')).start()
threading.Thread(target=rpm.queue_request,
args=('chromeos-rack8e-hostbs2', 'ON')).start()
def test_parrallel_sshrequests():
"""Simple integration testing."""
- rpm = SentryRPMController('chromeos-rack1-rpm1')
+ rpm = SentryRPMController('chromeos-rack8-rpm1')
+ rpm2 = SentryRPMController('chromeos2-row2-rack3-rpm1')
threading.Thread(target=rpm.queue_request,
- args=('chromeos-rack1-hostbs1', 'OFF')).start()
+ args=('chromeos-rack8-hostbs1', 'OFF')).start()
threading.Thread(target=rpm.queue_request,
- args=('chromeos-rack1-hostbs2', 'ON')).start()
+ args=('chromeos-rack8-hostbs2', 'OFF')).start()
+ threading.Thread(target=rpm2.queue_request,
+ args=('chromeos2-row2-rack3-hostbs', 'ON')).start()
+ threading.Thread(target=rpm2.queue_request,
+ args=('chromeos2-row2-rack3-hostbs2', 'ON')).start()
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG, format=LOGGING_FORMAT)
test_in_order_requests()
test_parrallel_webrequests()
- test_parrallel_sshrequests()
-
+ test_parrallel_sshrequests()
\ No newline at end of file
diff --git a/site_utils/rpm_control_system/rpm_controller_unittest.py b/site_utils/rpm_control_system/rpm_controller_unittest.py
index 4ab2b3b..56315c6 100644
--- a/site_utils/rpm_control_system/rpm_controller_unittest.py
+++ b/site_utils/rpm_control_system/rpm_controller_unittest.py
@@ -18,9 +18,10 @@
def setUp(self):
super(TestSentryRPMController, self).setUp()
- self.ssh = self.mox.CreateMock(pexpect.spawn)
- self.rpm = rpm_controller.SentryRPMController('chromeos-rack1-host8',
- self.ssh)
+ self.ssh = self.mox.CreateMockAnything()
+ rpm_controller.pexpect.spawn = self.mox.CreateMockAnything()
+ rpm_controller.pexpect.spawn(mox.IgnoreArg()).AndReturn(self.ssh)
+ self.rpm = rpm_controller.SentryRPMController('chromeos-rack1-host8')
def testSuccessfullyChangeOutlet(self):