[autotest] Add a log socket server for RPM controller to write log from multiple processes
set_power_state call is executed in a new process. logging to a single file
from multiple process may cause the process to hang. A reliable solution is to
create a SocketHandler to log to a TCP server. The source code is from python
documentation:
https://docs.python.org/2/howto/logging-cookbook.html#network-logging
BUG=chromium:243567
DEPLOY=rpm
TEST=unittest, local test script
modify frontend_addr in rpm_config.ini to localhost
Add a dummy set_power_state call in rpm_controller.py to wait indefinitely.
Also add some logging call in it.
Run frontend_server and rpm_dispatcher in localhost
Run a test script that calls 500 duts, e.g., chromeos1-rack#-host 1, #=1-500
Wait for all calls to time out, check ps to make sure only 2 rpm_dispatcher
processes are running at the end (the additional one is for the TCP server
collecting log).
Change-Id: I594cb55c9a2230ac34c90eca7e9de355c21a0dca
Reviewed-on: https://chromium-review.googlesource.com/194432
Tested-by: Dan Shi <dshi@chromium.org>
Reviewed-by: Alex Miller <milleral@chromium.org>
Commit-Queue: Dan Shi <dshi@chromium.org>
diff --git a/site_utils/rpm_control_system/rpm_logging_config.py b/site_utils/rpm_control_system/rpm_logging_config.py
index e887f55..cfa2f1b 100644
--- a/site_utils/rpm_control_system/rpm_logging_config.py
+++ b/site_utils/rpm_control_system/rpm_logging_config.py
@@ -9,6 +9,7 @@
import time
from config import rpm_config
+from autotest_lib.site_utils import log_socket_server
LOGGING_FORMAT = rpm_config.get('GENERAL', 'logging_format')
RECEIVERS = rpm_config.get('RPM_INFRASTRUCTURE',
@@ -54,26 +55,59 @@
return super(SuspendableSMTPHandler, self).emit(record)
-def set_up_logging(log_filename_format):
+def get_log_filename(log_filename_format):
+ """Get file name of log based on given log_filename_format.
+
+ @param log_filename_format: Format to use to create the log file.
+
+ @raise Exception: If log_filename_format is None.
+ """
+ if not log_filename_format:
+ raise Exception('log_filename_format must be set.')
+
+ log_filename = os.path.abspath(time.strftime(log_filename_format))
+ log_dir = os.path.dirname(log_filename)
+ if not os.path.isdir(log_dir):
+ os.makedirs(log_dir)
+ return log_filename
+
+
+def set_up_logging(log_filename_format=None, use_log_server=False):
"""
Correctly set up logging to have the correct format/level, log to a file,
and send out email notifications in case of error level messages.
@param log_filename_format: Format to use to create the log file.
+ @param use_log_server: True if log to a TCP server.
@returns email_handler: Logging handler used to send out email alerts.
"""
- log_filename = os.path.abspath(time.strftime(log_filename_format))
- log_dir = os.path.dirname(log_filename)
- if not os.path.isdir(log_dir):
- os.makedirs(log_dir)
- logging.basicConfig(filename=log_filename, level=logging.INFO,
- format=LOGGING_FORMAT)
+ if use_log_server:
+ socketHandler = logging.handlers.SocketHandler(
+ 'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
+ logging.getLogger().addHandler(socketHandler)
+ else:
+ log_filename = get_log_filename(log_filename_format)
+ logging.basicConfig(filename=log_filename, level=logging.INFO,
+ format=LOGGING_FORMAT)
+
if rpm_config.getboolean('GENERAL', 'debug'):
logging.getLogger().setLevel(logging.DEBUG)
+
email_handler = SuspendableSMTPHandler('localhost', 'rpm@google.com',
RECEIVERS, SUBJECT_LINE, None)
email_handler.setLevel(logging.ERROR)
email_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
logging.getLogger('').addHandler(email_handler)
- return email_handler
\ No newline at end of file
+ return email_handler
+
+
+def start_log_server(log_filename_format):
+ """Start log server to accept logging through a TCP server.
+
+ @param log_filename_format: Format to use to create the log file.
+ """
+ log_filename = get_log_filename(log_filename_format)
+ log_socket_server.LogSocketServer.start(filename=log_filename,
+ level=logging.INFO,
+ format=LOGGING_FORMAT)