| #!/usr/bin/python |
| # |
| # Copyright 2010 Google Inc. All Rights Reserved. |
| |
| import datetime |
| import optparse |
| import os |
| import smtplib |
| import sys |
| import time |
| from email.mime.text import MIMEText |
| |
| from autotest_gatherer import AutotestGatherer as AutotestGatherer |
| from autotest_run import AutotestRun as AutotestRun |
| from machine_manager_singleton import MachineManagerSingleton as MachineManagerSingleton |
| from utils import logger |
| from utils.file_utils import FileUtils |
| |
| |
| def CanonicalizeChromeOSRoot(chromeos_root): |
| chromeos_root = os.path.expanduser(chromeos_root) |
| if os.path.isfile(os.path.join(chromeos_root, |
| "src/scripts/enter_chroot.sh")): |
| return chromeos_root |
| else: |
| return None |
| |
| |
| class Autotest(object): |
| def __init__(self, autotest_string): |
| self.name = None |
| self.iterations = None |
| self.args = None |
| fields = autotest_string.split(",", 1) |
| self.name = fields[0] |
| if len(fields) > 1: |
| autotest_string = fields[1] |
| fields = autotest_string.split(",", 1) |
| else: return |
| self.iterations = int(fields[0]) |
| if len(fields) > 1: |
| self.args = fields[1] |
| else: return |
| |
| def __str__(self): |
| return "\n".join([self.name, self.iterations, self.args]) |
| |
| |
| def CreateAutotestListFromString(autotest_strings, default_iterations=None): |
| autotest_list = [] |
| for autotest_string in autotest_strings.split(":"): |
| autotest = Autotest(autotest_string) |
| if default_iterations and not autotest.iterations: |
| autotest.iterations = default_iterations |
| |
| autotest_list.append(autotest) |
| return autotest_list |
| |
| |
| def CreateAutotestRuns(images, autotests, remote, board, exact_remote, |
| rerun, rerun_if_failed, main_chromeos_root=None): |
| autotest_runs = [] |
| for image in images: |
| logger.GetLogger().LogOutput("Computing md5sum of: %s" % image) |
| image_checksum = FileUtils().Md5File(image) |
| logger.GetLogger().LogOutput("md5sum %s: %s" % (image, image_checksum)) |
| ### image_checksum = "abcdefghi" |
| |
| chromeos_root = main_chromeos_root |
| if not main_chromeos_root: |
| image_chromeos_root = os.path.join(os.path.dirname(image), |
| "../../../../..") |
| chromeos_root = CanonicalizeChromeOSRoot(image_chromeos_root) |
| assert chromeos_root, "chromeos_root: %s invalid" % image_chromeos_root |
| else: |
| chromeos_root = CanonicalizeChromeOSRoot(main_chromeos_root) |
| assert chromeos_root, "chromeos_root: %s invalid" % main_chromeos_root |
| |
| # We just need a single ChromeOS root in the MachineManagerSingleton. It is |
| # needed because we can save re-image time by checking the image checksum at |
| # the beginning and assigning autotests to machines appropriately. |
| if not MachineManagerSingleton().chromeos_root: |
| MachineManagerSingleton().chromeos_root = chromeos_root |
| |
| for autotest in autotests: |
| for iteration in range(autotest.iterations): |
| autotest_run = AutotestRun(autotest, |
| chromeos_root=chromeos_root, |
| chromeos_image=image, |
| board=board, |
| remote=remote, |
| iteration=iteration, |
| image_checksum=image_checksum, |
| exact_remote=exact_remote, |
| rerun=rerun, |
| rerun_if_failed=rerun_if_failed) |
| autotest_runs.append(autotest_run) |
| return autotest_runs |
| |
| |
| def GetNamesAndIterations(autotest_runs): |
| strings = [] |
| for autotest_run in autotest_runs: |
| strings.append("%s:%s" % (autotest_run.autotest.name, |
| autotest_run.iteration)) |
| return " %s (%s)" % (len(strings), " ".join(strings)) |
| |
| |
| def GetStatusString(autotest_runs): |
| status_bins = {} |
| for autotest_run in autotest_runs: |
| if autotest_run.status not in status_bins: |
| status_bins[autotest_run.status] = [] |
| status_bins[autotest_run.status].append(autotest_run) |
| |
| status_strings = [] |
| for key, val in status_bins.items(): |
| status_strings.append("%s: %s" % (key, GetNamesAndIterations(val))) |
| return "Thread Status:\n%s" % "\n".join(status_strings) |
| |
| |
| def GetProgressBar(num_done, num_total): |
| ret = "Done: %s%%" % int(100.0 * num_done / num_total) |
| bar_length = 50 |
| done_char = ">" |
| undone_char = " " |
| num_done_chars = bar_length * num_done / num_total |
| num_undone_chars = bar_length - num_done_chars |
| ret += " [%s%s]" % (num_done_chars * done_char, num_undone_chars * |
| undone_char) |
| return ret |
| |
| |
| def GetProgressString(start_time, num_remain, num_total): |
| current_time = time.time() |
| elapsed_time = current_time - start_time |
| try: |
| eta_seconds = float(num_remain) * elapsed_time / (num_total - num_remain) |
| eta_seconds = int(eta_seconds) |
| eta = datetime.timedelta(seconds=eta_seconds) |
| except ZeroDivisionError: |
| eta = "Unknown" |
| strings = [] |
| strings.append("Current time: %s Elapsed: %s ETA: %s" % |
| (datetime.datetime.now(), |
| datetime.timedelta(seconds=int(elapsed_time)), |
| eta)) |
| strings.append(GetProgressBar(num_total - num_remain, num_total)) |
| return "\n".join(strings) |
| |
| |
| def RunAutotestRunsInParallel(autotest_runs): |
| start_time = time.time() |
| active_threads = [] |
| for autotest_run in autotest_runs: |
| # Set threads to daemon so program exits when ctrl-c is pressed. |
| autotest_run.daemon = True |
| autotest_run.start() |
| active_threads.append(autotest_run) |
| |
| print_interval = 30 |
| last_printed_time = time.time() |
| while active_threads: |
| try: |
| active_threads = [t for t in active_threads if t is not None |
| and t.isAlive()] |
| for t in active_threads: |
| t.join(1) |
| if time.time() - last_printed_time > print_interval: |
| border = "==============================" |
| logger.GetLogger().LogOutput(border) |
| logger.GetLogger().LogOutput(GetProgressString( |
| start_time, |
| len([t for t in autotest_runs if t.status not in ["SUCCEEDED", |
| "FAILED"]]), |
| len(autotest_runs))) |
| logger.GetLogger().LogOutput(GetStatusString(autotest_runs)) |
| logger.GetLogger().LogOutput("%s\n" % |
| MachineManagerSingleton().AsString()) |
| logger.GetLogger().LogOutput(border) |
| last_printed_time = time.time() |
| except KeyboardInterrupt: |
| print "C-c received... cleaning up threads." |
| for t in active_threads: |
| t.terminate = True |
| return 1 |
| return 0 |
| |
| |
| def RunAutotestRunsSerially(autotest_runs): |
| for autotest_run in autotest_runs: |
| retval = autotest_run.Run() |
| if retval: return retval |
| |
| |
| def ProduceTables(autotest_runs, full_table, fit_string): |
| l = logger.GetLogger() |
| ags_dict = {} |
| for autotest_run in autotest_runs: |
| name = autotest_run.full_name |
| if name not in ags_dict: |
| ags_dict[name] = AutotestGatherer() |
| ags_dict[name].runs.append(autotest_run) |
| output = "" |
| for b, ag in ags_dict.items(): |
| output += "Benchmark: %s\n" % b |
| output += ag.GetFormattedMainTable(percents_only=not full_table, |
| fit_string=fit_string) |
| output += "\n" |
| |
| summary = "" |
| for b, ag in ags_dict.items(): |
| summary += "Benchmark Summary Table: %s\n" % b |
| summary += ag.GetFormattedSummaryTable(percents_only=not full_table, |
| fit_string=fit_string) |
| summary += "\n" |
| |
| output += summary |
| output += ("Number of re-images performed: %s" % |
| MachineManagerSingleton().num_reimages) |
| l.LogOutput(output) |
| |
| if autotest_runs: |
| board = autotest_runs[0].board |
| else: |
| board = "" |
| |
| subject = "%s: %s" % (board, ", ".join(ags_dict.keys())) |
| |
| if any(autotest_run.run_completed for autotest_run in autotest_runs): |
| SendEmailToUser(subject, summary) |
| |
| |
| def SendEmailToUser(subject, text_to_send): |
| # Email summary to the current user. |
| msg = MIMEText(text_to_send) |
| |
| # me == the sender's email address |
| # you == the recipient's email address |
| me = os.path.basename(__file__) |
| you = os.getlogin() |
| msg["Subject"] = "[%s] %s" % (os.path.basename(__file__), subject) |
| msg["From"] = me |
| msg["To"] = you |
| |
| # Send the message via our own SMTP server, but don't include the |
| # envelope header. |
| s = smtplib.SMTP("localhost") |
| s.sendmail(me, [you], msg.as_string()) |
| s.quit() |
| |
| |
| def Main(argv): |
| """The main function.""" |
| # Common initializations |
| ### command_executer.InitCommandExecuter(True) |
| l = logger.GetLogger() |
| |
| parser = optparse.OptionParser() |
| parser.add_option("-t", "--tests", dest="tests", |
| help=("Tests to compare." |
| "Optionally specify per-test iterations by:" |
| "<test>,<iter>:<args>")) |
| parser.add_option("-c", "--chromeos_root", dest="chromeos_root", |
| help="A *single* chromeos_root where scripts can be found.") |
| parser.add_option("-n", "--iterations", dest="iterations", |
| help="Iterations to run per benchmark.", |
| default=1) |
| parser.add_option("-r", "--remote", dest="remote", |
| help="The remote chromeos machine.") |
| parser.add_option("-b", "--board", dest="board", |
| help="The remote board.") |
| parser.add_option("--full_table", dest="full_table", |
| help="Print full tables.", |
| action="store_true", |
| default=True) |
| parser.add_option("--exact_remote", |
| dest="exact_remote", |
| help="Run tests on the exact remote.", |
| action="store_true", |
| default=False) |
| parser.add_option("--fit_string", dest="fit_string", |
| help="Fit strings to fixed sizes.", |
| action="store_true", |
| default=False) |
| parser.add_option("--rerun", |
| dest="rerun", |
| help="Re-run regardless of cache hit.", |
| action="store_true", |
| default=False) |
| parser.add_option("--rerun_if_failed", |
| dest="rerun_if_failed", |
| help="Re-run if previous run was a failure.", |
| action="store_true", |
| default=False) |
| parser.add_option("--no_lock", |
| dest="no_lock", |
| help="Do not lock the machine before running the tests.", |
| action="store_true", |
| default=False) |
| l.LogOutput(" ".join(argv)) |
| [options, args] = parser.parse_args(argv) |
| |
| if options.remote is None: |
| l.LogError("No remote machine specified.") |
| parser.print_help() |
| return 1 |
| |
| if not options.board: |
| l.LogError("No board specified.") |
| parser.print_help() |
| return 1 |
| |
| remote = options.remote |
| tests = options.tests |
| board = options.board |
| exact_remote = options.exact_remote |
| iterations = int(options.iterations) |
| |
| autotests = CreateAutotestListFromString(tests, iterations) |
| |
| main_chromeos_root = options.chromeos_root |
| images = args[1:] |
| fit_string = options.fit_string |
| full_table = options.full_table |
| rerun = options.rerun |
| rerun_if_failed = options.rerun_if_failed |
| |
| MachineManagerSingleton().no_lock = options.no_lock |
| |
| # Now try creating all the Autotests |
| autotest_runs = CreateAutotestRuns(images, autotests, remote, board, |
| exact_remote, rerun, rerun_if_failed, |
| main_chromeos_root) |
| |
| try: |
| # At this point we have all the autotest runs. |
| for machine in remote.split(","): |
| MachineManagerSingleton().AddMachine(machine) |
| |
| retval = RunAutotestRunsInParallel(autotest_runs) |
| if retval: return retval |
| |
| # Now print tables |
| ProduceTables(autotest_runs, full_table, fit_string) |
| finally: |
| # not sure why this isn't called at the end normally... |
| MachineManagerSingleton().__del__() |
| |
| return 0 |
| |
| if __name__ == "__main__": |
| sys.exit(Main(sys.argv)) |