| #!/usr/bin/python |
| # |
| # Copyright 2010 Google Inc. All Rights Reserved. |
| """Script to compare a baseline results file to a new results file.""" |
| |
| __author__ = 'raymes@google.com (Raymes Khoury)' |
| |
| import sys |
| from utils import logger |
| from utils import html_tools |
| |
| PASS = 'pass' |
| FAIL = 'fail' |
| NOT_EXECUTED = 'not executed' |
| |
| |
| class ResultsReport: |
| |
| def __init__(self, report, num_tests_executed, num_passes, num_failures, |
| num_regressions): |
| self.report = report |
| self.num_tests_executed = num_tests_executed |
| self.num_passes = num_passes |
| self.num_failures = num_failures |
| self.num_regressions = num_regressions |
| |
| def GetReport(self): |
| return self.report |
| |
| def GetNumExecuted(self): |
| return self.num_tests_executed |
| |
| def GetNumPasses(self): |
| return self.num_passes |
| |
| def GetNumFailures(self): |
| return self.num_failures |
| |
| def GetNumRegressions(self): |
| return self.num_regressions |
| |
| def GetSummary(self): |
| summary = 'Tests executed: %s\n' % str(self.num_tests_executed) |
| summary += 'Tests Passing: %s\n' % str(self.num_passes) |
| summary += 'Tests Failing: %s\n' % str(self.num_failures) |
| summary += 'Regressions: %s\n' % str(self.num_regressions) |
| return summary |
| |
| |
| def Usage(): |
| print 'Usage: %s baseline_results new_results' % sys.argv[0] |
| sys.exit(1) |
| |
| |
| def ParseResults(results_filename): |
| results = [] |
| try: |
| results_file = open(results_filename, 'rb') |
| for line in results_file: |
| if line.strip() != '': |
| results.append(line.strip().split('\t')) |
| results_file.close() |
| except IOError: |
| logger.GetLogger().LogWarning('Could not open results file: ' + |
| results_filename) |
| return results |
| |
| |
| def GenerateResultsReport(baseline_file, new_result_file): |
| baseline_results = ParseResults(baseline_file) |
| new_results = ParseResults(new_result_file) |
| |
| test_status = {} |
| |
| for new_result in new_results: |
| new_test_name = new_result[0] |
| new_test_result = new_result[1] |
| test_status[new_test_name] = (new_test_result, NOT_EXECUTED) |
| |
| for baseline_result in baseline_results: |
| baseline_test_name = baseline_result[0] |
| baseline_test_result = baseline_result[1] |
| if baseline_test_name in test_status: |
| new_test_result = test_status[baseline_test_name][0] |
| test_status[baseline_test_name] = (new_test_result, baseline_test_result) |
| else: |
| test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result) |
| |
| regressions = [] |
| for result in test_status.keys(): |
| if test_status[result][0] != test_status[result][1]: |
| regressions.append(result) |
| |
| num_tests_executed = len(new_results) |
| num_regressions = len(regressions) |
| num_passes = 0 |
| num_failures = 0 |
| for result in new_results: |
| if result[1] == PASS: |
| num_passes += 1 |
| else: |
| num_failures += 1 |
| |
| report = html_tools.GetPageHeader('Test Summary') |
| report += html_tools.GetHeader('Test Summary') |
| report += html_tools.GetListHeader() |
| report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed)) |
| report += html_tools.GetListItem('Passes: ' + str(num_passes)) |
| report += html_tools.GetListItem('Failures: ' + str(num_failures)) |
| report += html_tools.GetListItem('Regressions: ' + str(num_regressions)) |
| report += html_tools.GetListFooter() |
| report += html_tools.GetHeader('Regressions', 2) |
| report += html_tools.GetTableHeader(['Test name', 'Expected result', |
| 'Actual result']) |
| |
| for regression in regressions: |
| report += html_tools.GetTableRow([regression[:150], test_status[regression][ |
| 1], test_status[regression][0]]) |
| report += '\n' |
| report += html_tools.GetTableFooter() |
| report += html_tools.GetHeader('All Tests', 2) |
| report += html_tools.GetTableHeader(['Test name', 'Expected result', |
| 'Actual result']) |
| for result in test_status.keys(): |
| report += html_tools.GetTableRow([result[:150], test_status[result][1], |
| test_status[result][0]]) |
| report += '\n' |
| report += html_tools.GetTableFooter() |
| report += html_tools.GetFooter() |
| return ResultsReport(report, num_tests_executed, num_passes, num_failures, |
| num_regressions) |
| |
| |
| def Main(argv): |
| if len(argv) < 2: |
| Usage() |
| |
| print GenerateResultsReport(argv[1], argv[2])[0] |
| |
| |
| if __name__ == '__main__': |
| Main(sys.argv) |