report_generator.py

#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.

"""Script to  compare a baseline results file to a new results file."""

__author__ = "raymes@google.com (Raymes Khoury)"

import sys
from utils import logger
from utils import html_tools

PASS = "pass"
FAIL = "fail"
NOT_EXECUTED = "not executed"

class ResultsReport:
  def __init__(self, report, num_tests_executed, num_passes, num_failures, num_regressions):
    self.report = report
    self.num_tests_executed = num_tests_executed
    self.num_passes = num_passes
    self.num_failures = num_failures
    self.num_regressions = num_regressions
  
  def GetReport(self):
    return self.report
  
  def GetNumExecuted(self):
    return self.num_tests_executed
  
  def GetNumPasses(self):
    return self.num_passes
  
  def GetNumFailures(self):
    return self.num_failures
  
  def GetNumRegressions(self):
    return self.num_regressions

  def GetSummary(self):
    summary = "Tests executed: %s\n" % str(self.num_tests_executed)
    summary += "Tests Passing: %s\n" % str(self.num_passes)
    summary += "Tests Failing: %s\n" % str(self.num_failures)
    summary += "Regressions: %s\n" % str(self.num_regressions)
    return summary

def Usage():
  print "Usage: %s baseline_results new_results" % sys.argv[0]
  sys.exit(1)

def ParseResults(results_filename):
  results = []
  try:
    results_file = open(results_filename, 'rb')
    for line in results_file:
      if line.strip() != "":
        results.append(line.strip().split("\t"))
    results_file.close()
  except IOError:
    logger.GetLogger().LogWarning("Could not open results file: " +
                                  results_filename)
  return results

def GenerateResultsReport(baseline_file, new_result_file):
  baseline_results = ParseResults(baseline_file)
  new_results = ParseResults(new_result_file)

  test_status = {}

  for new_result in new_results:
    new_test_name = new_result[0]
    new_test_result = new_result[1]
    test_status[new_test_name] = (new_test_result, NOT_EXECUTED)

  for baseline_result in baseline_results:
    baseline_test_name = baseline_result[0]
    baseline_test_result = baseline_result[1]
    if baseline_test_name in test_status:
      new_test_result = test_status[baseline_test_name][0]
      test_status[baseline_test_name] = (new_test_result, baseline_test_result)
    else:
      test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result)

  regressions = []
  for result in test_status.keys():
    if test_status[result][0] != test_status[result][1]:
      regressions.append(result)
  
  num_tests_executed = len(new_results)
  num_regressions = len(regressions)
  num_passes = 0
  num_failures = 0
  for result in new_results:
    if result[1] == PASS:
      num_passes += 1
    else:
      num_failures += 1

  report = html_tools.GetPageHeader("Test Summary")
  report += html_tools.GetHeader("Test Summary")
  report += html_tools.GetListHeader()
  report += html_tools.GetListItem("Tests executed: " + str(num_tests_executed))
  report += html_tools.GetListItem("Passes: " + str(num_passes))
  report += html_tools.GetListItem("Failures: " + str(num_failures))
  report += html_tools.GetListItem("Regressions: " + str(num_regressions))
  report += html_tools.GetListFooter()
  report += html_tools.GetHeader("Regressions", 2)
  report += html_tools.GetTableHeader(["Test name", "Expected result", 
                                       "Actual result"])
  
  for regression in regressions:
    report += html_tools.GetTableRow([regression[:150], test_status[regression][1],
                                      test_status[regression][0]])
    report += "\n"
  report += html_tools.GetTableFooter()
  report += html_tools.GetHeader("All Tests", 2)
  report += html_tools.GetTableHeader(["Test name", "Expected result", "Actual result"])
  for result in test_status.keys():
    report += html_tools.GetTableRow([result[:150], test_status[result][1],
                              test_status[result][0]])
    report += "\n"
  report += html_tools.GetTableFooter()
  report += html_tools.GetFooter()
  return ResultsReport(report, num_tests_executed, num_passes, num_failures, num_regressions)

def Main(argv):
  if len(argv) < 2:
    Usage()

  print GenerateResultsReport(argv[1], argv[2])[0]

if __name__ == "__main__":
  Main(sys.argv)