client/common_lib/test_unittest.py

#!/usr/bin/python
#pylint: disable-msg=C0111
"""Unit Tests for autotest.client.common_lib.test"""

__author__ = 'gps@google.com (Gregory P. Smith)'

import json
import tempfile
import unittest
import common
from autotest_lib.client.common_lib import test
from autotest_lib.client.common_lib.test_utils import mock
from autotest_lib.client.common_lib import error as common_lib_error

class TestTestCase(unittest.TestCase):
    class _neutered_base_test(test.base_test):
        """A child class of base_test to avoid calling the constructor."""
        def __init__(self, *args, **kwargs):
            class MockJob(object):
                pass
            class MockProfilerManager(object):
                def active(self):
                    return False
                def present(self):
                    return True
            self.job = MockJob()
            self.job.default_profile_only = False
            self.job.profilers = MockProfilerManager()
            self.job.test_retry = 0
            self._new_keyval = False
            self.iteration = 0
            self.before_iteration_hooks = []
            self.after_iteration_hooks = []


    def setUp(self):
        self.god = mock.mock_god()
        self.test = self._neutered_base_test()


    def tearDown(self):
        self.god.unstub_all()



class Test_base_test_execute(TestTestCase):
    # Test the various behaviors of the base_test.execute() method.
    def setUp(self):
        TestTestCase.setUp(self)
        self.god.stub_function(self.test, 'run_once_profiling')
        self.god.stub_function(self.test, 'postprocess')
        self.god.stub_function(self.test, 'process_failed_constraints')


    def test_call_run_once(self):
        # setup
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.god.stub_function(self.test, 'run_once')
        self.god.stub_function(self.test, 'postprocess_iteration')
        self.god.stub_function(self.test, 'analyze_perf_constraints')
        before_hook = self.god.create_mock_function('before_hook')
        after_hook = self.god.create_mock_function('after_hook')
        self.test.register_before_iteration_hook(before_hook)
        self.test.register_after_iteration_hook(after_hook)

        # tests the test._call_run_once implementation
        self.test.drop_caches_between_iterations.expect_call()
        before_hook.expect_call(self.test)
        self.test.run_once.expect_call(1, 2, arg='val')
        self.test.postprocess_iteration.expect_call()
        self.test.analyze_perf_constraints.expect_call([])
        after_hook.expect_call(self.test)
        self.test._call_run_once([], False, None, (1, 2), {'arg': 'val'})
        self.god.check_playback()


    def test_call_run_once_with_exception(self):
        # setup
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.god.stub_function(self.test, 'run_once')
        before_hook = self.god.create_mock_function('before_hook')
        after_hook = self.god.create_mock_function('after_hook')
        self.test.register_before_iteration_hook(before_hook)
        self.test.register_after_iteration_hook(after_hook)
        error = Exception('fail')

        # tests the test._call_run_once implementation
        self.test.drop_caches_between_iterations.expect_call()
        before_hook.expect_call(self.test)
        self.test.run_once.expect_call(1, 2, arg='val').and_raises(error)
        after_hook.expect_call(self.test)
        try:
            self.test._call_run_once([], False, None, (1, 2), {'arg': 'val'})
        except:
            pass
        self.god.check_playback()


    def _setup_failed_test_calls(self, fail_count, error):
        """
        Set up failed test calls for use with call_run_once_with_retry.

        @param fail_count: The amount of times to mock a failure.
        @param error: The error to raise while failing.
        """
        self.god.stub_function(self.test.job, 'record')
        self.god.stub_function(self.test, '_call_run_once')
        # tests the test._call_run_once implementation
        for run in xrange(0, fail_count):
            self.test._call_run_once.expect_call([], False, None, (1, 2),
                                                 {'arg': 'val'}).and_raises(
                                                          error)
            info_str = 'Run %s failed with %s' % (run, error)
            # On the final run we do not emit this message.
            if run != self.test.job.test_retry and isinstance(error,
                                               common_lib_error.TestFailRetry):
                self.test.job.record.expect_call('INFO', None, None, info_str)


    def test_call_run_once_with_retry_exception(self):
        """
        Test call_run_once_with_retry duplicating a test that will always fail.
        """
        self.test.job.test_retry = 5
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.god.stub_function(self.test, 'run_once')
        before_hook = self.god.create_mock_function('before_hook')
        after_hook = self.god.create_mock_function('after_hook')
        self.test.register_before_iteration_hook(before_hook)
        self.test.register_after_iteration_hook(after_hook)
        error = common_lib_error.TestFailRetry('fail')
        self._setup_failed_test_calls(self.test.job.test_retry+1, error)
        try:
            self.test._call_run_once_with_retry([], False, None, (1, 2),
                                                {'arg': 'val'})
        except Exception as err:
            if err != error:
                raise
        self.god.check_playback()


    def test_call_run_once_with_retry_exception_unretryable(self):
        """
        Test call_run_once_with_retry duplicating a test that will always fail
        with a non-retryable exception.
        """
        self.test.job.test_retry = 5
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.god.stub_function(self.test, 'run_once')
        before_hook = self.god.create_mock_function('before_hook')
        after_hook = self.god.create_mock_function('after_hook')
        self.test.register_before_iteration_hook(before_hook)
        self.test.register_after_iteration_hook(after_hook)
        error = common_lib_error.TestFail('fail')
        self._setup_failed_test_calls(1, error)
        try:
            self.test._call_run_once_with_retry([], False, None, (1, 2),
                                                {'arg': 'val'})
        except Exception as err:
            if err != error:
                raise
        self.god.check_playback()


    def test_call_run_once_with_retry_exception_and_pass(self):
        """
        Test call_run_once_with_retry duplicating a test that fails at first
        and later passes.
        """
        # Stubbed out for the write_keyval call.
        self.test.outputdir = '/tmp'

        num_to_fail = 2
        self.test.job.test_retry = 5
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.god.stub_function(self.test, 'run_once')
        before_hook = self.god.create_mock_function('before_hook')
        after_hook = self.god.create_mock_function('after_hook')
        self.god.stub_function(self.test, '_call_run_once')
        self.test.register_before_iteration_hook(before_hook)
        self.test.register_after_iteration_hook(after_hook)
        self.god.stub_function(self.test.job, 'record')
        # tests the test._call_run_once implementation
        error = common_lib_error.TestFailRetry('fail')
        self._setup_failed_test_calls(num_to_fail, error)
        # Passing call
        self.test._call_run_once.expect_call([], False, None, (1, 2),
                                             {'arg': 'val'})
        self.test._call_run_once_with_retry([], False, None, (1, 2),
                                            {'arg': 'val'})
        self.god.check_playback()


    def _expect_call_run_once(self):
        self.test._call_run_once.expect_call((), False, None, (), {})


    def test_execute_test_length(self):
        # test that test_length overrides iterations and works.
        self.god.stub_function(self.test, '_call_run_once')

        self._expect_call_run_once()
        self._expect_call_run_once()
        self._expect_call_run_once()
        self.test.run_once_profiling.expect_call(None)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()

        fake_time = iter(xrange(4)).next
        self.test.execute(iterations=1, test_length=3, _get_time=fake_time)
        self.god.check_playback()


    def test_execute_iterations(self):
        # test that iterations works.
        self.god.stub_function(self.test, '_call_run_once')

        iterations = 2
        for _ in range(iterations):
            self._expect_call_run_once()
        self.test.run_once_profiling.expect_call(None)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()

        self.test.execute(iterations=iterations)
        self.god.check_playback()


    def _mock_calls_for_execute_no_iterations(self):
        self.test.run_once_profiling.expect_call(None)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()


    def test_execute_iteration_zero(self):
        # test that iterations=0 works.
        self._mock_calls_for_execute_no_iterations()

        self.test.execute(iterations=0)
        self.god.check_playback()


    def test_execute_profile_only(self):
        # test that profile_only=True works.
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        self.test.drop_caches_between_iterations.expect_call()
        self.test.run_once_profiling.expect_call(None)
        self.test.drop_caches_between_iterations.expect_call()
        self.test.run_once_profiling.expect_call(None)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()
        self.test.execute(profile_only=True, iterations=2)
        self.god.check_playback()


    def test_execute_default_profile_only(self):
        # test that profile_only=True works.
        self.god.stub_function(self.test, 'drop_caches_between_iterations')
        for _ in xrange(3):
            self.test.drop_caches_between_iterations.expect_call()
            self.test.run_once_profiling.expect_call(None)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()
        self.test.job.default_profile_only = True
        self.test.execute(iterations=3)
        self.god.check_playback()


    def test_execute_postprocess_profiled_false(self):
        # test that postprocess_profiled_run=False works
        self.god.stub_function(self.test, '_call_run_once')

        self.test._call_run_once.expect_call((), False, False, (), {})
        self.test.run_once_profiling.expect_call(False)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()

        self.test.execute(postprocess_profiled_run=False, iterations=1)
        self.god.check_playback()


    def test_execute_postprocess_profiled_true(self):
        # test that postprocess_profiled_run=True works
        self.god.stub_function(self.test, '_call_run_once')

        self.test._call_run_once.expect_call((), False, True, (), {})
        self.test.run_once_profiling.expect_call(True)
        self.test.postprocess.expect_call()
        self.test.process_failed_constraints.expect_call()

        self.test.execute(postprocess_profiled_run=True, iterations=1)
        self.god.check_playback()


    def test_output_single_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", 1, units="ms", higher_is_better=True)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms", "type": "scalar",
                           "value": 1, "improvement_direction": "up"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_single_perf_value_twice(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", 1, units="ms", higher_is_better=True)
        self.test.output_perf_value("Test", 2, units="ms", higher_is_better=True)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values", "values": [1, 2],
                           "improvement_direction": "up"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_single_perf_value_three_times(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", 1, units="ms",
                                    higher_is_better=True)
        self.test.output_perf_value("Test", 2, units="ms", higher_is_better=True)
        self.test.output_perf_value("Test", 3, units="ms", higher_is_better=True)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values", "values": [1, 2, 3],
                           "improvement_direction": "up"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_list_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", [1, 2, 3], units="ms",
                                    higher_is_better=False)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values", "values": [1, 2, 3],
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_single_then_list_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", 1, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", [4, 3, 2], units="ms",
                                    higher_is_better=False)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values",
                           "values": [1, 4, 3, 2],
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_list_then_list_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", [1, 2, 3], units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", [4, 3, 2], units="ms",
                                    higher_is_better=False)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values",
                           "values": [1, 2, 3, 4, 3, 2],
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_single_perf_value_input_string(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", u'-0.34', units="ms",
                                    higher_is_better=True)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms", "type": "scalar",
                           "value": -0.34, "improvement_direction": "up"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))


    def test_output_single_perf_value_input_list_of_string(self):
        self.test.resultsdir = tempfile.mkdtemp()

        self.test.output_perf_value("Test", [0, u'-0.34', 1], units="ms",
                                    higher_is_better=True)

        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values",
                           "values": [0, -0.34, 1],
                           "improvement_direction": "up"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_output_list_then_replace_list_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", [1, 2, 3], units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", [4, 5, 6], units="ms",
                                    higher_is_better=False,
                                    replace_existing_values=True)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values",
                           "values": [4, 5, 6],
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_output_single_then_replace_list_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", 3, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", [4, 5, 6], units="ms",
                                    higher_is_better=False,
                                    replace_existing_values=True)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "list_of_scalar_values",
                           "values": [4, 5, 6],
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_output_list_then_replace_single_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", [1,2,3], units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", 4, units="ms",
                                    higher_is_better=False,
                                    replace_existing_values=True)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "scalar",
                           "value": 4,
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_output_single_then_replace_single_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test", 1, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test", 2, units="ms",
                                    higher_is_better=False,
                                    replace_existing_values=True)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test": {"summary": {"units": "ms",
                           "type": "scalar",
                           "value": 2,
                           "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_output_perf_then_replace_certain_perf_value(self):
        self.test.resultsdir = tempfile.mkdtemp()
        self.test.output_perf_value("Test1", 1, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test2", 2, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test3", 3, units="ms",
                                    higher_is_better=False)
        self.test.output_perf_value("Test2", -1, units="ms",
                                    higher_is_better=False,
                                    replace_existing_values=True)
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {"Test1": {"summary":
                                       {"units": "ms",
                                        "type": "scalar",
                                        "value": 1,
                                        "improvement_direction": "down"}},
                           "Test2": {"summary":
                                       {"units": "ms",
                                        "type": "scalar",
                                        "value": -1,
                                        "improvement_direction": "down"}},
                           "Test3": {"summary":
                                       {"units": "ms",
                                        "type": "scalar",
                                        "value": 3,
                                        "improvement_direction": "down"}}}
        self.assertDictEqual(expected_result, json.loads(f.read()))

    def test_chart_supplied(self):
        self.test.resultsdir = tempfile.mkdtemp()

        test_data = [("tcp_tx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0),
                     ("tcp_tx", "ch006_mode11B_none", "BT_streaming_audiofile", 5),
                     ("tcp_tx", "ch006_mode11B_none", "BT_disconnected_again", 0),
                     ("tcp_rx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0),
                     ("tcp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 8),
                     ("tcp_rx", "ch006_mode11B_none", "BT_disconnected_again", 0),
                     ("udp_tx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0),
                     ("udp_tx", "ch006_mode11B_none", "BT_streaming_audiofile", 6),
                     ("udp_tx", "ch006_mode11B_none", "BT_disconnected_again", 0),
                     ("udp_rx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0),
                     ("udp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 8),
                     ("udp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 9),
                     ("udp_rx", "ch006_mode11B_none", "BT_disconnected_again", 0)]


        for (config_tag, ap_config_tag, bt_tag, drop) in test_data:
          self.test.output_perf_value(config_tag + '_' + bt_tag + '_drop',
                                      drop, units='percent_drop',
                                      higher_is_better=False,
                                      graph=ap_config_tag + '_drop')
        f = open(self.test.resultsdir + "/results-chart.json")
        expected_result = {
          "ch006_mode11B_none_drop": {
            "udp_tx_BT_streaming_audiofile_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 6.0,
              "improvement_direction": "down"
            },
            "udp_rx_BT_disconnected_again_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "tcp_tx_BT_disconnected_again_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "tcp_rx_BT_streaming_audiofile_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 8.0,
              "improvement_direction": "down"
            },
            "udp_tx_BT_connected_but_not_streaming_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "tcp_tx_BT_connected_but_not_streaming_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "udp_tx_BT_disconnected_again_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "tcp_tx_BT_streaming_audiofile_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 5.0,
              "improvement_direction": "down"
            },
            "tcp_rx_BT_connected_but_not_streaming_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "udp_rx_BT_connected_but_not_streaming_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            },
            "udp_rx_BT_streaming_audiofile_drop": {
              "units": "percent_drop",
              "type": "list_of_scalar_values",
              "values": [
                8.0,
                9.0
              ],
              "improvement_direction": "down"
            },
            "tcp_rx_BT_disconnected_again_drop": {
              "units": "percent_drop",
              "type": "scalar",
              "value": 0.0,
              "improvement_direction": "down"
            }
          }
        }
        self.maxDiff = None
        self.assertDictEqual(expected_result, json.loads(f.read()))

if __name__ == '__main__':
    unittest.main()