| #!/usr/bin/env python3 |
| # |
| # Copyright 2016 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import mock |
| import unittest |
| |
| from acts import asserts |
| from acts import base_test |
| from acts import signals |
| from acts import test_runner |
| |
| MSG_EXPECTED_EXCEPTION = "This is an expected exception." |
| MSG_EXPECTED_TEST_FAILURE = "This is an expected test failure." |
| MSG_UNEXPECTED_EXCEPTION = "Unexpected exception!" |
| |
| MOCK_EXTRA = {"key": "value", "answer_to_everything": 42} |
| |
| |
| def never_call(): |
| raise Exception(MSG_UNEXPECTED_EXCEPTION) |
| |
| |
| class SomeError(Exception): |
| """A custom exception class used for tests in this module.""" |
| |
| |
| class ActsBaseClassTest(unittest.TestCase): |
| def setUp(self): |
| self.mock_test_cls_configs = { |
| 'reporter': mock.MagicMock(), |
| 'log': mock.MagicMock(), |
| 'log_path': '/tmp', |
| 'cli_args': None, |
| 'user_params': { |
| "some_param": "hahaha" |
| } |
| } |
| self.mock_test_name = "test_something" |
| |
| def test_current_test_case_name(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.assert_true( |
| self.current_test_name == "test_func", |
| ("Got " |
| "unexpected test name %s.") % self.current_test_name) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_self_tests_list(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def __init__(self, controllers): |
| super(MockBaseTest, self).__init__(controllers) |
| self.tests = ("test_something", ) |
| |
| def test_something(self): |
| pass |
| |
| def test_never(self): |
| # This should not execute it's not on default test list. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_something") |
| |
| def test_self_tests_list_fail_by_convention(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def __init__(self, controllers): |
| super(MockBaseTest, self).__init__(controllers) |
| self.tests = ("not_a_test_something", ) |
| |
| def not_a_test_something(self): |
| pass |
| |
| def test_never(self): |
| # This should not execute it's not on default test list. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| expected_msg = ("Test case name not_a_test_something does not follow " |
| "naming convention test_\*, abort.") |
| with self.assertRaisesRegexp(base_test.Error, expected_msg): |
| bt_cls.run() |
| |
| def test_cli_test_selection_override_self_tests_list(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def __init__(self, controllers): |
| super(MockBaseTest, self).__init__(controllers) |
| self.tests = ("test_never", ) |
| |
| def test_something(self): |
| pass |
| |
| def test_never(self): |
| # This should not execute it's not selected by cmd line input. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_something") |
| |
| def test_cli_test_selection_fail_by_convention(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def __init__(self, controllers): |
| super(MockBaseTest, self).__init__(controllers) |
| self.tests = ("not_a_test_something", ) |
| |
| def not_a_test_something(self): |
| pass |
| |
| def test_never(self): |
| # This should not execute it's not selected by cmd line input. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| expected_msg = ("Test case name not_a_test_something does not follow " |
| "naming convention test_*, abort.") |
| with self.assertRaises(base_test.Error, msg=expected_msg): |
| bt_cls.run(test_names=["not_a_test_something"]) |
| |
| def test_default_execution_of_all_tests(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_something(self): |
| pass |
| |
| def not_a_test(self): |
| # This should not execute its name doesn't follow test case |
| # naming convention. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_something") |
| |
| def test_missing_requested_test_func(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| self.assertFalse(bt_cls.results.executed) |
| self.assertTrue(bt_cls.results.skipped) |
| |
| def test_setup_class_fail_by_exception(self): |
| call_check = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_class(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| # This should not execute because setup_class failed. |
| never_call() |
| |
| def on_blocked(self, test_name, begin_time): |
| call_check("haha") |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.blocked[0] |
| self.assertEqual(actual_record.test_name, "test_something") |
| expected_summary = ( |
| "Blocked 1, ControllerInfo {}, Executed 0, Failed 0, Passed 0," |
| " Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| call_check.assert_called_once_with("haha") |
| |
| def test_setup_test_fail_by_exception(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| # This should not execute because setup_test failed. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_setup_test_fail_by_test_signal(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| raise signals.TestFailure(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| # This should not execute because setup_test failed. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_setup_test_fail_by_return_False(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| return False |
| |
| def test_something(self): |
| # This should not execute because setup_test failed. |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_something"]) |
| actual_record = bt_cls.results.failed[0] |
| expected_msg = "Setup for %s failed." % self.mock_test_name |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, expected_msg) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_teardown_test_assert_fail(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_teardown_test_raise_exception(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| expected_extra_error = {"teardown_test": MSG_EXPECTED_EXCEPTION} |
| self.assertEqual(actual_record.additional_errors, expected_extra_error) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_teardown_test_executed_if_test_pass(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| my_mock("teardown_test") |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.passed[0] |
| my_mock.assert_called_once_with("teardown_test") |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 1, " |
| "Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_teardown_test_executed_if_setup_test_fails(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def teardown_test(self): |
| my_mock("teardown_test") |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| my_mock.assert_called_once_with("teardown_test") |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_teardown_test_executed_if_test_fails(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| my_mock("teardown_test") |
| |
| def test_something(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| my_mock.assert_called_once_with("teardown_test") |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_exception_executed_if_teardown_test_fails(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_exception(self, test_name, begin_time): |
| my_mock("on_exception") |
| |
| def teardown_test(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| my_mock.assert_called_once_with("on_exception") |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_fail_executed_if_test_fails(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_fail(self, test_name, begin_time): |
| my_mock("on_fail") |
| |
| def test_something(self): |
| asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| my_mock.assert_called_once_with("on_fail") |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_fail_executed_if_test_setup_fails_by_exception(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def on_fail(self, test_name, begin_time): |
| my_mock("on_fail") |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| my_mock.assert_called_once_with("on_fail") |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_fail_executed_if_test_setup_fails_by_return_False(self): |
| my_mock = mock.MagicMock() |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def setup_test(self): |
| return False |
| |
| def on_fail(self, test_name, begin_time): |
| my_mock("on_fail") |
| |
| def test_something(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| my_mock.assert_called_once_with("on_fail") |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, |
| 'Setup for test_something failed.') |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 0") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_failure_to_call_procedure_function_is_recorded(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_fail(self): |
| pass |
| |
| def test_something(self): |
| asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertIn('_on_fail', actual_record.additional_errors) |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_failure_in_procedure_functions_is_recorded(self): |
| expected_msg = "Something failed in on_pass." |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_pass(self, test_name, begin_time): |
| raise Exception(expected_msg) |
| |
| def test_something(self): |
| asserts.explicit_pass(MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| expected_extra_error = {'_on_pass': expected_msg} |
| self.assertEqual(actual_record.additional_errors, expected_extra_error) |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_both_teardown_and_test_body_raise_exceptions(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| raise Exception("Test Body Exception.") |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, "Test Body Exception.") |
| self.assertIsNone(actual_record.extras) |
| self.assertEqual(actual_record.additional_errors["teardown_test"], |
| "Details=This is an expected exception., Extras=None") |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_explicit_pass_but_teardown_test_raises_an_exception(self): |
| """Test record result should be marked as UNKNOWN as opposed to PASS. |
| """ |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def teardown_test(self): |
| asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| asserts.explicit_pass("Test Passed!") |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, "Test Passed!") |
| self.assertIsNone(actual_record.extras) |
| self.assertEqual(actual_record.additional_errors["teardown_test"], |
| "Details=This is an expected exception., Extras=None") |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_pass_raise_exception(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_pass(self, test_name, begin_time): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| asserts.explicit_pass( |
| MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| self.assertEqual(actual_record.additional_errors, { |
| '_on_pass': MSG_EXPECTED_EXCEPTION |
| }) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_on_fail_raise_exception(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def on_fail(self, test_name, begin_time): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| |
| def test_something(self): |
| asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(bt_cls.results.failed, []) |
| self.assertEqual(actual_record.test_name, self.mock_test_name) |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| self.assertEqual(actual_record.additional_errors, { |
| '_on_fail': MSG_EXPECTED_EXCEPTION |
| }) |
| expected_summary = ( |
| "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " |
| "Requested 1, Skipped 0, Unknown 1") |
| self.assertEqual(bt_cls.results.summary_str(), expected_summary) |
| |
| def test_abort_class(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_1(self): |
| pass |
| |
| def test_2(self): |
| asserts.abort_class(MSG_EXPECTED_EXCEPTION) |
| never_call() |
| |
| def test_3(self): |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_1", "test_2", "test_3"]) |
| self.assertEqual(bt_cls.results.passed[0].test_name, "test_1") |
| self.assertEqual(bt_cls.results.failed[0].details, |
| MSG_EXPECTED_EXCEPTION) |
| self.assertEqual( |
| bt_cls.results.summary_str(), |
| ("Blocked 0, ControllerInfo {}, Executed 2, Failed 1, Passed 1, " |
| "Requested 3, Skipped 0, Unknown 0")) |
| |
| def test_uncaught_exception(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| raise Exception(MSG_EXPECTED_EXCEPTION) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_fail(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_true(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.assert_true( |
| False, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_equal_pass(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.assert_equal(1, 1, extras=MOCK_EXTRA) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_assert_equal_fail(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.assert_equal(1, 2, extras=MOCK_EXTRA) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, "1 != 2") |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_equal_fail_with_msg(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.assert_equal( |
| 1, 2, msg=MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| expected_msg = "1 != 2 " + MSG_EXPECTED_EXCEPTION |
| self.assertEqual(actual_record.details, expected_msg) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_raises_pass(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): |
| raise SomeError(MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_assert_raises_fail_with_noop(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, "SomeError not raised") |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_raises_fail_with_wrong_error(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): |
| raise AttributeError(MSG_UNEXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_assert_raises_regex_pass(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises_regex( |
| SomeError, |
| expected_regex=MSG_EXPECTED_EXCEPTION, |
| extras=MOCK_EXTRA): |
| raise SomeError(MSG_EXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_assert_raises_fail_with_noop(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises_regex( |
| SomeError, |
| expected_regex=MSG_EXPECTED_EXCEPTION, |
| extras=MOCK_EXTRA): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, "SomeError not raised") |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_raises_fail_with_wrong_regex(self): |
| wrong_msg = "ha" |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises_regex( |
| SomeError, |
| expected_regex=MSG_EXPECTED_EXCEPTION, |
| extras=MOCK_EXTRA): |
| raise SomeError(wrong_msg) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.failed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| expected_details = ('"This is an expected exception." does not match ' |
| '"%s"') % wrong_msg |
| self.assertEqual(actual_record.details, expected_details) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_assert_raises_fail_with_wrong_error(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| with asserts.assert_raises_regex( |
| SomeError, |
| expected_regex=MSG_EXPECTED_EXCEPTION, |
| extras=MOCK_EXTRA): |
| raise AttributeError(MSG_UNEXPECTED_EXCEPTION) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run() |
| actual_record = bt_cls.results.unknown[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_explicit_pass(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.explicit_pass( |
| MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_implicit_pass(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| pass |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.passed[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertIsNone(actual_record.details) |
| self.assertIsNone(actual_record.extras) |
| |
| def test_skip(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.skipped[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_skip_if(self): |
| class MockBaseTest(base_test.BaseTestClass): |
| def test_func(self): |
| asserts.skip_if(False, MSG_UNEXPECTED_EXCEPTION) |
| asserts.skip_if( |
| True, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| never_call() |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| actual_record = bt_cls.results.skipped[0] |
| self.assertEqual(actual_record.test_name, "test_func") |
| self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(actual_record.extras, MOCK_EXTRA) |
| |
| def test_unpack_userparams_required(self): |
| """Missing a required param should raise an error.""" |
| required = ["some_param"] |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(required) |
| expected_value = self.mock_test_cls_configs["user_params"][ |
| "some_param"] |
| self.assertEqual(bc.some_param, expected_value) |
| |
| def test_unpack_userparams_required_missing(self): |
| """Missing a required param should raise an error.""" |
| required = ["something"] |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| expected_msg = ("Missing required user param '%s' in test " |
| "configuration.") % required[0] |
| with self.assertRaises(base_test.Error, msg=expected_msg): |
| bc.unpack_userparams(required) |
| |
| def test_unpack_userparams_optional(self): |
| """If an optional param is specified, the value should be what's in the |
| config. |
| """ |
| opt = ["some_param"] |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(opt_param_names=opt) |
| expected_value = self.mock_test_cls_configs["user_params"][ |
| "some_param"] |
| self.assertEqual(bc.some_param, expected_value) |
| |
| def test_unpack_userparams_optional_with_default(self): |
| """If an optional param is specified with a default value, and the |
| param is not in the config, the value should be the default value. |
| """ |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(optional_thing="whatever") |
| self.assertEqual(bc.optional_thing, "whatever") |
| |
| def test_unpack_userparams_default_overwrite_by_optional_param_list(self): |
| """If an optional param is specified in kwargs, and the param is in the |
| config, the value should be the one in the config. |
| """ |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(some_param="whatever") |
| expected_value = self.mock_test_cls_configs["user_params"][ |
| "some_param"] |
| self.assertEqual(bc.some_param, expected_value) |
| |
| def test_unpack_userparams_default_overwrite_by_required_param_list(self): |
| """If an optional param is specified in kwargs, the param is in the |
| required param list, and the param is not specified in the config, the |
| param's alue should be the default value and there should be no error |
| thrown. |
| """ |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams( |
| req_param_names=['a_kwarg_param'], a_kwarg_param="whatever") |
| self.assertEqual(bc.a_kwarg_param, "whatever") |
| |
| def test_unpack_userparams_optional_missing(self): |
| """Missing an optional param should not raise an error.""" |
| opt = ["something"] |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(opt_param_names=opt) |
| |
| def test_unpack_userparams_basic(self): |
| """Required and optional params are unpacked properly.""" |
| required = ["something"] |
| optional = ["something_else"] |
| configs = dict(self.mock_test_cls_configs) |
| configs["user_params"]["something"] = 42 |
| configs["user_params"]["something_else"] = 53 |
| bc = base_test.BaseTestClass(configs) |
| bc.unpack_userparams( |
| req_param_names=required, opt_param_names=optional) |
| self.assertEqual(bc.something, 42) |
| self.assertEqual(bc.something_else, 53) |
| |
| def test_unpack_userparams_default_overwrite(self): |
| default_arg_val = "haha" |
| actual_arg_val = "wawa" |
| arg_name = "arg1" |
| configs = dict(self.mock_test_cls_configs) |
| configs["user_params"][arg_name] = actual_arg_val |
| bc = base_test.BaseTestClass(configs) |
| bc.unpack_userparams(opt_param_names=[arg_name], arg1=default_arg_val) |
| self.assertEqual(bc.arg1, actual_arg_val) |
| |
| def test_unpack_userparams_default_None(self): |
| bc = base_test.BaseTestClass(self.mock_test_cls_configs) |
| bc.unpack_userparams(arg1="haha") |
| self.assertEqual(bc.arg1, "haha") |
| |
| def test_generated_tests(self): |
| """Execute code paths for generated test cases. |
| |
| Three test cases are generated, each of them produces a different |
| result: one pass, one fail, and one skip. |
| |
| This test verifies that the exact three tests are executed and their |
| results are reported correctly. |
| """ |
| static_arg = "haha" |
| static_kwarg = "meh" |
| itrs = ["pass", "fail", "skip"] |
| |
| class MockBaseTest(base_test.BaseTestClass): |
| def name_gen(self, setting, arg, special_arg=None): |
| return "test_%s_%s" % (setting, arg) |
| |
| def logic(self, setting, arg, special_arg=None): |
| asserts.assert_true( |
| setting in itrs, |
| ("%s is not in acceptable settings range %s") % (setting, |
| itrs)) |
| asserts.assert_true(arg == static_arg, |
| "Expected %s, got %s" % (static_arg, arg)) |
| asserts.assert_true(arg == static_arg, "Expected %s, got %s" % |
| (static_kwarg, special_arg)) |
| if setting == "pass": |
| asserts.explicit_pass( |
| MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| elif setting == "fail": |
| asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| elif setting == "skip": |
| asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) |
| |
| @signals.generated_test |
| def test_func(self): |
| self.run_generated_testcases( |
| test_func=self.logic, |
| settings=itrs, |
| args=(static_arg, ), |
| name_func=self.name_gen) |
| |
| bt_cls = MockBaseTest(self.mock_test_cls_configs) |
| bt_cls.run(test_names=["test_func"]) |
| self.assertEqual(len(bt_cls.results.requested), 3) |
| pass_record = bt_cls.results.passed[0] |
| self.assertEqual(pass_record.test_name, "test_pass_%s" % static_arg) |
| self.assertEqual(pass_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(pass_record.extras, MOCK_EXTRA) |
| skip_record = bt_cls.results.skipped[0] |
| self.assertEqual(skip_record.test_name, "test_skip_%s" % static_arg) |
| self.assertEqual(skip_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(skip_record.extras, MOCK_EXTRA) |
| fail_record = bt_cls.results.failed[0] |
| self.assertEqual(fail_record.test_name, "test_fail_%s" % static_arg) |
| self.assertEqual(fail_record.details, MSG_EXPECTED_EXCEPTION) |
| self.assertEqual(fail_record.extras, MOCK_EXTRA) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |