Add functionality for test classes to be specified using wildcards.
Bug: 29458336
Test: act.py -c <config> -tc SampleT*
Test: act.py -c <config> -tc Ble*Api*
Test: act.py -c <config> -tc ?ampleT[aeiou]st
Test: Preupload tests
Change-Id: I56bd57c4dbc6f5e1ce0a9c03bd0bfce6ba66c387
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg
index 6d1b108..8c9f7ea 100644
--- a/PREUPLOAD.cfg
+++ b/PREUPLOAD.cfg
@@ -16,6 +16,7 @@
acts_import_test_utils_test = ./acts/framework/tests/acts_import_test_utils_test.py
acts_import_unit_test = ./acts/framework/tests/acts_import_unit_test.py
acts_relay_controller_test = ./acts/framework/tests/acts_relay_controller_test.py
+test_runner_test = ./acts/framework/tests/test_runner_test.py
commit_message_hook = ./tools/commit_message_check.py
yapf_hook = ./tools/yapf_checker.py
lab_test = ./tools/lab/lab_upload_hooks.py
diff --git a/acts/framework/acts/config_parser.py b/acts/framework/acts/config_parser.py
index a18ba3f..8729418 100755
--- a/acts/framework/acts/config_parser.py
+++ b/acts/framework/acts/config_parser.py
@@ -142,13 +142,6 @@
seen_names.add(name)
-def _verify_test_class_name(test_cls_name):
- if not test_cls_name.endswith("Test"):
- raise ActsConfigError(
- ("Requested test class '%s' does not follow the test class naming "
- "convention *Test.") % test_cls_name)
-
-
def gen_term_signal_handler(test_runners):
def termination_sig_handler(signal_num, frame):
for t in test_runners:
@@ -180,14 +173,12 @@
if len(tokens) == 1:
# This should be considered a test class name
test_cls_name = tokens[0]
- _verify_test_class_name(test_cls_name)
return test_cls_name, None
elif len(tokens) == 2:
# This should be considered a test class name followed by
# a list of test case names.
test_cls_name, test_case_names = tokens
clean_names = []
- _verify_test_class_name(test_cls_name)
for elem in test_case_names.split(','):
test_case_name = elem.strip()
if not test_case_name.startswith("test_"):
@@ -299,13 +290,13 @@
len(tbs)))
configs[keys.Config.key_testbed.value] = tbs
- if (keys.Config.key_log_path.value not in configs and
- _ENV_ACTS_LOGPATH in os.environ):
+ if (keys.Config.key_log_path.value not in configs
+ and _ENV_ACTS_LOGPATH in os.environ):
print('Using environment log path: %s' %
(os.environ[_ENV_ACTS_LOGPATH]))
configs[keys.Config.key_log_path.value] = os.environ[_ENV_ACTS_LOGPATH]
- if (keys.Config.key_test_paths.value not in configs and
- _ENV_ACTS_TESTPATHS in os.environ):
+ if (keys.Config.key_test_paths.value not in configs
+ and _ENV_ACTS_TESTPATHS in os.environ):
print('Using environment test paths: %s' %
(os.environ[_ENV_ACTS_TESTPATHS]))
configs[keys.Config.key_test_paths.value] = os.environ[
diff --git a/acts/framework/acts/test_runner.py b/acts/framework/acts/test_runner.py
index f27f1f3..501955b 100644
--- a/acts/framework/acts/test_runner.py
+++ b/acts/framework/acts/test_runner.py
@@ -21,6 +21,7 @@
import copy
import importlib
import inspect
+import fnmatch
import logging
import os
import pkgutil
@@ -176,7 +177,8 @@
self.controller_destructors: A dictionary that holds the controller
distructors. Keys are controllers' names.
self.test_classes: A dictionary where we can look up the test classes
- by name to instantiate.
+ by name to instantiate. Supports unix shell style
+ wildcards.
self.run_list: A list of tuples specifying what tests to run.
self.results: The test result object used to record the results of
this test run.
@@ -512,31 +514,38 @@
ValueError is raised if the requested test class could not be found
in the test_paths directories.
"""
- try:
- test_cls = self.test_classes[test_cls_name]
- except KeyError:
+ matches = fnmatch.filter(self.test_classes.keys(), test_cls_name)
+ if not matches:
self.log.info(
- "Cannot find test class %s skipping for now." % test_cls_name)
+ "Cannot find test class %s or classes matching pattern,"
+ "skipping for now." % test_cls_name)
record = records.TestResultRecord("*all*", test_cls_name)
record.test_skip(signals.TestSkip("Test class does not exist."))
self.results.add_record(record)
return
- if self.test_configs.get(keys.Config.key_random.value) or (
- "Preflight" in test_cls_name) or "Postflight" in test_cls_name:
- test_case_iterations = 1
- else:
- test_case_iterations = self.test_configs.get(
- keys.Config.key_test_case_iterations.value, 1)
+ if matches != [test_cls_name]:
+ self.log.info("Found classes matching pattern %s: %s",
+ test_cls_name, matches)
- with test_cls(self.test_run_info) as test_cls_instance:
- try:
- cls_result = test_cls_instance.run(test_cases,
- test_case_iterations)
- self.results += cls_result
- self._write_results_json_str()
- except signals.TestAbortAll as e:
- self.results += e.results
- raise e
+ for test_cls_name_match in matches:
+ test_cls = self.test_classes[test_cls_name_match]
+ if self.test_configs.get(keys.Config.key_random.value) or (
+ "Preflight" in test_cls_name_match) or (
+ "Postflight" in test_cls_name_match):
+ test_case_iterations = 1
+ else:
+ test_case_iterations = self.test_configs.get(
+ keys.Config.key_test_case_iterations.value, 1)
+
+ with test_cls(self.test_run_info) as test_cls_instance:
+ try:
+ cls_result = test_cls_instance.run(test_cases,
+ test_case_iterations)
+ self.results += cls_result
+ self._write_results_json_str()
+ except signals.TestAbortAll as e:
+ self.results += e.results
+ raise e
def run(self, test_class=None):
"""Executes test cases.
@@ -567,6 +576,7 @@
for test_cls_name, test_case_names in self.run_list:
if not self.running:
break
+
if test_case_names:
self.log.debug("Executing test cases %s in test class %s.",
test_case_names, test_cls_name)
diff --git a/acts/framework/tests/test_runner_test.py b/acts/framework/tests/test_runner_test.py
new file mode 100755
index 0000000..546ac37
--- /dev/null
+++ b/acts/framework/tests/test_runner_test.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3.4
+#
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import Mock
+import unittest
+import tempfile
+
+from acts import keys
+from acts import test_runner
+
+import mock_controller
+
+
+class TestRunnerTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp_dir = tempfile.mkdtemp()
+ self.base_mock_test_config = {
+ "testbed": {
+ "name": "SampleTestBed",
+ },
+ "logpath": self.tmp_dir,
+ "cli_args": None,
+ "testpaths": ["./"],
+ "icecream": 42,
+ "extra_param": "haha"
+ }
+
+ def create_mock_context(self):
+ context = Mock()
+ context.__exit__ = Mock()
+ context.__enter__ = Mock()
+ return context
+
+ def create_test_classes(self, class_names):
+ return {
+ class_name: Mock(return_value=self.create_mock_context())
+ for class_name in class_names
+ }
+
+ def test_class_name_pattern_single(self):
+ class_names = ['test_class_1', 'test_class_2']
+ pattern = 'test*1'
+ tr = test_runner.TestRunner(self.base_mock_test_config, [(pattern,
+ None)])
+
+ test_classes = self.create_test_classes(class_names)
+ tr.import_test_modules = Mock(return_value=test_classes)
+ tr.run()
+ self.assertTrue(test_classes[class_names[0]].called)
+ self.assertFalse(test_classes[class_names[1]].called)
+
+ def test_class_name_pattern_multi(self):
+ class_names = ['test_class_1', 'test_class_2', 'other_name']
+ pattern = 'test_class*'
+ tr = test_runner.TestRunner(self.base_mock_test_config, [(pattern,
+ None)])
+
+ test_classes = self.create_test_classes(class_names)
+ tr.import_test_modules = Mock(return_value=test_classes)
+ tr.run()
+ self.assertTrue(test_classes[class_names[0]].called)
+ self.assertTrue(test_classes[class_names[1]].called)
+ self.assertFalse(test_classes[class_names[2]].called)
+
+ def test_class_name_pattern_question_mark(self):
+ class_names = ['test_class_1', 'test_class_12']
+ pattern = 'test_class_?'
+ tr = test_runner.TestRunner(self.base_mock_test_config, [(pattern,
+ None)])
+
+ test_classes = self.create_test_classes(class_names)
+ tr.import_test_modules = Mock(return_value=test_classes)
+ tr.run()
+ self.assertTrue(test_classes[class_names[0]].called)
+ self.assertFalse(test_classes[class_names[1]].called)
+
+ def test_class_name_pattern_char_seq(self):
+ class_names = ['test_class_1', 'test_class_2', 'test_class_3']
+ pattern = 'test_class_[1357]'
+ tr = test_runner.TestRunner(self.base_mock_test_config, [(pattern,
+ None)])
+
+ test_classes = self.create_test_classes(class_names)
+ tr.import_test_modules = Mock(return_value=test_classes)
+ tr.run()
+ self.assertTrue(test_classes[class_names[0]].called)
+ self.assertFalse(test_classes[class_names[1]].called)
+ self.assertTrue(test_classes[class_names[2]].called)
+
+
+if __name__ == "__main__":
+ unittest.main()