Merge from Chromium at DEPS revision r216972
This commit was generated by merge_to_master.py.
Change-Id: Ie5904a921ece9c5959b52c8e0b74db09fa08f144
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
index 59b0f9b..5fef700 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
@@ -213,7 +213,7 @@
initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
int(self._options.child_processes), retrying=False)
- tests_to_retry = self._tests_to_retry(initial_results, include_crashes=self._port.should_retry_crashes())
+ tests_to_retry = self._tests_to_retry(initial_results)
if should_retry_failures and tests_to_retry and not initial_results.interrupted:
enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed()
@@ -242,7 +242,7 @@
summarized_full_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry)
summarized_failing_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, only_include_failing=True)
- exit_code = self._port.exit_code_from_summarized_results(summarized_failing_results)
+ exit_code = summarized_failing_results['num_regressions']
if not self._options.dry_run:
self._write_json_files(summarized_full_results, summarized_failing_results, initial_results)
self._upload_json_files()
@@ -347,11 +347,8 @@
if self._filesystem.isdir(self._filesystem.join(layout_tests_dir, dirname)):
self._filesystem.rmtree(self._filesystem.join(self._results_directory, dirname))
- def _tests_to_retry(self, run_results, include_crashes):
- return [result.test_name for result in run_results.unexpected_results_by_name.values() if
- ((result.type != test_expectations.PASS) and
- (result.type != test_expectations.MISSING) and
- (result.type != test_expectations.CRASH or include_crashes))]
+ def _tests_to_retry(self, run_results):
+ return [result.test_name for result in run_results.unexpected_results_by_name.values() if result.type != test_expectations.PASS]
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results):
_log.debug("Writing JSON files in %s." % self._results_directory)
diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py b/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py
index 7293af0..3579d23 100644
--- a/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py
+++ b/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py
@@ -130,7 +130,6 @@
'skipped': The number of skipped tests (NOW & SKIPPED)
'num_regressions': The number of non-flaky failures
'num_flaky': The number of flaky failures
- 'num_missing': The number of tests with missing results
'num_passes': The number of unexpected passes
'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
"""
@@ -145,7 +144,6 @@
num_passes = 0
num_flaky = 0
- num_missing = 0
num_regressions = 0
keywords = {}
for expecation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.iteritems():
@@ -163,9 +161,6 @@
tests = {}
for test_name, result in initial_results.results_by_name.iteritems():
- # Note that if a test crashed in the original run, we ignore
- # whether or not it crashed when we retried it (if we retried it),
- # and always consider the result not flaky.
expected = expectations.get_expectations_string(test_name)
result_type = result.type
actual = [keywords[result_type]]
@@ -177,12 +172,6 @@
num_passes += 1
if not result.has_stderr and only_include_failing:
continue
- elif result_type == test_expectations.CRASH:
- if test_name in initial_results.unexpected_results_by_name:
- num_regressions += 1
- elif result_type == test_expectations.MISSING:
- if test_name in initial_results.unexpected_results_by_name:
- num_missing += 1
elif result_type != test_expectations.SKIP and test_name in initial_results.unexpected_results_by_name:
if retry_results and test_name not in retry_results.unexpected_results_by_name:
actual.extend(expectations.get_expectations_string(test_name).split(" "))
@@ -259,7 +248,6 @@
results['num_passes'] = num_passes
results['num_flaky'] = num_flaky
# FIXME: Remove this. It is redundant with results['num_failures_by_type'].
- results['num_missing'] = num_missing
results['num_regressions'] = num_regressions
results['interrupted'] = initial_results.interrupted # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?)
results['layout_tests_dir'] = port_obj.layout_tests_dir()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py
index f8a6fac..5ab1250 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py
@@ -188,9 +188,6 @@
self._pretty_patch_available = self.check_pretty_patch(logging=False)
return self._pretty_patch_available
- def should_retry_crashes(self):
- return self.get_option('retry_crashes', False)
-
def default_child_processes(self):
"""Return the number of drivers to use for this port."""
return self._executive.cpu_count()
@@ -985,13 +982,6 @@
self._websocket_server.stop()
self._websocket_server = None
- def exit_code_from_summarized_results(self, unexpected_results):
- """Given summarized results, compute the exit code to be returned by new-run-webkit-tests.
- Bots turn red when this function returns a non-zero value. By default, return the number of regressions
- to avoid turning bots red by flaky failures, unexpected passes, and missing results"""
- # Don't turn bots red for flaky failures, unexpected passes, and missing results.
- return unexpected_results['num_regressions']
-
#
# TEST EXPECTATION-RELATED METHODS
#
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
index 6b99360..55ff290 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
@@ -257,11 +257,6 @@
finally:
self._helper = None
-
- def exit_code_from_summarized_results(self, unexpected_results):
- # Turn bots red for missing results.
- return unexpected_results['num_regressions'] + unexpected_results['num_missing']
-
def configuration_specifier_macros(self):
return self.CONFIGURATION_SPECIFIER_MACROS
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py
index df2c024..8871c87 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/test.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py
@@ -103,10 +103,9 @@
#
TOTAL_TESTS = 106
TOTAL_SKIPS = 27
-TOTAL_RETRIES = 14
-UNEXPECTED_PASSES = 6
-UNEXPECTED_FAILURES = 17
+UNEXPECTED_PASSES = 1
+UNEXPECTED_FAILURES = 22
def unit_test_list():
tests = TestList()
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index b349fc5..0d8578a 100644
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -235,9 +235,12 @@
optparse.make_option("--no-retry-failures", action="store_false",
dest="retry_failures",
help="Don't re-try any tests that produce unexpected results."),
+
+ # FIXME: Remove this after we remove the flag from the v8 bot.
optparse.make_option("--retry-crashes", action="store_true",
default=False,
- help="Do also retry crashes if retry-failures is enabled."),
+ help="ignored (we now always retry crashes when we retry failures)."),
+
optparse.make_option("--max-locked-shards", type="int", default=0,
help="Set the maximum number of locked shards"),
optparse.make_option("--additional-env-var", type="string", action="append", default=[],
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
index 78baf7c..54b928d 100644
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
@@ -199,7 +199,7 @@
self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
- self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES)
+ self.assertEqual(details.retry_results.total, test.UNEXPECTED_FAILURES)
expected_tests = details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name)
expected_summary_str = ''
@@ -492,13 +492,12 @@
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host)
file_list = host.filesystem.written_files.keys()
- self.assertEqual(details.exit_code, 1)
+ self.assertEqual(details.exit_code, 2)
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find('"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","is_unexpected":true') != -1)
self.assertTrue(json_string.find('"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_unexpected":true') != -1)
- self.assertTrue(json_string.find('"num_regressions":1') != -1)
+ self.assertTrue(json_string.find('"num_regressions":2') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
- self.assertTrue(json_string.find('"num_missing":1') != -1)
def test_pixel_test_directories(self):
host = MockHost()
@@ -514,23 +513,6 @@
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
- def test_missing_and_unexpected_results_with_custom_exit_code(self):
- # Test that we update expectations in place. If the expectation
- # is missing, update the expected generic location.
- class CustomExitCodePort(test.TestPort):
- def exit_code_from_summarized_results(self, unexpected_results):
- return unexpected_results['num_regressions'] + unexpected_results['num_missing']
-
- host = MockHost()
- options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
- test_port = CustomExitCodePort(host, options=options)
- details, err, _ = logging_run(['--no-show-results',
- 'failures/expected/missing_image.html',
- 'failures/unexpected/missing_text.html',
- 'failures/unexpected/text-image-checksum.html'],
- tests_included=True, host=host, port_obj=test_port)
- self.assertEqual(details.exit_code, 2)
-
def test_crash_with_stderr(self):
host = MockHost()
_, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host)
@@ -619,7 +601,7 @@
host = MockHost()
details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected'], tests_included=True, host=host)
- self.assertEqual(details.exit_code, 12)
+ self.assertEqual(details.exit_code, 16)
self.assertTrue('Retrying' in err.getvalue())
def test_retrying_default_value_test_list(self):
@@ -634,7 +616,7 @@
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'failures')
details, err, _ = logging_run(['--debug-rwt-logging', '--test-list=%s' % filename], tests_included=True, host=host)
- self.assertEqual(details.exit_code, 12)
+ self.assertEqual(details.exit_code, 16)
self.assertTrue('Retrying' in err.getvalue())
def test_retrying_and_flaky_tests(self):
@@ -657,21 +639,12 @@
self.assertFalse(host.filesystem.exists('retries'))
self.assertEqual(len(host.user.opened_urls), 1)
- def test_retrying_chrashed_tests(self):
+ def test_retrying_crashed_tests(self):
host = MockHost()
- details, err, _ = logging_run(['--retry-failures', '--retry-crashes', 'failures/unexpected/crash.html'], tests_included=True, host=host)
+ details, err, _ = logging_run(['--retry-failures', 'failures/unexpected/crash.html'], tests_included=True, host=host)
self.assertEqual(details.exit_code, 1)
self.assertTrue('Retrying' in err.getvalue())
- # Now we test that --clobber-old-results does remove the old entries and the old retries,
- # and that we don't retry again.
- host = MockHost()
- details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/unexpected/crash.html'], tests_included=True, host=host)
- self.assertEqual(details.exit_code, 1)
- self.assertTrue('Clobbering old results' in err.getvalue())
- self.assertTrue('unexpected/crash.html' in err.getvalue())
- self.assertFalse(host.filesystem.exists('retries'))
-
def test_retrying_force_pixel_tests(self):
host = MockHost()
details, err, _ = logging_run(['--no-pixel-tests', '--retry-failures', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
@@ -732,11 +705,11 @@
def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
host = MockHost()
_, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
- json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
- self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"is_missing_image":true,"actual":"MISSING","is_unexpected":true') != -1)
- self.assertTrue(json_string.find('"num_regressions":4') != -1)
- self.assertTrue(json_string.find('"num_flaky":0') != -1)
- self.assertTrue(json_string.find('"num_missing":1') != -1)
+ results = parse_full_results(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json'))
+
+ self.assertEqual(results["tests"]["reftests"]["foo"]["unlistedtest.html"]["actual"], "MISSING"),
+ self.assertEqual(results["num_regressions"], 5)
+ self.assertEqual(results["num_flaky"], 0)
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
@@ -871,11 +844,10 @@
details, err, _ = logging_run(['--no-show-results',
'failures/unexpected/missing_text.html',
'failures/unexpected/missing_image.html',
- 'failures/unexpected/missing_audio.html',
'failures/unexpected/missing_render_tree_dump.html'],
tests_included=True, host=host, new_results=True)
file_list = host.filesystem.written_files.keys()
- self.assertEqual(details.exit_code, 0)
+ self.assertEqual(details.exit_code, 3)
self.assertEqual(len(file_list), 10)
self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp.py b/Tools/Scripts/webkitpy/style/checkers/cpp.py
index cd3522f..e1543ce 100644
--- a/Tools/Scripts/webkitpy/style/checkers/cpp.py
+++ b/Tools/Scripts/webkitpy/style/checkers/cpp.py
@@ -1695,8 +1695,10 @@
error: The function to call with any errors found.
"""
match_ref_or_own_ptr = '(?=\W|^)(Ref|Own)Ptr(?=\W)'
+ exceptions = '(?:&|\*|\*\s*=\s*0)$'
bad_type_usage = search(match_ref_or_own_ptr, type_text)
- if not bad_type_usage or type_text.endswith('&') or type_text.endswith('*'):
+ exception_usage = search(exceptions, type_text)
+ if not bad_type_usage or exception_usage:
return
type_name = bad_type_usage.group(0)
error(row, 'readability/pass_ptr', 5,
diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py
index 1b3d97f..4e22dc3 100644
--- a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py
+++ b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py
@@ -3571,6 +3571,16 @@
'{\n'
'}',
'')
+ self.assert_pass_ptr_check(
+ 'int myFunction(RefPtr<Type1>* = 0)\n'
+ '{\n'
+ '}',
+ '')
+ self.assert_pass_ptr_check(
+ 'int myFunction(RefPtr<Type1>* = 0)\n'
+ '{\n'
+ '}',
+ '')
def test_own_ptr_parameter_value(self):
self.assert_pass_ptr_check(
diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
index 0a1849c..214395f 100644
--- a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
+++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
@@ -288,6 +288,17 @@
class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
# not overriding execute() - pylint: disable=W0223
+ def __init__(self, options=None):
+ super(AbstractParallelRebaselineCommand, self).__init__(options=options)
+ self._builder_data = {}
+
+ def builder_data(self):
+ if not self._builder_data:
+ for builder_name in self._release_builders():
+ builder = self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name)
+ self._builder_data[builder_name] = builder.latest_layout_test_results()
+ return self._builder_data
+
# The release builders cycle much faster than the debug ones and cover all the platforms.
def _release_builders(self):
release_builders = []
@@ -337,7 +348,11 @@
for test_prefix in test_prefix_list:
for test in port.tests([test_prefix]):
for builder in self._builders_to_fetch_from(test_prefix_list[test_prefix]):
- suffixes = ','.join(test_prefix_list[test_prefix][builder])
+ actual_failures_suffixes = self._suffixes_for_actual_failures(test, builder, test_prefix_list[test_prefix][builder])
+ if not actual_failures_suffixes:
+ continue
+
+ suffixes = ','.join(actual_failures_suffixes)
cmd_line = ['--suffixes', suffixes, '--builder', builder, '--test', test]
if options.results_directory:
cmd_line.extend(['--results-directory', options.results_directory])
@@ -379,7 +394,7 @@
for test in test_prefix_list:
all_suffixes = set()
for builder in self._builders_to_fetch_from(test_prefix_list[test]):
- all_suffixes.update(test_prefix_list[test][builder])
+ all_suffixes.update(self._suffixes_for_actual_failures(test, builder, test_prefix_list[test][builder]))
# FIXME: We should propagate the platform options as well.
self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test], verbose)
@@ -420,6 +435,12 @@
if options.optimize:
self._optimize_baselines(test_prefix_list, options.verbose)
+ def _suffixes_for_actual_failures(self, test, builder_name, existing_suffixes):
+ actual_results = self.builder_data()[builder_name].actual_results(test)
+ if not actual_results:
+ return set()
+ return set(existing_suffixes) & TestExpectations.suffixes_for_actual_expectations_string(actual_results)
+
class RebaselineJson(AbstractParallelRebaselineCommand):
name = "rebaseline-json"
@@ -546,14 +567,6 @@
# FIXME: Remove this option.
self.results_directory_option,
])
- self._builder_data = {}
-
- def builder_data(self):
- if not self._builder_data:
- for builder_name in self._release_builders():
- builder = self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name)
- self._builder_data[builder_name] = builder.latest_layout_test_results()
- return self._builder_data
def latest_revision_processed_on_all_bots(self):
revisions = []
@@ -622,7 +635,6 @@
def get_test_prefix_list(self, tests):
test_prefix_list = {}
lines_to_remove = {}
- builder_data = self.builder_data()
for builder_name in self._release_builders():
port_name = builders.port_name_for_builder_name(builder_name)
@@ -632,21 +644,11 @@
if test not in tests:
continue
- if test not in lines_to_remove:
- lines_to_remove[test] = []
- lines_to_remove[test].append(builder_name)
-
- actual_results = builder_data[builder_name].actual_results(test)
- if not actual_results:
- continue
-
- suffixes = TestExpectations.suffixes_for_actual_expectations_string(actual_results)
- if not suffixes:
- continue
-
if test not in test_prefix_list:
+ lines_to_remove[test] = []
test_prefix_list[test] = {}
- test_prefix_list[test][builder_name] = suffixes
+ lines_to_remove[test].append(builder_name)
+ test_prefix_list[test][builder_name] = BASELINE_SUFFIX_LIST
return test_prefix_list, lines_to_remove
diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
index d30ef9e..9602cf0 100644
--- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
+++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
@@ -73,6 +73,24 @@
self._write(path, '')
self.tool.filesystem.written_files = {}
+ def _setup_mock_builder_data(self):
+ data = LayoutTestResults.results_from_string("""ADD_RESULTS({
+ "tests": {
+ "userscripts": {
+ "first-test.html": {
+ "expected": "PASS",
+ "actual": "IMAGE+TEXT"
+ },
+ "second-test.html": {
+ "expected": "FAIL",
+ "actual": "IMAGE+TEXT"
+ }
+ }
+ }
+});""")
+ for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']:
+ self.command._builder_data[builder] = data
+
class TestCopyExistingBaselinesInternal(_BaseTestCase):
command_constructor = CopyExistingBaselinesInternal # AKA webkit-patch rebaseline-test-internal
@@ -289,46 +307,54 @@
super(TestRebaselineJson, self).tearDown()
def test_rebaseline_all(self):
+ self._setup_mock_builder_data()
+
options = MockOptions(optimize=True, verbose=True, results_directory=None)
- self._write("user-scripts/another-test.html", "Dummy test contents")
- self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}})
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}})
# Note that we have one run_in_parallel() call followed by a run_command()
self.assertEqual(self.tool.executive.calls,
- [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--verbose']],
- [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--verbose']],
- ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']])
+ [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
+ [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
+ ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'userscripts/first-test.html']])
def test_rebaseline_debug(self):
+ self._setup_mock_builder_data()
+
options = MockOptions(optimize=True, verbose=True, results_directory=None)
- self._write("user-scripts/another-test.html", "Dummy test contents")
- self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
# Note that we have one run_in_parallel() call followed by a run_command()
self.assertEqual(self.tool.executive.calls,
- [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']],
- [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']],
- ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']])
+ [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
+ [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
+ ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'userscripts/first-test.html']])
def test_no_optimize(self):
+ self._setup_mock_builder_data()
+
options = MockOptions(optimize=False, verbose=True, results_directory=None)
- self._write("user-scripts/another-test.html", "Dummy test contents")
- self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
# Note that we have only one run_in_parallel() call
self.assertEqual(self.tool.executive.calls,
- [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']],
- [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']]])
+ [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
+ [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']]])
def test_results_directory(self):
+ self._setup_mock_builder_data()
+
options = MockOptions(optimize=False, verbose=True, results_directory='/tmp')
- self._write("user-scripts/another-test.html", "Dummy test contents")
- self.command._rebaseline(options, {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}})
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}})
# Note that we have only one run_in_parallel() call
self.assertEqual(self.tool.executive.calls,
- [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--results-directory', '/tmp', '--verbose']],
- [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--results-directory', '/tmp', '--verbose']]])
+ [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']],
+ [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']]])
class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase):
command_constructor = RebaselineJson
@@ -345,30 +371,32 @@
return_stderr=True,
decode_output=False,
env=None):
- return '{"add": [], "remove-lines": [{"test": "userscripts/another-test.html", "builder": "WebKit Mac10.7"}]}\n'
+ return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "WebKit Mac10.7"}]}\n'
self.tool.executive.run_command = mock_run_command
def test_rebaseline_updates_expectations_file(self):
options = MockOptions(optimize=False, verbose=True, results_directory=None)
- self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
- self._write("userscripts/another-test.html", "Dummy test contents")
+ self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self._setup_mock_builder_data()
- self.command._rebaseline(options, {"userscripts/another-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
new_expectations = self._read(self.lion_expectations_path)
- self.assertMultiLineEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
+ self.assertMultiLineEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
def test_rebaseline_updates_expectations_file_all_platforms(self):
options = MockOptions(optimize=False, verbose=True, results_directory=None)
- self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ ImageOnlyFailure ]\n")
- self._write("userscripts/another-test.html", "Dummy test contents")
+ self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n")
+ self._write("userscripts/first-test.html", "Dummy test contents")
+ self._setup_mock_builder_data()
- self.command._rebaseline(options, {"userscripts/another-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
+ self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
new_expectations = self._read(self.lion_expectations_path)
- self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux MountainLion SnowLeopard Win ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
+ self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux MountainLion SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
class TestRebaseline(_BaseTestCase):
@@ -379,23 +407,24 @@
def test_rebaseline(self):
self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
- self._write("mock/path/to/test.html", "Dummy test contents")
+ self._write("userscripts/first-test.html", "test data")
self._zero_out_test_expectations()
+ self._setup_mock_builder_data()
old_exact_matches = builders._exact_matches
try:
builders._exact_matches = {
"MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
}
- self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['mock/path/to/test.html'], self.tool)
+ self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool)
finally:
builders._exact_matches = old_exact_matches
calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls)
self.assertEqual(calls,
- [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html', '--verbose']],
- [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html', '--verbose']]])
+ [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
+ [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]])
def test_rebaseline_directory(self):
self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
@@ -403,6 +432,8 @@
self._write("userscripts/first-test.html", "test data")
self._write("userscripts/second-test.html", "test data")
+ self._setup_mock_builder_data()
+
old_exact_matches = builders._exact_matches
try:
builders._exact_matches = {
@@ -432,15 +463,59 @@
self.tool.executive = MockExecutive2()
+ def builder_data():
+ self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
+ "tests": {
+ "userscripts": {
+ "another-test.html": {
+ "expected": "PASS",
+ "actual": "PASS TEXT"
+ },
+ "images.svg": {
+ "expected": "FAIL",
+ "actual": "IMAGE+TEXT"
+ }
+ }
+ }
+});""")
+ return self.command._builder_data
+
+ self.command.builder_data = builder_data
+
self._write("userscripts/another-test.html", "Dummy test contents")
self._write("userscripts/images.svg", "Dummy test contents")
- self.command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])}
- self.command.execute(self.options, [], self.tool)
+ self.command._tests_to_rebaseline = lambda port: {
+ 'userscripts/another-test.html': set(['txt']),
+ 'userscripts/images.svg': set(['png']),
+ 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']),
+ }
+
+ old_exact_matches = builders._exact_matches
+ try:
+ builders._exact_matches = {
+ "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
+ "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
+ }
+ self.command.execute(self.options, [], self.tool)
+ finally:
+ builders._exact_matches = old_exact_matches
# FIXME: change this to use the test- ports.
calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls)
- self.assertEqual(len(calls), 2)
- self.assertEqual(len(calls[0]), 14)
+ self.assertEqual(self.tool.executive.calls, [
+ [
+ ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'],
+ ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'],
+ ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],
+ ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg']
+ ],
+ [
+ ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'],
+ ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'],
+ ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],
+ ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg']
+ ]
+ ])
def test_rebaseline_expectations_noop(self):
self._zero_out_test_expectations()