Remove the least important WebRTC video tests.
These are considered expandable, and since video tests are very
expensive (45s each), let's remove them.
Bug: webrtc:11426
Change-Id: I4aea18e93d3b3672900650aacf0b5524c52c2900
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/170364
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30790}
diff --git a/video/full_stack_tests.cc b/video/full_stack_tests.cc
index cb13a97..7307b46 100644
--- a/video/full_stack_tests.cc
+++ b/video/full_stack_tests.cc
@@ -824,9 +824,6 @@
#if !defined(WEBRTC_MAC) && !defined(WEBRTC_WIN)
// TODO(bugs.webrtc.org/9840): Investigate why is this test flaky on Win/Mac.
-const char kScreenshareSimulcastVariableFramerateExperiment[] =
- "WebRTC-VP8VariableFramerateScreenshare/"
- "Enabled,min_fps:5.0,min_qp:15,undershoot:30/";
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging screenshare;
@@ -855,64 +852,6 @@
fixture->RunWithAnalyzer(screenshare);
}
-TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast_Variable_Framerate) {
- test::ScopedFieldTrials field_trial(
- AppendFieldTrials(kScreenshareSimulcastVariableFramerateExperiment));
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging screenshare;
- screenshare.call.send_side_bwe = true;
- screenshare.screenshare[0] = {true, false, 10};
- screenshare.video[0] = {true, 1850, 1110, 30, 800000, 2500000,
- 2500000, false, "VP8", 2, 1, 400000,
- false, false, false, ""};
- screenshare.analyzer = {"screenshare_slides_simulcast_variable_framerate",
- 0.0, 0.0, kFullStackTestDurationSecs};
- ParamsWithLogging screenshare_params_high;
- screenshare_params_high.video[0] = {
- true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
- "VP8", 2, 0, 400000, false, false, false, ""};
- VideoQualityTest::Params screenshare_params_low;
- screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
- 1000000, false, "VP8", 2, 0, 400000,
- false, false, false, ""};
-
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
- VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
- screenshare.ss[0] = {
- streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
- fixture->RunWithAnalyzer(screenshare);
-}
-
-TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast_low) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging screenshare;
- screenshare.call.send_side_bwe = true;
- screenshare.screenshare[0] = {true, false, 10};
- screenshare.video[0] = {true, 1850, 1110, 30, 800000, 2500000,
- 2500000, false, "VP8", 2, 1, 400000,
- false, false, false, ""};
- screenshare.analyzer = {"screenshare_slides_simulcast_low", 0.0, 0.0,
- kFullStackTestDurationSecs};
- VideoQualityTest::Params screenshare_params_high;
- screenshare_params_high.video[0] = {
- true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
- "VP8", 2, 0, 400000, false, false, false, ""};
- VideoQualityTest::Params screenshare_params_low;
- screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
- 1000000, false, "VP8", 2, 0, 400000,
- false, false, false, ""};
-
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
- VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
- screenshare.ss[0] = {
- streams, 0, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
- fixture->RunWithAnalyzer(screenshare);
-}
-
#endif // !defined(WEBRTC_MAC) && !defined(WEBRTC_WIN)
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) {
@@ -1063,66 +1002,7 @@
fixture->RunWithAnalyzer(screenshare);
}
-TEST(FullStackTest, ScreenshareSlidesVP9_3SL_Variable_Fps) {
- webrtc::test::ScopedFieldTrials override_trials(
- AppendFieldTrials("WebRTC-VP9VariableFramerateScreenshare/"
- "Enabled,min_qp:32,min_fps:5.0,undershoot:30,frames_"
- "before_steady_state:5/"));
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging screenshare;
- screenshare.call.send_side_bwe = true;
- screenshare.video[0] = {true, 1850, 1110, 30, 50000, 200000,
- 2000000, false, "VP9", 1, 0, 400000,
- false, false, false, ""};
- screenshare.screenshare[0] = {true, false, 10};
- screenshare.analyzer = {"screenshare_slides_vp9_3sl_variable_fps", 0.0, 0.0,
- kFullStackTestDurationSecs};
- screenshare.ss[0] = {
- std::vector<VideoStream>(), 0, 3, 2, InterLayerPredMode::kOn,
- std::vector<SpatialLayer>(), true};
- fixture->RunWithAnalyzer(screenshare);
-}
-
-TEST(FullStackTest, VP9SVC_3SL_High) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging simulcast;
- simulcast.call.send_side_bwe = true;
- simulcast.video[0] = SvcVp9Video();
- simulcast.analyzer = {"vp9svc_3sl_high", 0.0, 0.0,
- kFullStackTestDurationSecs};
-
- simulcast.ss[0] = {
- std::vector<VideoStream>(), 0, 3, 2, InterLayerPredMode::kOn,
- std::vector<SpatialLayer>(), false};
- fixture->RunWithAnalyzer(simulcast);
-}
-
-TEST(FullStackTest, VP9SVC_3SL_Medium) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging simulcast;
- simulcast.call.send_side_bwe = true;
- simulcast.video[0] = SvcVp9Video();
- simulcast.analyzer = {"vp9svc_3sl_medium", 0.0, 0.0,
- kFullStackTestDurationSecs};
- simulcast.ss[0] = {
- std::vector<VideoStream>(), 0, 3, 1, InterLayerPredMode::kOn,
- std::vector<SpatialLayer>(), false};
- fixture->RunWithAnalyzer(simulcast);
-}
-
-TEST(FullStackTest, VP9SVC_3SL_Low) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging simulcast;
- simulcast.call.send_side_bwe = true;
- simulcast.video[0] = SvcVp9Video();
- simulcast.analyzer = {"vp9svc_3sl_low", 0.0, 0.0, kFullStackTestDurationSecs};
- simulcast.ss[0] = {
- std::vector<VideoStream>(), 0, 3, 0, InterLayerPredMode::kOn,
- std::vector<SpatialLayer>(), false};
- fixture->RunWithAnalyzer(simulcast);
-}
-
-// bugs.webrtc.org/9506
+// TODO(http://bugs.webrtc.org/9506): investigate.
#if !defined(WEBRTC_MAC)
TEST(FullStackTest, VP9KSVC_3SL_High) {
@@ -1140,21 +1020,6 @@
fixture->RunWithAnalyzer(simulcast);
}
-TEST(FullStackTest, VP9KSVC_3SL_Medium) {
- webrtc::test::ScopedFieldTrials override_trials(
- AppendFieldTrials("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation/Enabled/"));
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging simulcast;
- simulcast.call.send_side_bwe = true;
- simulcast.video[0] = SvcVp9Video();
- simulcast.analyzer = {"vp9ksvc_3sl_medium", 0.0, 0.0,
- kFullStackTestDurationSecs};
- simulcast.ss[0] = {
- std::vector<VideoStream>(), 0, 3, 1, InterLayerPredMode::kOnKeyPic,
- std::vector<SpatialLayer>(), false};
- fixture->RunWithAnalyzer(simulcast);
-}
-
TEST(FullStackTest, VP9KSVC_3SL_Low) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation/Enabled/"));
@@ -1283,32 +1148,6 @@
fixture->RunWithAnalyzer(simulcast);
}
-TEST(FullStackTest, SimulcastVP8_3SL_Medium) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging simulcast;
- simulcast.call.send_side_bwe = true;
- simulcast.video[0] = SimulcastVp8VideoHigh();
- simulcast.analyzer = {"simulcast_vp8_3sl_medium", 0.0, 0.0,
- kFullStackTestDurationSecs};
- simulcast.config->loss_percent = 0;
- simulcast.config->queue_delay_ms = 100;
- ParamsWithLogging video_params_high;
- video_params_high.video[0] = SimulcastVp8VideoHigh();
- ParamsWithLogging video_params_medium;
- video_params_medium.video[0] = SimulcastVp8VideoMedium();
- ParamsWithLogging video_params_low;
- video_params_low.video[0] = SimulcastVp8VideoLow();
-
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(video_params_low, 0),
- VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
- VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
- simulcast.ss[0] = {
- streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
- fixture->RunWithAnalyzer(simulcast);
-}
-
TEST(FullStackTest, SimulcastVP8_3SL_Low) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
@@ -1372,70 +1211,14 @@
fixture->RunWithAnalyzer(generator);
}
-TEST(FullStackTest, LargeRoomVP8_5thumb) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging large_room;
- large_room.call.send_side_bwe = true;
- large_room.video[0] = SimulcastVp8VideoHigh();
- large_room.analyzer = {"largeroom_5thumb", 0.0, 0.0,
- kFullStackTestDurationSecs};
- large_room.config->loss_percent = 0;
- large_room.config->queue_delay_ms = 100;
- ParamsWithLogging video_params_high;
- video_params_high.video[0] = SimulcastVp8VideoHigh();
- ParamsWithLogging video_params_medium;
- video_params_medium.video[0] = SimulcastVp8VideoMedium();
- ParamsWithLogging video_params_low;
- video_params_low.video[0] = SimulcastVp8VideoLow();
-
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(video_params_low, 0),
- VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
- VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
- large_room.call.num_thumbnails = 5;
- large_room.ss[0] = {
- streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
- fixture->RunWithAnalyzer(large_room);
-}
-
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
// Fails on mobile devices:
// https://bugs.chromium.org/p/webrtc/issues/detail?id=7301
#define MAYBE_LargeRoomVP8_50thumb DISABLED_LargeRoomVP8_50thumb
-#define MAYBE_LargeRoomVP8_15thumb DISABLED_LargeRoomVP8_15thumb
#else
#define MAYBE_LargeRoomVP8_50thumb LargeRoomVP8_50thumb
-#define MAYBE_LargeRoomVP8_15thumb LargeRoomVP8_15thumb
#endif
-TEST(FullStackTest, MAYBE_LargeRoomVP8_15thumb) {
- auto fixture = CreateVideoQualityTestFixture();
- ParamsWithLogging large_room;
- large_room.call.send_side_bwe = true;
- large_room.video[0] = SimulcastVp8VideoHigh();
- large_room.analyzer = {"largeroom_15thumb", 0.0, 0.0,
- kFullStackTestDurationSecs};
- large_room.config->loss_percent = 0;
- large_room.config->queue_delay_ms = 100;
- ParamsWithLogging video_params_high;
- video_params_high.video[0] = SimulcastVp8VideoHigh();
- ParamsWithLogging video_params_medium;
- video_params_medium.video[0] = SimulcastVp8VideoMedium();
- ParamsWithLogging video_params_low;
- video_params_low.video[0] = SimulcastVp8VideoLow();
-
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(video_params_low, 0),
- VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
- VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
- large_room.call.num_thumbnails = 15;
- large_room.ss[0] = {
- streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
- fixture->RunWithAnalyzer(large_room);
-}
-
TEST(FullStackTest, MAYBE_LargeRoomVP8_50thumb) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging large_room;
@@ -1469,110 +1252,4 @@
::testing::Values("WebRTC-GenericDescriptor/Disabled/",
"WebRTC-GenericDescriptor/Enabled/"));
-class DualStreamsTest : public ::testing::TestWithParam<int> {};
-
-// Disable dual video test on mobile device becuase it's too heavy.
-// TODO(bugs.webrtc.org/9840): Investigate why is this test flaky on MAC.
-#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) && !defined(WEBRTC_MAC)
-TEST_P(DualStreamsTest,
- ModeratelyRestricted_SlidesVp8_2TL_Simulcast_Video_Simulcast_High) {
- const int first_stream = GetParam();
- ParamsWithLogging dual_streams;
-
- // Screenshare Settings.
- dual_streams.screenshare[first_stream] = {true, false, 10};
- dual_streams.video[first_stream] = {true, 1850, 1110, 5, 800000, 2500000,
- 2500000, false, "VP8", 2, 1, 400000,
- false, false, false, ""};
-
- ParamsWithLogging screenshare_params_high;
- screenshare_params_high.video[0] = {
- true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
- "VP8", 2, 0, 400000, false, false, false, ""};
- VideoQualityTest::Params screenshare_params_low;
- screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
- 1000000, false, "VP8", 2, 0, 400000,
- false, false, false, ""};
- std::vector<VideoStream> screenhsare_streams = {
- VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
- VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
-
- dual_streams.ss[first_stream] = {
- screenhsare_streams, 1, 1, 0, InterLayerPredMode::kOn,
- std::vector<SpatialLayer>(), false};
-
- // Video settings.
- dual_streams.video[1 - first_stream] = SimulcastVp8VideoHigh();
-
- ParamsWithLogging video_params_high;
- video_params_high.video[0] = SimulcastVp8VideoHigh();
- ParamsWithLogging video_params_medium;
- video_params_medium.video[0] = SimulcastVp8VideoMedium();
- ParamsWithLogging video_params_low;
- video_params_low.video[0] = SimulcastVp8VideoLow();
- std::vector<VideoStream> streams = {
- VideoQualityTest::DefaultVideoStream(video_params_low, 0),
- VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
- VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
-
- dual_streams.ss[1 - first_stream] = {
- streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
- false};
-
- // Call settings.
- dual_streams.call.send_side_bwe = true;
- dual_streams.call.dual_video = true;
- std::string test_label = "dualstreams_moderately_restricted_screenshare_" +
- std::to_string(first_stream);
- dual_streams.analyzer = {test_label, 0.0, 0.0, kFullStackTestDurationSecs};
- dual_streams.config->loss_percent = 1;
- dual_streams.config->link_capacity_kbps = 7500;
- dual_streams.config->queue_length_packets = 30;
- dual_streams.config->queue_delay_ms = 100;
-
- auto fixture = CreateVideoQualityTestFixture();
- fixture->RunWithAnalyzer(dual_streams);
-}
-#endif // !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) &&
- // !defined(WEBRTC_MAC)
-
-TEST_P(DualStreamsTest, Conference_Restricted) {
- const int first_stream = GetParam();
- ParamsWithLogging dual_streams;
-
- // Screenshare Settings.
- dual_streams.screenshare[first_stream] = {true, false, 10};
- dual_streams.video[first_stream] = {true, 1850, 1110, 5, 800000, 2500000,
- 2500000, false, "VP8", 3, 2, 400000,
- false, false, false, ""};
- // Video settings.
- dual_streams.video[1 - first_stream] = {
- true, 1280,
- 720, 30,
- 150000, 500000,
- 700000, false,
- "VP8", 3,
- 2, 400000,
- false, false,
- false, ClipNameToClipPath("ConferenceMotion_1280_720_50")};
-
- // Call settings.
- dual_streams.call.send_side_bwe = true;
- dual_streams.call.dual_video = true;
- std::string test_label = "dualstreams_conference_restricted_screenshare_" +
- std::to_string(first_stream);
- dual_streams.analyzer = {test_label, 0.0, 0.0, kFullStackTestDurationSecs};
- dual_streams.config->loss_percent = 1;
- dual_streams.config->link_capacity_kbps = 5000;
- dual_streams.config->queue_length_packets = 30;
- dual_streams.config->queue_delay_ms = 100;
-
- auto fixture = CreateVideoQualityTestFixture();
- fixture->RunWithAnalyzer(dual_streams);
-}
-
-INSTANTIATE_TEST_SUITE_P(FullStackTest,
- DualStreamsTest,
- ::testing::Values(0, 1));
-
} // namespace webrtc
diff --git a/video/pc_full_stack_tests.cc b/video/pc_full_stack_tests.cc
index 54a6c8a..a955071 100644
--- a/video/pc_full_stack_tests.cc
+++ b/video/pc_full_stack_tests.cc
@@ -1411,36 +1411,6 @@
fixture->Run(std::move(run_params));
}
-TEST(PCFullStackTest, ScreenshareSlidesVP9_3SL_Variable_Fps) {
- webrtc::test::ScopedFieldTrials override_trials(
- AppendFieldTrials("WebRTC-VP9VariableFramerateScreenshare/"
- "Enabled,min_qp:32,min_fps:5.0,undershoot:30,frames_"
- "before_steady_state:5/"
- "WebRTC-Vp9InterLayerPred/"
- "Enabled,inter_layer_pred_mode:on/"));
- std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
- CreateNetworkEmulationManager();
- auto fixture = CreateTestFixture(
- "pc_screenshare_slides_vp9_3sl_variable_fps",
- CreateTwoNetworkLinks(network_emulation_manager.get(),
- BuiltInNetworkBehaviorConfig()),
- [](PeerConfigurer* alice) {
- VideoConfig video(1850, 1110, 30);
- video.stream_label = "alice-video";
- video.screen_share_config = ScreenShareConfig(TimeDelta::Seconds(10));
- video.simulcast_config = VideoSimulcastConfig(3, 2);
- alice->AddVideoConfig(std::move(video));
- },
- [](PeerConfigurer* bob) {});
- RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
- run_params.video_codecs = {VideoCodecConfig(
- /*name=*/cricket::kVp9CodecName, /*required_params=*/{
- {kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
- run_params.use_flex_fec = false;
- run_params.use_ulp_fec = false;
- fixture->Run(std::move(run_params));
-}
-
TEST(PCFullStackTest, VP9SVC_3SL_High) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
@@ -1470,35 +1440,6 @@
fixture->Run(std::move(run_params));
}
-TEST(PCFullStackTest, VP9SVC_3SL_Medium) {
- webrtc::test::ScopedFieldTrials override_trials(
- AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
- "Enabled,inter_layer_pred_mode:on/"));
- std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
- CreateNetworkEmulationManager();
- auto fixture = CreateTestFixture(
- "pc_vp9svc_3sl_medium",
- CreateTwoNetworkLinks(network_emulation_manager.get(),
- BuiltInNetworkBehaviorConfig()),
- [](PeerConfigurer* alice) {
- VideoConfig video(1280, 720, 30);
- video.stream_label = "alice-video";
- video.input_file_name =
- ClipNameToClipPath("ConferenceMotion_1280_720_50");
- video.simulcast_config = VideoSimulcastConfig(3, 1);
- video.temporal_layers_count = 3;
- alice->AddVideoConfig(std::move(video));
- },
- [](PeerConfigurer* bob) {});
- RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
- run_params.video_codecs = {VideoCodecConfig(
- /*name=*/cricket::kVp9CodecName, /*required_params=*/{
- {kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
- run_params.use_flex_fec = false;
- run_params.use_ulp_fec = false;
- fixture->Run(std::move(run_params));
-}
-
TEST(PCFullStackTest, VP9SVC_3SL_Low) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
@@ -1680,31 +1621,6 @@
fixture->Run(std::move(run_params));
}
-TEST(PCFullStackTest, SimulcastVP8_3SL_Medium) {
- std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
- CreateNetworkEmulationManager();
- BuiltInNetworkBehaviorConfig config;
- config.loss_percent = 0;
- config.queue_delay_ms = 100;
- auto fixture = CreateTestFixture(
- "pc_simulcast_vp8_3sl_medium",
- CreateTwoNetworkLinks(network_emulation_manager.get(), config),
- [](PeerConfigurer* alice) {
- VideoConfig video(1280, 720, 30);
- video.input_file_name =
- ClipNameToClipPath("ConferenceMotion_1280_720_50");
- video.simulcast_config = VideoSimulcastConfig(3, 1);
- video.stream_label = "alice-video";
- alice->AddVideoConfig(std::move(video));
- },
- [](PeerConfigurer* bob) {});
- RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
- run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
- run_params.use_flex_fec = false;
- run_params.use_ulp_fec = false;
- fixture->Run(std::move(run_params));
-}
-
TEST(PCFullStackTest, SimulcastVP8_3SL_Low) {
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();