Reland "Adds more performance stats collection to scenario tests."
This is a reland of 63b0b2cf307b47bae3c10b295ece9a5f6d9bd7a4
Original change's description:
> Adds more performance stats collection to scenario tests.
>
> Bug: webrtc:10365
> Change-Id: I66dce6ff21242c30af674f89fc9fd19172d4a3af
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131948
> Commit-Queue: Sebastian Jansson <srte@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#27585}
Bug: webrtc:10365
Change-Id: Id7ddb64ac17ecbb4de223dec497bc562040ba7c4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132711
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27618}diff --git a/test/scenario/stats_collection_unittest.cc b/test/scenario/stats_collection_unittest.cc
index 7e65ec1..48a4bc3 100644
--- a/test/scenario/stats_collection_unittest.cc
+++ b/test/scenario/stats_collection_unittest.cc
@@ -16,49 +16,71 @@
namespace {
void CreateAnalyzedStream(Scenario* s,
NetworkNodeConfig network_config,
- VideoQualityAnalyzer* analyzer) {
+ VideoQualityAnalyzer* analyzer,
+ CallStatsCollectors* collectors) {
VideoStreamConfig config;
config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
config.encoder.implementation =
VideoStreamConfig::Encoder::Implementation::kSoftware;
config.hooks.frame_pair_handlers = {analyzer->Handler()};
- auto route = s->CreateRoutes(s->CreateClient("caller", CallClientConfig()),
- {s->CreateSimulationNode(network_config)},
- s->CreateClient("callee", CallClientConfig()),
- {s->CreateSimulationNode(NetworkNodeConfig())});
- s->CreateVideoStream(route->forward(), config);
+ auto* caller = s->CreateClient("caller", CallClientConfig());
+ auto route =
+ s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)},
+ s->CreateClient("callee", CallClientConfig()),
+ {s->CreateSimulationNode(NetworkNodeConfig())});
+ auto* video = s->CreateVideoStream(route->forward(), config);
+ auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig());
+ if (collectors) {
+ s->Every(TimeDelta::seconds(1), [=] {
+ collectors->call.AddStats(caller->GetStats());
+ collectors->audio_receive.AddStats(audio->receive()->GetStats());
+ collectors->video_send.AddStats(video->send()->GetStats(), s->Now());
+ collectors->video_receive.AddStats(video->receive()->GetStats());
+ });
+ }
}
} // namespace
TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) {
VideoQualityAnalyzer analyzer;
+ CallStatsCollectors stats;
{
- Scenario s("", /*real_time*/ false);
+ Scenario s;
NetworkNodeConfig good_network;
good_network.simulation.bandwidth = DataRate::kbps(1000);
- CreateAnalyzedStream(&s, good_network, &analyzer);
- s.RunFor(TimeDelta::seconds(1));
+ CreateAnalyzedStream(&s, good_network, &analyzer, &stats);
+ s.RunFor(TimeDelta::seconds(3));
}
- // This is mainty a regression test, the target is based on previous runs and
- // might change due to changes in configuration and encoder etc.
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_GT(*analyzer.stats().psnr.GetMean(), 40);
+ // This is a change detecting test, the targets are based on previous runs and
+ // might change due to changes in configuration and encoder etc. The main
+ // purpose is to show how the stats can be used. To avoid being overly
+ // sensistive to change, the ranges are chosen to be quite large.
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 43, 10);
+ EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300);
+ EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200);
+ EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
+ EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20);
}
TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) {
VideoQualityAnalyzer analyzer;
+ CallStatsCollectors stats;
{
- Scenario s("", /*real_time*/ false);
+ Scenario s;
NetworkNodeConfig bad_network;
bad_network.simulation.bandwidth = DataRate::kbps(100);
bad_network.simulation.loss_rate = 0.02;
- CreateAnalyzedStream(&s, bad_network, &analyzer);
- s.RunFor(TimeDelta::seconds(1));
+ CreateAnalyzedStream(&s, bad_network, &analyzer, &stats);
+ s.RunFor(TimeDelta::seconds(3));
}
- // This is mainty a regression test, the target is based on previous runs and
+ // This is a change detecting test, the targets are based on previous runs and
// might change due to changes in configuration and encoder etc.
- ASSERT_TRUE(analyzer.stats().psnr.GetMean());
- EXPECT_LT(*analyzer.stats().psnr.GetMean(), 30);
+ EXPECT_NEAR(analyzer.stats().psnr.Mean(), 16, 10);
+ EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50);
+ EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50);
+ EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
+ EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 45, 20);
}
+
} // namespace test
} // namespace webrtc