Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
Sebastian Jansson | 7150d8c | 2019-04-09 14:18:09 +0200 | [diff] [blame] | 10 | #include "test/scenario/stats_collection.h" |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 11 | |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 12 | #include "test/gtest.h" |
| 13 | #include "test/scenario/scenario.h" |
| 14 | |
| 15 | namespace webrtc { |
| 16 | namespace test { |
| 17 | namespace { |
Sebastian Jansson | cf2df2f | 2019-04-02 11:51:28 +0200 | [diff] [blame] | 18 | void CreateAnalyzedStream(Scenario* s, |
Sebastian Jansson | ef86d14 | 2019-04-15 14:42:42 +0200 | [diff] [blame] | 19 | NetworkSimulationConfig network_config, |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 20 | VideoQualityAnalyzer* analyzer, |
| 21 | CallStatsCollectors* collectors) { |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 22 | VideoStreamConfig config; |
| 23 | config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8; |
| 24 | config.encoder.implementation = |
| 25 | VideoStreamConfig::Encoder::Implementation::kSoftware; |
Sebastian Jansson | cf2df2f | 2019-04-02 11:51:28 +0200 | [diff] [blame] | 26 | config.hooks.frame_pair_handlers = {analyzer->Handler()}; |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 27 | auto* caller = s->CreateClient("caller", CallClientConfig()); |
Tommi | 3c9bcc1 | 2020-04-15 16:45:47 +0200 | [diff] [blame^] | 28 | auto* callee = s->CreateClient("callee", CallClientConfig()); |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 29 | auto route = |
Tommi | 3c9bcc1 | 2020-04-15 16:45:47 +0200 | [diff] [blame^] | 30 | s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)}, callee, |
Sebastian Jansson | ef86d14 | 2019-04-15 14:42:42 +0200 | [diff] [blame] | 31 | {s->CreateSimulationNode(NetworkSimulationConfig())}); |
Tommi | 3c9bcc1 | 2020-04-15 16:45:47 +0200 | [diff] [blame^] | 32 | VideoStreamPair* video = s->CreateVideoStream(route->forward(), config); |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 33 | auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig()); |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 34 | s->Every(TimeDelta::Seconds(1), [=] { |
Sebastian Jansson | dab21c6 | 2019-05-13 11:48:40 +0200 | [diff] [blame] | 35 | collectors->call.AddStats(caller->GetStats()); |
Artem Titov | 16cc9ef | 2020-04-07 18:02:39 +0000 | [diff] [blame] | 36 | collectors->video_send.AddStats(video->send()->GetStats(), s->Now()); |
Tommi | 3c9bcc1 | 2020-04-15 16:45:47 +0200 | [diff] [blame^] | 37 | collectors->audio_receive.AddStats(audio->receive()->GetStats()); |
| 38 | |
| 39 | // Querying the video stats from within the expected runtime environment |
| 40 | // (i.e. the TQ that belongs to the CallClient, not the Scenario TQ that |
| 41 | // we're currently on). |
| 42 | VideoReceiveStream::Stats video_receive_stats; |
| 43 | auto* video_stream = video->receive(); |
| 44 | callee->SendTask([&video_stream, &video_receive_stats]() { |
| 45 | video_receive_stats = video_stream->GetStats(); |
| 46 | }); |
| 47 | collectors->video_receive.AddStats(video_receive_stats); |
Sebastian Jansson | dab21c6 | 2019-05-13 11:48:40 +0200 | [diff] [blame] | 48 | }); |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 49 | } |
| 50 | } // namespace |
| 51 | |
| 52 | TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) { |
Sebastian Jansson | cf2df2f | 2019-04-02 11:51:28 +0200 | [diff] [blame] | 53 | VideoQualityAnalyzer analyzer; |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 54 | CallStatsCollectors stats; |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 55 | { |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 56 | Scenario s; |
Sebastian Jansson | ef86d14 | 2019-04-15 14:42:42 +0200 | [diff] [blame] | 57 | NetworkSimulationConfig good_network; |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 58 | good_network.bandwidth = DataRate::KilobitsPerSec(1000); |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 59 | CreateAnalyzedStream(&s, good_network, &analyzer, &stats); |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 60 | s.RunFor(TimeDelta::Seconds(3)); |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 61 | } |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 62 | // This is a change detecting test, the targets are based on previous runs and |
| 63 | // might change due to changes in configuration and encoder etc. The main |
| 64 | // purpose is to show how the stats can be used. To avoid being overly |
| 65 | // sensistive to change, the ranges are chosen to be quite large. |
Sebastian Jansson | e9cac4f | 2019-06-24 17:10:55 +0200 | [diff] [blame] | 66 | EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 43, 10); |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 67 | EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300); |
| 68 | EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200); |
| 69 | EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10); |
| 70 | EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20); |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 71 | } |
| 72 | |
| 73 | TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) { |
Sebastian Jansson | cf2df2f | 2019-04-02 11:51:28 +0200 | [diff] [blame] | 74 | VideoQualityAnalyzer analyzer; |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 75 | CallStatsCollectors stats; |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 76 | { |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 77 | Scenario s; |
Sebastian Jansson | ef86d14 | 2019-04-15 14:42:42 +0200 | [diff] [blame] | 78 | NetworkSimulationConfig bad_network; |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 79 | bad_network.bandwidth = DataRate::KilobitsPerSec(100); |
Sebastian Jansson | ef86d14 | 2019-04-15 14:42:42 +0200 | [diff] [blame] | 80 | bad_network.loss_rate = 0.02; |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 81 | CreateAnalyzedStream(&s, bad_network, &analyzer, &stats); |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 82 | s.RunFor(TimeDelta::Seconds(3)); |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 83 | } |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 84 | // This is a change detecting test, the targets are based on previous runs and |
Sebastian Jansson | cf2df2f | 2019-04-02 11:51:28 +0200 | [diff] [blame] | 85 | // might change due to changes in configuration and encoder etc. |
Sebastian Jansson | e9cac4f | 2019-06-24 17:10:55 +0200 | [diff] [blame] | 86 | EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 16, 10); |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 87 | EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50); |
| 88 | EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50); |
| 89 | EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10); |
Sebastian Jansson | c9f42ad | 2020-01-17 11:47:35 +0100 | [diff] [blame] | 90 | EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 200, 150); |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 91 | } |
Sebastian Jansson | 9a2ca0a | 2019-04-15 13:18:19 +0200 | [diff] [blame] | 92 | |
Sebastian Jansson | dab21c6 | 2019-05-13 11:48:40 +0200 | [diff] [blame] | 93 | TEST(ScenarioAnalyzerTest, CountsCapturedButNotRendered) { |
| 94 | VideoQualityAnalyzer analyzer; |
| 95 | CallStatsCollectors stats; |
| 96 | { |
| 97 | Scenario s; |
| 98 | NetworkSimulationConfig long_delays; |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 99 | long_delays.delay = TimeDelta::Seconds(5); |
Sebastian Jansson | dab21c6 | 2019-05-13 11:48:40 +0200 | [diff] [blame] | 100 | CreateAnalyzedStream(&s, long_delays, &analyzer, &stats); |
| 101 | // Enough time to send frames but not enough to deliver. |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 102 | s.RunFor(TimeDelta::Millis(100)); |
Sebastian Jansson | dab21c6 | 2019-05-13 11:48:40 +0200 | [diff] [blame] | 103 | } |
| 104 | EXPECT_GE(analyzer.stats().capture.count, 1); |
| 105 | EXPECT_EQ(analyzer.stats().render.count, 0); |
| 106 | } |
Sebastian Jansson | 9a4f38e | 2018-12-19 13:14:41 +0100 | [diff] [blame] | 107 | } // namespace test |
| 108 | } // namespace webrtc |