blob: 1486606d461f425da0d07ca79a7d9d3d1efe27f1 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080020#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010022#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020023#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020024#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010025#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020026#include "common_video/h264/h264_common.h"
Steve Anton10542f22019-01-11 09:11:00 -080027#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020028#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010030#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010034#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020035#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020036#include "system_wrappers/include/sleep.h"
37#include "test/encoder_settings.h"
38#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020039#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "test/frame_generator.h"
41#include "test/gmock.h"
42#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020043#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020044#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070045
46namespace webrtc {
47
sprangb1ca0732017-02-01 08:38:12 -080048using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080049using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080050
perkj803d97f2016-11-01 11:45:46 -070051namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020052const int kMinPixelsPerFrame = 320 * 180;
53const int kMinFramerateFps = 2;
54const int kMinBalancedFramerateFps = 7;
55const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080056const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010057const uint32_t kTargetBitrateBps = 1000000;
58const uint32_t kSimulcastTargetBitrateBps = 3150000;
59const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080060const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070061const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080063
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020064uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
65 0x00, 0x00, 0x03, 0x03, 0xF4,
66 0x05, 0x03, 0xC7, 0xE0, 0x1B,
67 0x41, 0x10, 0x8D, 0x00};
68
perkj803d97f2016-11-01 11:45:46 -070069class TestBuffer : public webrtc::I420Buffer {
70 public:
71 TestBuffer(rtc::Event* event, int width, int height)
72 : I420Buffer(width, height), event_(event) {}
73
74 private:
75 friend class rtc::RefCountedObject<TestBuffer>;
76 ~TestBuffer() override {
77 if (event_)
78 event_->Set();
79 }
80 rtc::Event* const event_;
81};
82
Niels Möller7dc26b72017-12-06 10:27:48 +010083class CpuOveruseDetectorProxy : public OveruseFrameDetector {
84 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020085 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
86 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010087 last_target_framerate_fps_(-1) {}
88 virtual ~CpuOveruseDetectorProxy() {}
89
90 void OnTargetFramerateUpdated(int framerate_fps) override {
91 rtc::CritScope cs(&lock_);
92 last_target_framerate_fps_ = framerate_fps;
93 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
94 }
95
96 int GetLastTargetFramerate() {
97 rtc::CritScope cs(&lock_);
98 return last_target_framerate_fps_;
99 }
100
Niels Möller4db138e2018-04-19 09:04:13 +0200101 CpuOveruseOptions GetOptions() { return options_; }
102
Niels Möller7dc26b72017-12-06 10:27:48 +0100103 private:
104 rtc::CriticalSection lock_;
105 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
106};
107
mflodmancc3d4422017-08-03 08:27:51 -0700108class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700109 public:
Niels Möller213618e2018-07-24 09:29:58 +0200110 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200111 const VideoStreamEncoderSettings& settings,
112 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100113 : VideoStreamEncoder(Clock::GetRealTimeClock(),
114 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200115 stats_proxy,
116 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200117 std::unique_ptr<OveruseFrameDetector>(
118 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100119 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200120 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700121
sprangb1ca0732017-02-01 08:38:12 -0800122 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100123 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800124 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800125 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700126 event.Set();
127 });
perkj070ba852017-02-16 15:46:27 -0800128 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700129 }
130
kthelgason2fc52542017-03-03 00:24:41 -0800131 // This is used as a synchronisation mechanism, to make sure that the
132 // encoder queue is not blocked before we start sending it frames.
133 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100134 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200135 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800136 ASSERT_TRUE(event.Wait(5000));
137 }
138
sprangb1ca0732017-02-01 08:38:12 -0800139 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800140
sprangb1ca0732017-02-01 08:38:12 -0800141 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800142
sprangb1ca0732017-02-01 08:38:12 -0800143 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800144
sprangb1ca0732017-02-01 08:38:12 -0800145 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700146
Niels Möller7dc26b72017-12-06 10:27:48 +0100147 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700148};
149
asapersson5f7226f2016-11-25 04:37:00 -0800150class VideoStreamFactory
151 : public VideoEncoderConfig::VideoStreamFactoryInterface {
152 public:
sprangfda496a2017-06-15 04:21:07 -0700153 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
154 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800155 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700156 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800157 }
158
159 private:
160 std::vector<VideoStream> CreateEncoderStreams(
161 int width,
162 int height,
163 const VideoEncoderConfig& encoder_config) override {
164 std::vector<VideoStream> streams =
165 test::CreateVideoStreams(width, height, encoder_config);
166 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100167 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700168 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800169 }
170 return streams;
171 }
sprangfda496a2017-06-15 04:21:07 -0700172
asapersson5f7226f2016-11-25 04:37:00 -0800173 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700174 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800175};
176
sprangb1ca0732017-02-01 08:38:12 -0800177class AdaptingFrameForwarder : public test::FrameForwarder {
178 public:
179 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700180 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800181
182 void set_adaptation_enabled(bool enabled) {
183 rtc::CritScope cs(&crit_);
184 adaptation_enabled_ = enabled;
185 }
186
asaperssonfab67072017-04-04 05:51:49 -0700187 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800188 rtc::CritScope cs(&crit_);
189 return adaptation_enabled_;
190 }
191
asapersson09f05612017-05-15 23:40:18 -0700192 rtc::VideoSinkWants last_wants() const {
193 rtc::CritScope cs(&crit_);
194 return last_wants_;
195 }
196
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200197 absl::optional<int> last_sent_width() const { return last_width_; }
198 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800199
sprangb1ca0732017-02-01 08:38:12 -0800200 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
201 int cropped_width = 0;
202 int cropped_height = 0;
203 int out_width = 0;
204 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700205 if (adaption_enabled()) {
206 if (adapter_.AdaptFrameResolution(
207 video_frame.width(), video_frame.height(),
208 video_frame.timestamp_us() * 1000, &cropped_width,
209 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100210 VideoFrame adapted_frame =
211 VideoFrame::Builder()
212 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
213 nullptr, out_width, out_height))
214 .set_timestamp_rtp(99)
215 .set_timestamp_ms(99)
216 .set_rotation(kVideoRotation_0)
217 .build();
sprangc5d62e22017-04-02 23:53:04 -0700218 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
219 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800220 last_width_.emplace(adapted_frame.width());
221 last_height_.emplace(adapted_frame.height());
222 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200223 last_width_ = absl::nullopt;
224 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700225 }
sprangb1ca0732017-02-01 08:38:12 -0800226 } else {
227 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800228 last_width_.emplace(video_frame.width());
229 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800230 }
231 }
232
233 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
234 const rtc::VideoSinkWants& wants) override {
235 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700236 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700237 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
238 wants.max_pixel_count,
239 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800240 test::FrameForwarder::AddOrUpdateSink(sink, wants);
241 }
sprangb1ca0732017-02-01 08:38:12 -0800242 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700243 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
244 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200245 absl::optional<int> last_width_;
246 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800247};
sprangc5d62e22017-04-02 23:53:04 -0700248
Niels Möller213618e2018-07-24 09:29:58 +0200249// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700250class MockableSendStatisticsProxy : public SendStatisticsProxy {
251 public:
252 MockableSendStatisticsProxy(Clock* clock,
253 const VideoSendStream::Config& config,
254 VideoEncoderConfig::ContentType content_type)
255 : SendStatisticsProxy(clock, config, content_type) {}
256
257 VideoSendStream::Stats GetStats() override {
258 rtc::CritScope cs(&lock_);
259 if (mock_stats_)
260 return *mock_stats_;
261 return SendStatisticsProxy::GetStats();
262 }
263
Niels Möller213618e2018-07-24 09:29:58 +0200264 int GetInputFrameRate() const override {
265 rtc::CritScope cs(&lock_);
266 if (mock_stats_)
267 return mock_stats_->input_frame_rate;
268 return SendStatisticsProxy::GetInputFrameRate();
269 }
sprangc5d62e22017-04-02 23:53:04 -0700270 void SetMockStats(const VideoSendStream::Stats& stats) {
271 rtc::CritScope cs(&lock_);
272 mock_stats_.emplace(stats);
273 }
274
275 void ResetMockStats() {
276 rtc::CritScope cs(&lock_);
277 mock_stats_.reset();
278 }
279
280 private:
281 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200282 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700283};
284
sprang4847ae62017-06-27 07:06:52 -0700285class MockBitrateObserver : public VideoBitrateAllocationObserver {
286 public:
Erik Språng566124a2018-04-23 12:32:22 +0200287 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700288};
289
perkj803d97f2016-11-01 11:45:46 -0700290} // namespace
291
mflodmancc3d4422017-08-03 08:27:51 -0700292class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700293 public:
294 static const int kDefaultTimeoutMs = 30 * 1000;
295
mflodmancc3d4422017-08-03 08:27:51 -0700296 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700297 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700298 codec_width_(320),
299 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200300 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200301 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700302 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200303 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800304 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700305 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700306 Clock::GetRealTimeClock(),
307 video_send_config_,
308 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700309 sink_(&fake_encoder_) {}
310
311 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700312 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700313 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200314 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800315 video_send_config_.encoder_settings.bitrate_allocator_factory =
316 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200317 video_send_config_.rtp.payload_name = "FAKE";
318 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700319
Per512ecb32016-09-23 15:52:06 +0200320 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200321 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700322 video_encoder_config.video_stream_factory =
323 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100324 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700325
326 // Framerate limit is specified by the VideoStreamFactory.
327 std::vector<VideoStream> streams =
328 video_encoder_config.video_stream_factory->CreateEncoderStreams(
329 codec_width_, codec_height_, video_encoder_config);
330 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200331 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700332
Niels Möllerf1338562018-04-26 09:51:47 +0200333 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800334 }
335
Niels Möllerf1338562018-04-26 09:51:47 +0200336 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700337 if (video_stream_encoder_)
338 video_stream_encoder_->Stop();
339 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200340 stats_proxy_.get(), video_send_config_.encoder_settings,
341 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700342 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
343 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700344 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700345 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
346 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200347 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700348 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800349 }
350
351 void ResetEncoder(const std::string& payload_name,
352 size_t num_streams,
353 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700354 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700355 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200356 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800357
358 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200359 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800360 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100361 video_encoder_config.max_bitrate_bps =
362 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800363 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700364 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
365 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700366 video_encoder_config.content_type =
367 screenshare ? VideoEncoderConfig::ContentType::kScreen
368 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700369 if (payload_name == "VP9") {
370 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
371 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
372 video_encoder_config.encoder_specific_settings =
373 new rtc::RefCountedObject<
374 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
375 }
Niels Möllerf1338562018-04-26 09:51:47 +0200376 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700377 }
378
sprang57c2fff2017-01-16 06:24:02 -0800379 VideoFrame CreateFrame(int64_t ntp_time_ms,
380 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100381 VideoFrame frame =
382 VideoFrame::Builder()
383 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
384 destruction_event, codec_width_, codec_height_))
385 .set_timestamp_rtp(99)
386 .set_timestamp_ms(99)
387 .set_rotation(kVideoRotation_0)
388 .build();
sprang57c2fff2017-01-16 06:24:02 -0800389 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700390 return frame;
391 }
392
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100393 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
394 rtc::Event* destruction_event,
395 int offset_x) const {
396 VideoFrame frame =
397 VideoFrame::Builder()
398 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
399 destruction_event, codec_width_, codec_height_))
400 .set_timestamp_rtp(99)
401 .set_timestamp_ms(99)
402 .set_rotation(kVideoRotation_0)
403 .set_update_rect({offset_x, 0, 1, 1})
404 .build();
405 frame.set_ntp_time_ms(ntp_time_ms);
406 return frame;
407 }
408
sprang57c2fff2017-01-16 06:24:02 -0800409 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100410 VideoFrame frame =
411 VideoFrame::Builder()
412 .set_video_frame_buffer(
413 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
414 .set_timestamp_rtp(99)
415 .set_timestamp_ms(99)
416 .set_rotation(kVideoRotation_0)
417 .build();
sprang57c2fff2017-01-16 06:24:02 -0800418 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700419 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700420 return frame;
421 }
422
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100423 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
424 MockBitrateObserver bitrate_observer;
425 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
426
427 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
428 .Times(1);
429 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +0200430 DataRate::bps(kTargetBitrateBps), 0,
431 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100432
433 video_source_.IncomingCapturedFrame(
434 CreateFrame(1, codec_width_, codec_height_));
435 WaitForEncodedFrame(1);
436 }
437
asapersson02465b82017-04-10 01:12:52 -0700438 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700439 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700440 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
441 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700442 }
443
asapersson09f05612017-05-15 23:40:18 -0700444 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
445 const rtc::VideoSinkWants& wants2) {
446 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
447 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
448 }
449
Åsa Persson8c1bf952018-09-13 10:42:19 +0200450 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
451 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
452 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
453 EXPECT_FALSE(wants.target_pixel_count);
454 }
455
asapersson09f05612017-05-15 23:40:18 -0700456 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
457 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200458 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700459 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
460 EXPECT_GT(wants1.max_pixel_count, 0);
461 }
462
463 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
464 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200465 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700466 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
467 }
468
asaperssonf7e294d2017-06-13 23:25:22 -0700469 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
470 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200471 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700472 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
473 }
474
475 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
476 const rtc::VideoSinkWants& wants2) {
477 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
478 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
479 }
480
481 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
482 const rtc::VideoSinkWants& wants2) {
483 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
484 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
485 }
486
487 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
488 const rtc::VideoSinkWants& wants2) {
489 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
490 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
491 EXPECT_GT(wants1.max_pixel_count, 0);
492 }
493
494 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
495 const rtc::VideoSinkWants& wants2) {
496 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
497 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
498 }
499
asapersson09f05612017-05-15 23:40:18 -0700500 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
501 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200502 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700503 EXPECT_LT(wants.max_pixel_count, pixel_count);
504 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700505 }
506
507 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
508 EXPECT_LT(wants.max_framerate_fps, fps);
509 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
510 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700511 }
512
asaperssonf7e294d2017-06-13 23:25:22 -0700513 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
514 int expected_fps) {
515 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
516 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
517 EXPECT_FALSE(wants.target_pixel_count);
518 }
519
Jonathan Yubc771b72017-12-08 17:04:29 -0800520 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
521 int last_frame_pixels) {
522 // Balanced mode should always scale FPS to the desired range before
523 // attempting to scale resolution.
524 int fps_limit = wants.max_framerate_fps;
525 if (last_frame_pixels <= 320 * 240) {
526 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
527 } else if (last_frame_pixels <= 480 * 270) {
528 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
529 } else if (last_frame_pixels <= 640 * 480) {
530 EXPECT_LE(15, fps_limit);
531 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200532 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800533 }
534 }
535
sprang4847ae62017-06-27 07:06:52 -0700536 void WaitForEncodedFrame(int64_t expected_ntp_time) {
537 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200538 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700539 }
540
541 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
542 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200543 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700544 return ok;
545 }
546
547 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
548 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200549 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700550 }
551
552 void ExpectDroppedFrame() {
553 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200554 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700555 }
556
557 bool WaitForFrame(int64_t timeout_ms) {
558 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200559 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700560 return ok;
561 }
562
perkj26091b12016-09-01 01:17:40 -0700563 class TestEncoder : public test::FakeEncoder {
564 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100565 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700566
asaperssonfab67072017-04-04 05:51:49 -0700567 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800568 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700569 return config_;
570 }
571
572 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800573 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700574 block_next_encode_ = true;
575 }
576
Erik Språngaed30702018-11-05 12:57:17 +0100577 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800578 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100579 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100580 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100581 if (quality_scaling_) {
582 info.scaling_settings =
583 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
584 }
585 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100586 for (int i = 0; i < kMaxSpatialLayers; ++i) {
587 if (temporal_layers_supported_[i]) {
588 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
589 info.fps_allocation[i].resize(num_layers);
590 }
591 }
Erik Språngaed30702018-11-05 12:57:17 +0100592 }
593 return info;
kthelgason876222f2016-11-29 01:44:11 -0800594 }
595
Erik Språngb7cb7b52019-02-26 15:52:33 +0100596 int32_t RegisterEncodeCompleteCallback(
597 EncodedImageCallback* callback) override {
598 rtc::CritScope lock(&local_crit_sect_);
599 encoded_image_callback_ = callback;
600 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
601 }
602
perkjfa10b552016-10-02 23:45:26 -0700603 void ContinueEncode() { continue_encode_event_.Set(); }
604
605 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
606 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800607 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700608 EXPECT_EQ(timestamp_, timestamp);
609 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
610 }
611
kthelgason2fc52542017-03-03 00:24:41 -0800612 void SetQualityScaling(bool b) {
613 rtc::CritScope lock(&local_crit_sect_);
614 quality_scaling_ = b;
615 }
kthelgasonad9010c2017-02-14 00:46:51 -0800616
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100617 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
618 rtc::CritScope lock(&local_crit_sect_);
619 is_hardware_accelerated_ = is_hardware_accelerated;
620 }
621
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100622 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
623 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
624 rtc::CritScope lock(&local_crit_sect_);
625 temporal_layers_supported_[spatial_idx] = supported;
626 }
627
sprangfe627f32017-03-29 08:24:59 -0700628 void ForceInitEncodeFailure(bool force_failure) {
629 rtc::CritScope lock(&local_crit_sect_);
630 force_init_encode_failed_ = force_failure;
631 }
632
Niels Möller6bb5ab92019-01-11 11:11:10 +0100633 void SimulateOvershoot(double rate_factor) {
634 rtc::CritScope lock(&local_crit_sect_);
635 rate_factor_ = rate_factor;
636 }
637
Erik Språngd7329ca2019-02-21 21:19:53 +0100638 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100639 rtc::CritScope lock(&local_crit_sect_);
640 return last_framerate_;
641 }
642
Erik Språngd7329ca2019-02-21 21:19:53 +0100643 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100644 rtc::CritScope lock(&local_crit_sect_);
645 return last_update_rect_;
646 }
647
Niels Möller87e2d782019-03-07 10:18:23 +0100648 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100649 rtc::CritScope lock(&local_crit_sect_);
650 return last_frame_types_;
651 }
652
653 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100654 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100655 keyframe ? VideoFrameType::kVideoFrameKey
656 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100657 {
658 rtc::CritScope lock(&local_crit_sect_);
659 last_frame_types_ = frame_type;
660 }
Niels Möllerb859b322019-03-07 12:40:01 +0100661 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100662 }
663
Erik Språngb7cb7b52019-02-26 15:52:33 +0100664 void InjectEncodedImage(const EncodedImage& image) {
665 rtc::CritScope lock(&local_crit_sect_);
666 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
667 }
668
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200669 void InjectEncodedImage(const EncodedImage& image,
670 const CodecSpecificInfo* codec_specific_info,
671 const RTPFragmentationHeader* fragmentation) {
672 rtc::CritScope lock(&local_crit_sect_);
673 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
674 fragmentation);
675 }
676
Erik Språngd7329ca2019-02-21 21:19:53 +0100677 void ExpectNullFrame() {
678 rtc::CritScope lock(&local_crit_sect_);
679 expect_null_frame_ = true;
680 }
681
682 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
683 auto allocation = last_bitrate_allocation_;
684 last_bitrate_allocation_.reset();
685 return allocation;
686 }
687
perkjfa10b552016-10-02 23:45:26 -0700688 private:
perkj26091b12016-09-01 01:17:40 -0700689 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100690 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700691 bool block_encode;
692 {
brandtre78d2662017-01-16 05:57:16 -0800693 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100694 if (expect_null_frame_) {
695 EXPECT_EQ(input_image.timestamp(), 0u);
696 EXPECT_EQ(input_image.width(), 1);
697 last_frame_types_ = *frame_types;
698 expect_null_frame_ = false;
699 } else {
700 EXPECT_GT(input_image.timestamp(), timestamp_);
701 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
702 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
703 }
perkj26091b12016-09-01 01:17:40 -0700704
705 timestamp_ = input_image.timestamp();
706 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700707 last_input_width_ = input_image.width();
708 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700709 block_encode = block_next_encode_;
710 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100711 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100712 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700713 }
Niels Möllerb859b322019-03-07 12:40:01 +0100714 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700715 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700716 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700717 return result;
718 }
719
sprangfe627f32017-03-29 08:24:59 -0700720 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200721 const Settings& settings) override {
722 int res = FakeEncoder::InitEncode(config, settings);
sprangfe627f32017-03-29 08:24:59 -0700723 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100724 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100725 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700726 // Simulate setting up temporal layers, in order to validate the life
727 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100728 Vp8TemporalLayersFactory factory;
729 frame_buffer_controller_ = factory.Create(*config);
sprangfe627f32017-03-29 08:24:59 -0700730 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100731 if (force_init_encode_failed_) {
732 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700733 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100734 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100735
Erik Språngb7cb7b52019-02-26 15:52:33 +0100736 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700737 return res;
738 }
739
Erik Språngb7cb7b52019-02-26 15:52:33 +0100740 int32_t Release() override {
741 rtc::CritScope lock(&local_crit_sect_);
742 EXPECT_NE(initialized_, EncoderState::kUninitialized);
743 initialized_ = EncoderState::kUninitialized;
744 return FakeEncoder::Release();
745 }
746
Erik Språng16cb8f52019-04-12 13:59:09 +0200747 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100748 rtc::CritScope lock(&local_crit_sect_);
749 VideoBitrateAllocation adjusted_rate_allocation;
750 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
751 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200752 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100753 adjusted_rate_allocation.SetBitrate(
754 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200755 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100756 rate_factor_));
757 }
758 }
759 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200760 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
761 last_bitrate_allocation_ = parameters.bitrate;
762 RateControlParameters adjusted_paramters = parameters;
763 adjusted_paramters.bitrate = adjusted_rate_allocation;
764 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100765 }
766
brandtre78d2662017-01-16 05:57:16 -0800767 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100768 enum class EncoderState {
769 kUninitialized,
770 kInitializationFailed,
771 kInitialized
772 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
773 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700774 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700775 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700776 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
777 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
778 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
779 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
780 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100781 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100782 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700783 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100784 absl::optional<bool>
785 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
786 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700787 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100788 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
789 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100790 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100791 VideoFrame::UpdateRect last_update_rect_
792 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100793 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100794 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100795 EncodedImageCallback* encoded_image_callback_
796 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700797 };
798
mflodmancc3d4422017-08-03 08:27:51 -0700799 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700800 public:
801 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100802 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700803
perkj26091b12016-09-01 01:17:40 -0700804 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700805 EXPECT_TRUE(
806 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
807 }
808
809 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
810 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700811 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700812 if (!encoded_frame_event_.Wait(timeout_ms))
813 return false;
perkj26091b12016-09-01 01:17:40 -0700814 {
815 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800816 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700817 }
818 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700819 return true;
perkj26091b12016-09-01 01:17:40 -0700820 }
821
sprangb1ca0732017-02-01 08:38:12 -0800822 void WaitForEncodedFrame(uint32_t expected_width,
823 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700824 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100825 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700826 }
827
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100828 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700829 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800830 uint32_t width = 0;
831 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800832 {
833 rtc::CritScope lock(&crit_);
834 width = last_width_;
835 height = last_height_;
836 }
837 EXPECT_EQ(expected_height, height);
838 EXPECT_EQ(expected_width, width);
839 }
840
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200841 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
842 VideoRotation rotation;
843 {
844 rtc::CritScope lock(&crit_);
845 rotation = last_rotation_;
846 }
847 EXPECT_EQ(expected_rotation, rotation);
848 }
849
kthelgason2fc52542017-03-03 00:24:41 -0800850 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800851
sprangc5d62e22017-04-02 23:53:04 -0700852 bool WaitForFrame(int64_t timeout_ms) {
853 return encoded_frame_event_.Wait(timeout_ms);
854 }
855
perkj26091b12016-09-01 01:17:40 -0700856 void SetExpectNoFrames() {
857 rtc::CritScope lock(&crit_);
858 expect_frames_ = false;
859 }
860
asaperssonfab67072017-04-04 05:51:49 -0700861 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200862 rtc::CritScope lock(&crit_);
863 return number_of_reconfigurations_;
864 }
865
asaperssonfab67072017-04-04 05:51:49 -0700866 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200867 rtc::CritScope lock(&crit_);
868 return min_transmit_bitrate_bps_;
869 }
870
Erik Språngd7329ca2019-02-21 21:19:53 +0100871 void SetNumExpectedLayers(size_t num_layers) {
872 rtc::CritScope lock(&crit_);
873 num_expected_layers_ = num_layers;
874 }
875
Erik Språngb7cb7b52019-02-26 15:52:33 +0100876 int64_t GetLastCaptureTimeMs() const {
877 rtc::CritScope lock(&crit_);
878 return last_capture_time_ms_;
879 }
880
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200881 std::vector<uint8_t> GetLastEncodedImageData() {
882 rtc::CritScope lock(&crit_);
883 return std::move(last_encoded_image_data_);
884 }
885
886 RTPFragmentationHeader GetLastFragmentation() {
887 rtc::CritScope lock(&crit_);
888 return std::move(last_fragmentation_);
889 }
890
perkj26091b12016-09-01 01:17:40 -0700891 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700892 Result OnEncodedImage(
893 const EncodedImage& encoded_image,
894 const CodecSpecificInfo* codec_specific_info,
895 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200896 rtc::CritScope lock(&crit_);
897 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200898 last_encoded_image_data_ = std::vector<uint8_t>(
899 encoded_image.data(), encoded_image.data() + encoded_image.size());
900 if (fragmentation) {
901 last_fragmentation_.CopyFrom(*fragmentation);
902 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100903 uint32_t timestamp = encoded_image.Timestamp();
904 if (last_timestamp_ != timestamp) {
905 num_received_layers_ = 1;
906 } else {
907 ++num_received_layers_;
908 }
909 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100910 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800911 last_width_ = encoded_image._encodedWidth;
912 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200913 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100914 if (num_received_layers_ == num_expected_layers_) {
915 encoded_frame_event_.Set();
916 }
sprangb1ca0732017-02-01 08:38:12 -0800917 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200918 }
919
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100920 void OnEncoderConfigurationChanged(
921 std::vector<VideoStream> streams,
922 VideoEncoderConfig::ContentType content_type,
923 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200924 rtc::CriticalSection crit_;
925 ++number_of_reconfigurations_;
926 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
927 }
928
perkj26091b12016-09-01 01:17:40 -0700929 rtc::CriticalSection crit_;
930 TestEncoder* test_encoder_;
931 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200932 std::vector<uint8_t> last_encoded_image_data_;
933 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -0800934 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100935 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800936 uint32_t last_height_ = 0;
937 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200938 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100939 size_t num_expected_layers_ = 1;
940 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700941 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200942 int number_of_reconfigurations_ = 0;
943 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700944 };
945
946 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100947 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200948 int codec_width_;
949 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700950 int max_framerate_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200951 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -0700952 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200953 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800954 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700955 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700956 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800957 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700958 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700959 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700960};
961
mflodmancc3d4422017-08-03 08:27:51 -0700962TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200963 video_stream_encoder_->OnBitrateUpdated(
964 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100965 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700966 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700967 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700968 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700969 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700970}
971
mflodmancc3d4422017-08-03 08:27:51 -0700972TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700973 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100974 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200975 // The encoder will cache up to one frame for a short duration. Adding two
976 // frames means that the first frame will be dropped and the second frame will
977 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700978 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200979 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700980 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700981
Erik Språng4c6ca302019-04-08 15:14:01 +0200982 video_stream_encoder_->OnBitrateUpdated(
983 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -0700984
Sebastian Janssona3177052018-04-10 13:05:49 +0200985 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700986 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200987 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
988
989 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700990 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700991}
992
mflodmancc3d4422017-08-03 08:27:51 -0700993TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200994 video_stream_encoder_->OnBitrateUpdated(
995 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700996 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700997 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700998
Erik Språng4c6ca302019-04-08 15:14:01 +0200999 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01001000 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001001 // The encoder will cache up to one frame for a short duration. Adding two
1002 // frames means that the first frame will be dropped and the second frame will
1003 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001004 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001005 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001006
Erik Språng4c6ca302019-04-08 15:14:01 +02001007 video_stream_encoder_->OnBitrateUpdated(
1008 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001009 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001010 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1011 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001012 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001013}
1014
mflodmancc3d4422017-08-03 08:27:51 -07001015TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001016 video_stream_encoder_->OnBitrateUpdated(
1017 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001018 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001019 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001020
1021 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001022 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001023
perkja49cbd32016-09-16 07:53:41 -07001024 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001025 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001026 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001027}
1028
mflodmancc3d4422017-08-03 08:27:51 -07001029TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001030 video_stream_encoder_->OnBitrateUpdated(
1031 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001032
perkja49cbd32016-09-16 07:53:41 -07001033 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001034 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001035
mflodmancc3d4422017-08-03 08:27:51 -07001036 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001037 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001038 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001039 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1040 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001041}
1042
mflodmancc3d4422017-08-03 08:27:51 -07001043TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001044 video_stream_encoder_->OnBitrateUpdated(
1045 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001046
1047 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001048 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001049 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001050 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1051 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001052 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1053 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001054 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001055 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001056
mflodmancc3d4422017-08-03 08:27:51 -07001057 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001058}
1059
mflodmancc3d4422017-08-03 08:27:51 -07001060TEST_F(VideoStreamEncoderTest,
1061 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001062 video_stream_encoder_->OnBitrateUpdated(
1063 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001064 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001065
1066 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001067 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001068 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001069 // The encoder will have been configured once when the first frame is
1070 // received.
1071 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001072
1073 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001074 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001075 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001076 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001077 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001078
1079 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001080 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001081 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001082 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001083 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001084
mflodmancc3d4422017-08-03 08:27:51 -07001085 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001086}
1087
mflodmancc3d4422017-08-03 08:27:51 -07001088TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001089 video_stream_encoder_->OnBitrateUpdated(
1090 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001091
1092 // Capture a frame and wait for it to synchronize with the encoder thread.
1093 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001094 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001095 // The encoder will have been configured once.
1096 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001097 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1098 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1099
1100 codec_width_ *= 2;
1101 codec_height_ *= 2;
1102 // Capture a frame with a higher resolution and wait for it to synchronize
1103 // with the encoder thread.
1104 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001105 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001106 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1107 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001108 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001109
mflodmancc3d4422017-08-03 08:27:51 -07001110 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001111}
1112
mflodmancc3d4422017-08-03 08:27:51 -07001113TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001114 EXPECT_TRUE(video_source_.has_sinks());
1115 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001116 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001117 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001118 EXPECT_FALSE(video_source_.has_sinks());
1119 EXPECT_TRUE(new_video_source.has_sinks());
1120
mflodmancc3d4422017-08-03 08:27:51 -07001121 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001122}
1123
mflodmancc3d4422017-08-03 08:27:51 -07001124TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001125 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001126 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001127 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001128 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001129}
1130
Jonathan Yubc771b72017-12-08 17:04:29 -08001131TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1132 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001133 const int kWidth = 1280;
1134 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001135
1136 // We rely on the automatic resolution adaptation, but we handle framerate
1137 // adaptation manually by mocking the stats proxy.
1138 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001139
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001140 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001141 video_stream_encoder_->OnBitrateUpdated(
1142 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001143 video_stream_encoder_->SetSource(&video_source_,
1144 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001145 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001146 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001147 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001148 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1149
Jonathan Yubc771b72017-12-08 17:04:29 -08001150 // Adapt down as far as possible.
1151 rtc::VideoSinkWants last_wants;
1152 int64_t t = 1;
1153 int loop_count = 0;
1154 do {
1155 ++loop_count;
1156 last_wants = video_source_.sink_wants();
1157
1158 // Simulate the framerate we've been asked to adapt to.
1159 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1160 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1161 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1162 mock_stats.input_frame_rate = fps;
1163 stats_proxy_->SetMockStats(mock_stats);
1164
1165 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1166 sink_.WaitForEncodedFrame(t);
1167 t += frame_interval_ms;
1168
mflodmancc3d4422017-08-03 08:27:51 -07001169 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001170 VerifyBalancedModeFpsRange(
1171 video_source_.sink_wants(),
1172 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1173 } while (video_source_.sink_wants().max_pixel_count <
1174 last_wants.max_pixel_count ||
1175 video_source_.sink_wants().max_framerate_fps <
1176 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001177
Jonathan Yubc771b72017-12-08 17:04:29 -08001178 // Verify that we've adapted all the way down.
1179 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001180 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001181 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1182 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001183 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001184 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1185 *video_source_.last_sent_height());
1186 EXPECT_EQ(kMinBalancedFramerateFps,
1187 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001188
Jonathan Yubc771b72017-12-08 17:04:29 -08001189 // Adapt back up the same number of times we adapted down.
1190 for (int i = 0; i < loop_count - 1; ++i) {
1191 last_wants = video_source_.sink_wants();
1192
1193 // Simulate the framerate we've been asked to adapt to.
1194 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1195 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1196 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1197 mock_stats.input_frame_rate = fps;
1198 stats_proxy_->SetMockStats(mock_stats);
1199
1200 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1201 sink_.WaitForEncodedFrame(t);
1202 t += frame_interval_ms;
1203
mflodmancc3d4422017-08-03 08:27:51 -07001204 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001205 VerifyBalancedModeFpsRange(
1206 video_source_.sink_wants(),
1207 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1208 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1209 last_wants.max_pixel_count ||
1210 video_source_.sink_wants().max_framerate_fps >
1211 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001212 }
1213
Åsa Persson8c1bf952018-09-13 10:42:19 +02001214 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001215 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001216 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001217 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1218 EXPECT_EQ((loop_count - 1) * 2,
1219 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001220
mflodmancc3d4422017-08-03 08:27:51 -07001221 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001222}
mflodmancc3d4422017-08-03 08:27:51 -07001223TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001224 video_stream_encoder_->OnBitrateUpdated(
1225 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001226 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001227
sprangc5d62e22017-04-02 23:53:04 -07001228 const int kFrameWidth = 1280;
1229 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001230
Åsa Persson8c1bf952018-09-13 10:42:19 +02001231 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001232
kthelgason5e13d412016-12-01 03:59:51 -08001233 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001234 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001235 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001236 frame_timestamp += kFrameIntervalMs;
1237
perkj803d97f2016-11-01 11:45:46 -07001238 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001239 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001240 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001241 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001242 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001243 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001244
asapersson0944a802017-04-07 00:57:58 -07001245 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001246 // wanted resolution.
1247 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1248 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1249 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001250 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001251
1252 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001253 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001254 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001255 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001256
sprangc5d62e22017-04-02 23:53:04 -07001257 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001258 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001259
sprangc5d62e22017-04-02 23:53:04 -07001260 // Force an input frame rate to be available, or the adaptation call won't
1261 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001262 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001263 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001264 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001265 stats_proxy_->SetMockStats(stats);
1266
mflodmancc3d4422017-08-03 08:27:51 -07001267 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001268 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001269 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001270 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001271 frame_timestamp += kFrameIntervalMs;
1272
1273 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001274 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001275 EXPECT_EQ(std::numeric_limits<int>::max(),
1276 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001277 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001278
asapersson02465b82017-04-10 01:12:52 -07001279 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001280 video_stream_encoder_->SetSource(&new_video_source,
1281 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001282 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001283
mflodmancc3d4422017-08-03 08:27:51 -07001284 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001285 new_video_source.IncomingCapturedFrame(
1286 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001287 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001288 frame_timestamp += kFrameIntervalMs;
1289
1290 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001291 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001292
1293 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001294 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001295 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001296 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1297 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001298 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001299 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001300
1301 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001302 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001303 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001304 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1305 EXPECT_EQ(std::numeric_limits<int>::max(),
1306 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001307 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001308
mflodmancc3d4422017-08-03 08:27:51 -07001309 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001310}
1311
mflodmancc3d4422017-08-03 08:27:51 -07001312TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001313 video_stream_encoder_->OnBitrateUpdated(
1314 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001315
asaperssonfab67072017-04-04 05:51:49 -07001316 const int kWidth = 1280;
1317 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001318 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001319 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001320 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1321 EXPECT_FALSE(stats.bw_limited_resolution);
1322 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1323
1324 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001325 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001326 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001327 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001328
1329 stats = stats_proxy_->GetStats();
1330 EXPECT_TRUE(stats.bw_limited_resolution);
1331 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1332
1333 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001334 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001335 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001336 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001337
1338 stats = stats_proxy_->GetStats();
1339 EXPECT_FALSE(stats.bw_limited_resolution);
1340 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1341 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1342
mflodmancc3d4422017-08-03 08:27:51 -07001343 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001344}
1345
mflodmancc3d4422017-08-03 08:27:51 -07001346TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001347 video_stream_encoder_->OnBitrateUpdated(
1348 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001349
1350 const int kWidth = 1280;
1351 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001352 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001353 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001354 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1355 EXPECT_FALSE(stats.cpu_limited_resolution);
1356 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1357
1358 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001359 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001360 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001361 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001362
1363 stats = stats_proxy_->GetStats();
1364 EXPECT_TRUE(stats.cpu_limited_resolution);
1365 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1366
1367 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001368 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001369 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001370 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001371
1372 stats = stats_proxy_->GetStats();
1373 EXPECT_FALSE(stats.cpu_limited_resolution);
1374 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001375 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001376
mflodmancc3d4422017-08-03 08:27:51 -07001377 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001378}
1379
mflodmancc3d4422017-08-03 08:27:51 -07001380TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001381 video_stream_encoder_->OnBitrateUpdated(
1382 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001383
asaperssonfab67072017-04-04 05:51:49 -07001384 const int kWidth = 1280;
1385 const int kHeight = 720;
1386 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001387 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001388 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001389 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001390 EXPECT_FALSE(stats.cpu_limited_resolution);
1391 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1392
asaperssonfab67072017-04-04 05:51:49 -07001393 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001394 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001395 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001396 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001397 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001398 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001399 EXPECT_TRUE(stats.cpu_limited_resolution);
1400 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1401
1402 // Set new source with adaptation still enabled.
1403 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001405 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001406
asaperssonfab67072017-04-04 05:51:49 -07001407 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001408 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001409 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001410 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001411 EXPECT_TRUE(stats.cpu_limited_resolution);
1412 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1413
1414 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001415 video_stream_encoder_->SetSource(&new_video_source,
1416 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001417
asaperssonfab67072017-04-04 05:51:49 -07001418 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001419 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001420 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001421 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001422 EXPECT_FALSE(stats.cpu_limited_resolution);
1423 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1424
1425 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001426 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001427 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001428
asaperssonfab67072017-04-04 05:51:49 -07001429 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001430 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001431 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001432 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001433 EXPECT_TRUE(stats.cpu_limited_resolution);
1434 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1435
asaperssonfab67072017-04-04 05:51:49 -07001436 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001437 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001438 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001439 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001440 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001441 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001442 EXPECT_FALSE(stats.cpu_limited_resolution);
1443 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001444 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001445
mflodmancc3d4422017-08-03 08:27:51 -07001446 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001447}
1448
mflodmancc3d4422017-08-03 08:27:51 -07001449TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001450 video_stream_encoder_->OnBitrateUpdated(
1451 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001452
asaperssonfab67072017-04-04 05:51:49 -07001453 const int kWidth = 1280;
1454 const int kHeight = 720;
1455 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001456 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001457 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001458 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001459 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001460 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001461
1462 // Set new source with adaptation still enabled.
1463 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001464 video_stream_encoder_->SetSource(&new_video_source,
1465 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001466
asaperssonfab67072017-04-04 05:51:49 -07001467 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001468 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001469 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001470 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001471 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001472 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001473
asaperssonfab67072017-04-04 05:51:49 -07001474 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001475 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001476 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001477 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001478 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001479 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001480 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001481 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001482
asaperssonfab67072017-04-04 05:51:49 -07001483 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001484 video_stream_encoder_->SetSource(&new_video_source,
1485 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001486
asaperssonfab67072017-04-04 05:51:49 -07001487 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001488 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001489 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001490 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001491 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001492 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001493
asapersson02465b82017-04-10 01:12:52 -07001494 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001495 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001496 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001497
asaperssonfab67072017-04-04 05:51:49 -07001498 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001499 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001500 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001501 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001502 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001503 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1504 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001505
mflodmancc3d4422017-08-03 08:27:51 -07001506 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001507}
1508
mflodmancc3d4422017-08-03 08:27:51 -07001509TEST_F(VideoStreamEncoderTest,
1510 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001511 video_stream_encoder_->OnBitrateUpdated(
1512 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001513
1514 const int kWidth = 1280;
1515 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001516 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001517 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001518 video_source_.IncomingCapturedFrame(
1519 CreateFrame(timestamp_ms, kWidth, kHeight));
1520 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001521 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1523 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1524
1525 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001526 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001527 timestamp_ms += kFrameIntervalMs;
1528 video_source_.IncomingCapturedFrame(
1529 CreateFrame(timestamp_ms, kWidth, kHeight));
1530 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1532 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1533 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1534
1535 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001536 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001537 timestamp_ms += kFrameIntervalMs;
1538 video_source_.IncomingCapturedFrame(
1539 CreateFrame(timestamp_ms, kWidth, kHeight));
1540 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001541 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1543 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1544
Niels Möller4db138e2018-04-19 09:04:13 +02001545 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001546 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001547
1548 VideoEncoderConfig video_encoder_config;
1549 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1550 // Make format different, to force recreation of encoder.
1551 video_encoder_config.video_format.parameters["foo"] = "foo";
1552 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001553 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001554 timestamp_ms += kFrameIntervalMs;
1555 video_source_.IncomingCapturedFrame(
1556 CreateFrame(timestamp_ms, kWidth, kHeight));
1557 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001558 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1559 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1560 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1561
mflodmancc3d4422017-08-03 08:27:51 -07001562 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001563}
1564
mflodmancc3d4422017-08-03 08:27:51 -07001565TEST_F(VideoStreamEncoderTest,
1566 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001567 video_stream_encoder_->OnBitrateUpdated(
1568 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001569
asapersson0944a802017-04-07 00:57:58 -07001570 const int kWidth = 1280;
1571 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001572 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001573
asaperssonfab67072017-04-04 05:51:49 -07001574 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001575 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001576 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001577 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001578 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001579 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1580
asapersson02465b82017-04-10 01:12:52 -07001581 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001582 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001583 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001584 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001585 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001586 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001587 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001588 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1589
1590 // Set new source with adaptation still enabled.
1591 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001592 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001593 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001594
1595 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001596 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001597 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001598 stats = stats_proxy_->GetStats();
1599 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001600 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001601 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1602
sprangc5d62e22017-04-02 23:53:04 -07001603 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001604 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001605 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001606 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001607 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001608 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001609 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001610 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001611 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001612 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001613 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1614
sprangc5d62e22017-04-02 23:53:04 -07001615 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001616 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001617 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1618 mock_stats.input_frame_rate = 30;
1619 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001620 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001621 stats_proxy_->ResetMockStats();
1622
1623 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001624 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001625 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001626
1627 // Framerate now adapted.
1628 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001629 EXPECT_FALSE(stats.cpu_limited_resolution);
1630 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001631 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1632
1633 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001634 video_stream_encoder_->SetSource(&new_video_source,
1635 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001636 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001637 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001638 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001639
1640 stats = stats_proxy_->GetStats();
1641 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001642 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001643 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1644
1645 // Try to trigger overuse. Should not succeed.
1646 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001647 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001648 stats_proxy_->ResetMockStats();
1649
1650 stats = stats_proxy_->GetStats();
1651 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001652 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001653 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1654
1655 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001656 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001657 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001658 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001659 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001660 stats = stats_proxy_->GetStats();
1661 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001662 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001663 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001664
1665 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001666 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001667 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001668 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001669 stats = stats_proxy_->GetStats();
1670 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001671 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001672 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1673
1674 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001675 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001676 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001677 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001678 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001679 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001680 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001681 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001682 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001683 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001684 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1685
1686 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001687 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001688 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001689 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001690 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001691 stats = stats_proxy_->GetStats();
1692 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001693 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001694 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001695 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001696
mflodmancc3d4422017-08-03 08:27:51 -07001697 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001698}
1699
mflodmancc3d4422017-08-03 08:27:51 -07001700TEST_F(VideoStreamEncoderTest,
1701 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001702 const int kWidth = 1280;
1703 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001704 video_stream_encoder_->OnBitrateUpdated(
1705 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001706
asaperssonfab67072017-04-04 05:51:49 -07001707 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001708 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001709
asaperssonfab67072017-04-04 05:51:49 -07001710 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001711 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001712
asaperssonfab67072017-04-04 05:51:49 -07001713 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001714 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001715
asaperssonfab67072017-04-04 05:51:49 -07001716 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001717 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001718
kthelgason876222f2016-11-29 01:44:11 -08001719 // Expect a scale down.
1720 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001721 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001722
asapersson02465b82017-04-10 01:12:52 -07001723 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001724 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001725 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001726 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001727
asaperssonfab67072017-04-04 05:51:49 -07001728 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001729 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001730 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001731 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001732
asaperssonfab67072017-04-04 05:51:49 -07001733 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001734 EXPECT_EQ(std::numeric_limits<int>::max(),
1735 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001736
asaperssonfab67072017-04-04 05:51:49 -07001737 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001738 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001739 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001740 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001741
asapersson02465b82017-04-10 01:12:52 -07001742 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001743 EXPECT_EQ(std::numeric_limits<int>::max(),
1744 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001745
mflodmancc3d4422017-08-03 08:27:51 -07001746 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001747}
1748
mflodmancc3d4422017-08-03 08:27:51 -07001749TEST_F(VideoStreamEncoderTest,
1750 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001751 const int kWidth = 1280;
1752 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001753 video_stream_encoder_->OnBitrateUpdated(
1754 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001755
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001756 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001757 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001758 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001759 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001760
1761 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001762 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001763 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1765 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1766
1767 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001768 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001769 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001770 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1771 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1772 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1773
1774 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001775 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001776 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1777 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1778 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1779
mflodmancc3d4422017-08-03 08:27:51 -07001780 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001781}
1782
mflodmancc3d4422017-08-03 08:27:51 -07001783TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001784 const int kWidth = 1280;
1785 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001786 video_stream_encoder_->OnBitrateUpdated(
1787 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001788
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001789 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001790 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001791 video_stream_encoder_->SetSource(&source,
1792 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001793 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1794 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001795 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001796
1797 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001798 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001799 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1800 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1801 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1802 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1803
1804 // Trigger adapt down for same input resolution, expect no change.
1805 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1806 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001807 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001808 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1809 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1810 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1811
1812 // Trigger adapt down for larger input resolution, expect no change.
1813 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1814 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001815 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001816 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1817 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1818 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1819
mflodmancc3d4422017-08-03 08:27:51 -07001820 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001821}
1822
mflodmancc3d4422017-08-03 08:27:51 -07001823TEST_F(VideoStreamEncoderTest,
1824 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001825 const int kWidth = 1280;
1826 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001827 video_stream_encoder_->OnBitrateUpdated(
1828 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001830 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001831 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001832 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001833 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001834
1835 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001836 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001837 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001838 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1839 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1840
1841 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001842 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001843 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1846
mflodmancc3d4422017-08-03 08:27:51 -07001847 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001848}
1849
mflodmancc3d4422017-08-03 08:27:51 -07001850TEST_F(VideoStreamEncoderTest,
1851 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001852 const int kWidth = 1280;
1853 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001854 video_stream_encoder_->OnBitrateUpdated(
1855 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001856
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001857 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001858 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001859 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001860 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001861
1862 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001863 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001864 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001866 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1867
1868 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001869 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001870 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001872 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1873
mflodmancc3d4422017-08-03 08:27:51 -07001874 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001875}
1876
mflodmancc3d4422017-08-03 08:27:51 -07001877TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001878 const int kWidth = 1280;
1879 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001880 video_stream_encoder_->OnBitrateUpdated(
1881 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001882
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001883 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001884 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001885 video_stream_encoder_->SetSource(&source,
1886 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001887
1888 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1889 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001890 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001891 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1892 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1893 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1894
1895 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001896 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001897 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1899 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1900 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1901
mflodmancc3d4422017-08-03 08:27:51 -07001902 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001903}
1904
mflodmancc3d4422017-08-03 08:27:51 -07001905TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001906 const int kWidth = 1280;
1907 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001908 video_stream_encoder_->OnBitrateUpdated(
1909 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001910
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001911 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001912 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001913 video_stream_encoder_->SetSource(&source,
1914 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001915
1916 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1917 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001918 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1921 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1922
1923 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001924 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001925 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001926 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1927 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1928 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1929
mflodmancc3d4422017-08-03 08:27:51 -07001930 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001931}
1932
mflodmancc3d4422017-08-03 08:27:51 -07001933TEST_F(VideoStreamEncoderTest,
1934 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001935 const int kWidth = 1280;
1936 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001937 video_stream_encoder_->OnBitrateUpdated(
1938 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001939
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001940 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001941 AdaptingFrameForwarder source;
1942 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001943 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001944 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001945
1946 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001947 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001948 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001949 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1950 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1951
1952 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001953 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001954 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001955 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001956 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001957 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1958 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1959
1960 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001961 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001962 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001963 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1964 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1965 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1966
mflodmancc3d4422017-08-03 08:27:51 -07001967 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001968}
1969
mflodmancc3d4422017-08-03 08:27:51 -07001970TEST_F(VideoStreamEncoderTest,
1971 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001972 const int kWidth = 1280;
1973 const int kHeight = 720;
1974 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02001975 video_stream_encoder_->OnBitrateUpdated(
1976 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001977
1978 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1979 stats.input_frame_rate = kInputFps;
1980 stats_proxy_->SetMockStats(stats);
1981
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001982 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001983 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1984 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001985 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001986
1987 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001988 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001989 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1990 sink_.WaitForEncodedFrame(2);
1991 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1992
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001993 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001994 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001995 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001996 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001997 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001998
1999 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002000 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002001 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2002 sink_.WaitForEncodedFrame(3);
2003 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2004
2005 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002006 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002007 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002008
mflodmancc3d4422017-08-03 08:27:51 -07002009 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002010}
2011
mflodmancc3d4422017-08-03 08:27:51 -07002012TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002013 const int kWidth = 1280;
2014 const int kHeight = 720;
2015 const size_t kNumFrames = 10;
2016
Erik Språng4c6ca302019-04-08 15:14:01 +02002017 video_stream_encoder_->OnBitrateUpdated(
2018 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002019
asaperssond0de2952017-04-21 01:47:31 -07002020 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002021 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002022 video_source_.set_adaptation_enabled(true);
2023
2024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2025 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2026
2027 int downscales = 0;
2028 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002029 video_source_.IncomingCapturedFrame(
2030 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2031 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002032
asaperssonfab67072017-04-04 05:51:49 -07002033 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002034 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002035 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002036 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002037
2038 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2039 ++downscales;
2040
2041 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2042 EXPECT_EQ(downscales,
2043 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2044 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002045 }
mflodmancc3d4422017-08-03 08:27:51 -07002046 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002047}
2048
mflodmancc3d4422017-08-03 08:27:51 -07002049TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002050 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2051 const int kWidth = 1280;
2052 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002053 video_stream_encoder_->OnBitrateUpdated(
2054 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002055
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002056 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002057 AdaptingFrameForwarder source;
2058 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002059 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002060 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002061
Åsa Persson8c1bf952018-09-13 10:42:19 +02002062 int64_t timestamp_ms = kFrameIntervalMs;
2063 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002064 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002065 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002066 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2067 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2068
2069 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002070 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002071 timestamp_ms += kFrameIntervalMs;
2072 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2073 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002074 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2076 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2077
2078 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002079 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002080 timestamp_ms += kFrameIntervalMs;
2081 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002082 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002083 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002084 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2085 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2086
2087 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002088 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002089 timestamp_ms += kFrameIntervalMs;
2090 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2091 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002092 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002093 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2094 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2095
2096 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002097 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002098 timestamp_ms += kFrameIntervalMs;
2099 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002100 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002101 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002102 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2103 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2104
mflodmancc3d4422017-08-03 08:27:51 -07002105 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002106}
2107
mflodmancc3d4422017-08-03 08:27:51 -07002108TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002109 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2110 const int kWidth = 1280;
2111 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002112 video_stream_encoder_->OnBitrateUpdated(
2113 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002114
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002115 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002116 AdaptingFrameForwarder source;
2117 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002118 video_stream_encoder_->SetSource(&source,
2119 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002120
Åsa Persson8c1bf952018-09-13 10:42:19 +02002121 int64_t timestamp_ms = kFrameIntervalMs;
2122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002123 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002124 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002125 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2126 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2127
2128 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002129 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002130 timestamp_ms += kFrameIntervalMs;
2131 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2132 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002133 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2134 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2135 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2136
2137 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002138 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002139 timestamp_ms += kFrameIntervalMs;
2140 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002141 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002142 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002143 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2144 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2145
2146 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002147 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002148 timestamp_ms += kFrameIntervalMs;
2149 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2150 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002151 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2152 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2153 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2154
2155 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002156 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002157 timestamp_ms += kFrameIntervalMs;
2158 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002159 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002160 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002161 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2162 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2163
mflodmancc3d4422017-08-03 08:27:51 -07002164 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002165}
2166
mflodmancc3d4422017-08-03 08:27:51 -07002167TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002168 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2169 const int kWidth = 1280;
2170 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002171 video_stream_encoder_->OnBitrateUpdated(
2172 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002173
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002174 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002175 AdaptingFrameForwarder source;
2176 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002177 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002178 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002179
Åsa Persson8c1bf952018-09-13 10:42:19 +02002180 int64_t timestamp_ms = kFrameIntervalMs;
2181 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002182 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002183 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002184 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2185 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2186 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2187 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2188
2189 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002190 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002191 timestamp_ms += kFrameIntervalMs;
2192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2193 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002194 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002195 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2196 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2197 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2198 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2199
2200 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002201 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002202 timestamp_ms += kFrameIntervalMs;
2203 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2204 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002205 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002206 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2207 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2208 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2209 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2210
Jonathan Yubc771b72017-12-08 17:04:29 -08002211 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002212 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002213 timestamp_ms += kFrameIntervalMs;
2214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2215 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002216 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002217 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2218 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002219 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002220 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2221
Jonathan Yubc771b72017-12-08 17:04:29 -08002222 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002223 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002224 timestamp_ms += kFrameIntervalMs;
2225 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2226 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002227 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002228 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002229 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2230 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2231 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2232 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2233
Jonathan Yubc771b72017-12-08 17:04:29 -08002234 // Trigger quality adapt down, expect no change (min resolution reached).
2235 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002236 timestamp_ms += kFrameIntervalMs;
2237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2238 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002239 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2240 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2241 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2242 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2243 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2244
2245 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002246 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002247 timestamp_ms += kFrameIntervalMs;
2248 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2249 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002250 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002251 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2252 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2253 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2254 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2255
2256 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2257 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002258 timestamp_ms += kFrameIntervalMs;
2259 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2260 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002261 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2262 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2263 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2264 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2265 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2266
2267 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2268 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002269 timestamp_ms += kFrameIntervalMs;
2270 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2271 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002272 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002273 last_wants = source.sink_wants();
2274 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2275 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002276 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2278
2279 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002280 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002281 timestamp_ms += kFrameIntervalMs;
2282 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2283 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002284 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002285 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002287 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002288 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2289
2290 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002291 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002292 timestamp_ms += kFrameIntervalMs;
2293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002294 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002295 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002296 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002297 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002299 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002300 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002301
mflodmancc3d4422017-08-03 08:27:51 -07002302 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002303}
2304
mflodmancc3d4422017-08-03 08:27:51 -07002305TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002306 const int kWidth = 640;
2307 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002308
Erik Språng4c6ca302019-04-08 15:14:01 +02002309 video_stream_encoder_->OnBitrateUpdated(
2310 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002311
perkj803d97f2016-11-01 11:45:46 -07002312 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002313 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002314 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002315 }
2316
mflodmancc3d4422017-08-03 08:27:51 -07002317 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002318 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002319 video_source_.IncomingCapturedFrame(CreateFrame(
2320 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002321 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002322 }
2323
mflodmancc3d4422017-08-03 08:27:51 -07002324 video_stream_encoder_->Stop();
2325 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002326 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002327
perkj803d97f2016-11-01 11:45:46 -07002328 EXPECT_EQ(1,
2329 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2330 EXPECT_EQ(
2331 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2332}
2333
mflodmancc3d4422017-08-03 08:27:51 -07002334TEST_F(VideoStreamEncoderTest,
2335 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002336 video_stream_encoder_->OnBitrateUpdated(
2337 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002338 const int kWidth = 640;
2339 const int kHeight = 360;
2340
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002341 video_stream_encoder_->SetSource(&video_source_,
2342 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002343
2344 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2345 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002346 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002347 }
2348
mflodmancc3d4422017-08-03 08:27:51 -07002349 video_stream_encoder_->Stop();
2350 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002351 stats_proxy_.reset();
2352
2353 EXPECT_EQ(0,
2354 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2355}
2356
mflodmancc3d4422017-08-03 08:27:51 -07002357TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002358 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002359 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002360
2361 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002362 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002363 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002364 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002365
sprang57c2fff2017-01-16 06:24:02 -08002366 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002367 .Times(1);
Erik Språng610c7632019-03-06 15:37:33 +01002368 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002369 DataRate::bps(kLowTargetBitrateBps),
2370 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002371
sprang57c2fff2017-01-16 06:24:02 -08002372 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002373 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2374 WaitForEncodedFrame(rtc::TimeMillis());
2375 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2376 fake_encoder_.GetAndResetLastBitrateAllocation();
2377 // Check that encoder has been updated too, not just allocation observer.
2378 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02002379 // TODO(srte): The use of millisecs here looks like an error, but the tests
2380 // fails using seconds, this should be investigated.
2381 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002382
2383 // Not called on second frame.
2384 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2385 .Times(0);
2386 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002387 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2388 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002389 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002390
2391 // Called after a process interval.
2392 const int64_t kProcessIntervalMs =
2393 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002394 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2395 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002396 const int64_t start_time_ms = rtc::TimeMillis();
2397 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2398 video_source_.IncomingCapturedFrame(
2399 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2400 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002401 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01002402 }
2403
2404 // Since rates are unchanged, encoder should not be reconfigured.
2405 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002406
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002408}
2409
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01002410TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
2411 // 2 TLs configured, temporal layers supported by encoder.
2412 const int kNumTemporalLayers = 2;
2413 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
2414 fake_encoder_.SetTemporalLayersSupported(0, true);
2415
2416 // Bitrate allocated across temporal layers.
2417 const int kTl0Bps = kTargetBitrateBps *
2418 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2419 kNumTemporalLayers, /*temporal_id*/ 0);
2420 const int kTl1Bps = kTargetBitrateBps *
2421 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2422 kNumTemporalLayers, /*temporal_id*/ 1);
2423 VideoBitrateAllocation expected_bitrate;
2424 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
2425 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
2426
2427 VerifyAllocatedBitrate(expected_bitrate);
2428 video_stream_encoder_->Stop();
2429}
2430
2431TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
2432 // 2 TLs configured, temporal layers not supported by encoder.
2433 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2434 fake_encoder_.SetTemporalLayersSupported(0, false);
2435
2436 // Temporal layers not supported by the encoder.
2437 // Total bitrate should be at ti:0.
2438 VideoBitrateAllocation expected_bitrate;
2439 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
2440
2441 VerifyAllocatedBitrate(expected_bitrate);
2442 video_stream_encoder_->Stop();
2443}
2444
2445TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
2446 // 2 TLs configured, temporal layers only supported for first stream.
2447 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2448 fake_encoder_.SetTemporalLayersSupported(0, true);
2449 fake_encoder_.SetTemporalLayersSupported(1, false);
2450
2451 const int kS0Bps = 150000;
2452 const int kS0Tl0Bps =
2453 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2454 /*num_layers*/ 2, /*temporal_id*/ 0);
2455 const int kS0Tl1Bps =
2456 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2457 /*num_layers*/ 2, /*temporal_id*/ 1);
2458 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
2459 // Temporal layers not supported by si:1.
2460 VideoBitrateAllocation expected_bitrate;
2461 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
2462 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
2463 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
2464
2465 VerifyAllocatedBitrate(expected_bitrate);
2466 video_stream_encoder_->Stop();
2467}
2468
Niels Möller7dc26b72017-12-06 10:27:48 +01002469TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2470 const int kFrameWidth = 1280;
2471 const int kFrameHeight = 720;
2472 const int kFramerate = 24;
2473
Erik Språng4c6ca302019-04-08 15:14:01 +02002474 video_stream_encoder_->OnBitrateUpdated(
2475 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002476 test::FrameForwarder source;
2477 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002478 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002479
2480 // Insert a single frame, triggering initial configuration.
2481 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2482 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2483
2484 EXPECT_EQ(
2485 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2486 kDefaultFramerate);
2487
2488 // Trigger reconfigure encoder (without resetting the entire instance).
2489 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002490 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002491 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2492 video_encoder_config.number_of_streams = 1;
2493 video_encoder_config.video_stream_factory =
2494 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2495 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002496 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002497 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2498
2499 // Detector should be updated with fps limit from codec config.
2500 EXPECT_EQ(
2501 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2502 kFramerate);
2503
2504 // Trigger overuse, max framerate should be reduced.
2505 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2506 stats.input_frame_rate = kFramerate;
2507 stats_proxy_->SetMockStats(stats);
2508 video_stream_encoder_->TriggerCpuOveruse();
2509 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2510 int adapted_framerate =
2511 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2512 EXPECT_LT(adapted_framerate, kFramerate);
2513
2514 // Trigger underuse, max framerate should go back to codec configured fps.
2515 // Set extra low fps, to make sure it's actually reset, not just incremented.
2516 stats = stats_proxy_->GetStats();
2517 stats.input_frame_rate = adapted_framerate / 2;
2518 stats_proxy_->SetMockStats(stats);
2519 video_stream_encoder_->TriggerCpuNormalUsage();
2520 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2521 EXPECT_EQ(
2522 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2523 kFramerate);
2524
2525 video_stream_encoder_->Stop();
2526}
2527
2528TEST_F(VideoStreamEncoderTest,
2529 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2530 const int kFrameWidth = 1280;
2531 const int kFrameHeight = 720;
2532 const int kLowFramerate = 15;
2533 const int kHighFramerate = 25;
2534
Erik Språng4c6ca302019-04-08 15:14:01 +02002535 video_stream_encoder_->OnBitrateUpdated(
2536 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002537 test::FrameForwarder source;
2538 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002539 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002540
2541 // Trigger initial configuration.
2542 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002543 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002544 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2545 video_encoder_config.number_of_streams = 1;
2546 video_encoder_config.video_stream_factory =
2547 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2548 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2549 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002550 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002551 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2552
2553 EXPECT_EQ(
2554 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2555 kLowFramerate);
2556
2557 // Trigger overuse, max framerate should be reduced.
2558 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2559 stats.input_frame_rate = kLowFramerate;
2560 stats_proxy_->SetMockStats(stats);
2561 video_stream_encoder_->TriggerCpuOveruse();
2562 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2563 int adapted_framerate =
2564 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2565 EXPECT_LT(adapted_framerate, kLowFramerate);
2566
2567 // Reconfigure the encoder with a new (higher max framerate), max fps should
2568 // still respect the adaptation.
2569 video_encoder_config.video_stream_factory =
2570 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2571 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2572 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002573 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002574 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2575
2576 EXPECT_EQ(
2577 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2578 adapted_framerate);
2579
2580 // Trigger underuse, max framerate should go back to codec configured fps.
2581 stats = stats_proxy_->GetStats();
2582 stats.input_frame_rate = adapted_framerate;
2583 stats_proxy_->SetMockStats(stats);
2584 video_stream_encoder_->TriggerCpuNormalUsage();
2585 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2586 EXPECT_EQ(
2587 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2588 kHighFramerate);
2589
2590 video_stream_encoder_->Stop();
2591}
2592
mflodmancc3d4422017-08-03 08:27:51 -07002593TEST_F(VideoStreamEncoderTest,
2594 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002595 const int kFrameWidth = 1280;
2596 const int kFrameHeight = 720;
2597 const int kFramerate = 24;
2598
Erik Språng4c6ca302019-04-08 15:14:01 +02002599 video_stream_encoder_->OnBitrateUpdated(
2600 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002601 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002602 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002603 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002604
2605 // Trigger initial configuration.
2606 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002607 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002608 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2609 video_encoder_config.number_of_streams = 1;
2610 video_encoder_config.video_stream_factory =
2611 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2612 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002613 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002614 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002615 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002616
Niels Möller7dc26b72017-12-06 10:27:48 +01002617 EXPECT_EQ(
2618 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2619 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002620
2621 // Trigger overuse, max framerate should be reduced.
2622 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2623 stats.input_frame_rate = kFramerate;
2624 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002625 video_stream_encoder_->TriggerCpuOveruse();
2626 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002627 int adapted_framerate =
2628 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002629 EXPECT_LT(adapted_framerate, kFramerate);
2630
2631 // Change degradation preference to not enable framerate scaling. Target
2632 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002633 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002634 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002635 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002636 EXPECT_EQ(
2637 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2638 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002639
mflodmancc3d4422017-08-03 08:27:51 -07002640 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002641}
2642
mflodmancc3d4422017-08-03 08:27:51 -07002643TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002644 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002645 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002646 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2647 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002648 const int kWidth = 640;
2649 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002650
asaperssonfab67072017-04-04 05:51:49 -07002651 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002652
2653 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002654 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002655
2656 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002657 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002658
sprangc5d62e22017-04-02 23:53:04 -07002659 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002660
asaperssonfab67072017-04-04 05:51:49 -07002661 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002662 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002663 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002664
2665 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002666 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002667
sprangc5d62e22017-04-02 23:53:04 -07002668 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002669
mflodmancc3d4422017-08-03 08:27:51 -07002670 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002671}
2672
mflodmancc3d4422017-08-03 08:27:51 -07002673TEST_F(VideoStreamEncoderTest,
2674 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002675 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002676 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002677 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2678 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002679 const int kWidth = 640;
2680 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002681
2682 // We expect the n initial frames to get dropped.
2683 int i;
2684 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002685 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002686 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002687 }
2688 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002689 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002690 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002691
2692 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002693 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002694
mflodmancc3d4422017-08-03 08:27:51 -07002695 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002696}
2697
mflodmancc3d4422017-08-03 08:27:51 -07002698TEST_F(VideoStreamEncoderTest,
2699 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002700 const int kWidth = 640;
2701 const int kHeight = 360;
Erik Språng610c7632019-03-06 15:37:33 +01002702 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002703 DataRate::bps(kLowTargetBitrateBps),
2704 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002705
2706 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002707 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002708 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002709
asaperssonfab67072017-04-04 05:51:49 -07002710 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002711 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002712 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002713
mflodmancc3d4422017-08-03 08:27:51 -07002714 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002715}
2716
mflodmancc3d4422017-08-03 08:27:51 -07002717TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002718 const int kWidth = 640;
2719 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002720 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002721
2722 VideoEncoderConfig video_encoder_config;
2723 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2724 // Make format different, to force recreation of encoder.
2725 video_encoder_config.video_format.parameters["foo"] = "foo";
2726 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002727 kMaxPayloadLength);
Erik Språng610c7632019-03-06 15:37:33 +01002728 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002729 DataRate::bps(kLowTargetBitrateBps),
2730 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002731
kthelgasonb83797b2017-02-14 11:57:25 -08002732 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002733 video_stream_encoder_->SetSource(&video_source_,
2734 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002735
asaperssonfab67072017-04-04 05:51:49 -07002736 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002737 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002738 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002739
mflodmancc3d4422017-08-03 08:27:51 -07002740 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002741 fake_encoder_.SetQualityScaling(true);
2742}
2743
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002744TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2745 webrtc::test::ScopedFieldTrials field_trials(
2746 "WebRTC-InitialFramedrop/Enabled/");
2747 // Reset encoder for field trials to take effect.
2748 ConfigureEncoder(video_encoder_config_.Copy());
2749 const int kTooLowBitrateForFrameSizeBps = 10000;
2750 const int kWidth = 640;
2751 const int kHeight = 360;
2752
Erik Språng4c6ca302019-04-08 15:14:01 +02002753 video_stream_encoder_->OnBitrateUpdated(
2754 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002755 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2756 // Frame should not be dropped.
2757 WaitForEncodedFrame(1);
2758
Erik Språng610c7632019-03-06 15:37:33 +01002759 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002760 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2761 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002762 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2763 // Expect to drop this frame, the wait should time out.
2764 ExpectDroppedFrame();
2765
2766 // Expect the sink_wants to specify a scaled frame.
2767 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2768 video_stream_encoder_->Stop();
2769}
2770
mflodmancc3d4422017-08-03 08:27:51 -07002771TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002772 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2773 const int kTooSmallWidth = 10;
2774 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02002775 video_stream_encoder_->OnBitrateUpdated(
2776 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002777
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002779 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002780 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002781 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002782 VerifyNoLimitation(source.sink_wants());
2783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2784
2785 // Trigger adapt down, too small frame, expect no change.
2786 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002787 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002788 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002789 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002790 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2791 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2792
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002794}
2795
mflodmancc3d4422017-08-03 08:27:51 -07002796TEST_F(VideoStreamEncoderTest,
2797 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002798 const int kTooSmallWidth = 10;
2799 const int kTooSmallHeight = 10;
2800 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02002801 video_stream_encoder_->OnBitrateUpdated(
2802 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002803
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002804 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002805 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002806 video_stream_encoder_->SetSource(&source,
2807 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002808 VerifyNoLimitation(source.sink_wants());
2809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2811
2812 // Trigger adapt down, expect limited framerate.
2813 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002814 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002815 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002816 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2818 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2819 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2820
2821 // Trigger adapt down, too small frame, expect no change.
2822 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002823 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002824 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002825 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2826 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2827 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2828 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2829
mflodmancc3d4422017-08-03 08:27:51 -07002830 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002831}
2832
mflodmancc3d4422017-08-03 08:27:51 -07002833TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002834 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02002835 video_stream_encoder_->OnBitrateUpdated(
2836 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002837 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002838 const int kFrameWidth = 1280;
2839 const int kFrameHeight = 720;
2840 video_source_.IncomingCapturedFrame(
2841 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002842 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002843 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002844}
2845
sprangb1ca0732017-02-01 08:38:12 -08002846// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002847TEST_F(VideoStreamEncoderTest,
2848 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002849 video_stream_encoder_->OnBitrateUpdated(
2850 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002851
2852 const int kFrameWidth = 1280;
2853 const int kFrameHeight = 720;
2854 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002855 // requested by
2856 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002857 video_source_.set_adaptation_enabled(true);
2858
2859 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002860 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002861 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002862
2863 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002864 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002865 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002866 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002867 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002868
asaperssonfab67072017-04-04 05:51:49 -07002869 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002870 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002871 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002872 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002874
mflodmancc3d4422017-08-03 08:27:51 -07002875 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002876}
sprangfe627f32017-03-29 08:24:59 -07002877
mflodmancc3d4422017-08-03 08:27:51 -07002878TEST_F(VideoStreamEncoderTest,
2879 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002880 const int kFrameWidth = 1280;
2881 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002882
Erik Språng4c6ca302019-04-08 15:14:01 +02002883 video_stream_encoder_->OnBitrateUpdated(
2884 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002885 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002886 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002887 video_source_.set_adaptation_enabled(true);
2888
sprang4847ae62017-06-27 07:06:52 -07002889 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002890
2891 video_source_.IncomingCapturedFrame(
2892 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002893 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002894
2895 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002896 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002897
2898 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002899 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002900 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002901 video_source_.IncomingCapturedFrame(
2902 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002904 }
2905
2906 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002907 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002908 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002909 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002910 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002911 video_source_.IncomingCapturedFrame(
2912 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002914 ++num_frames_dropped;
2915 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002916 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002917 }
2918 }
2919
sprang4847ae62017-06-27 07:06:52 -07002920 // Add some slack to account for frames dropped by the frame dropper.
2921 const int kErrorMargin = 1;
2922 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002923 kErrorMargin);
2924
2925 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002927 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002928 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002929 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002930 video_source_.IncomingCapturedFrame(
2931 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002932 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002933 ++num_frames_dropped;
2934 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002935 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002936 }
2937 }
sprang4847ae62017-06-27 07:06:52 -07002938 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002939 kErrorMargin);
2940
2941 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002943 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002944 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002945 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002946 video_source_.IncomingCapturedFrame(
2947 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002948 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002949 ++num_frames_dropped;
2950 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002951 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002952 }
2953 }
sprang4847ae62017-06-27 07:06:52 -07002954 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002955 kErrorMargin);
2956
2957 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002958 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002959 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002960 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002961 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002962 video_source_.IncomingCapturedFrame(
2963 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002964 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002965 ++num_frames_dropped;
2966 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002967 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002968 }
2969 }
2970 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2971
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002973}
2974
mflodmancc3d4422017-08-03 08:27:51 -07002975TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002976 const int kFramerateFps = 5;
2977 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002978 const int kFrameWidth = 1280;
2979 const int kFrameHeight = 720;
2980
sprang4847ae62017-06-27 07:06:52 -07002981 // Reconfigure encoder with two temporal layers and screensharing, which will
2982 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002983 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002984
Erik Språng4c6ca302019-04-08 15:14:01 +02002985 video_stream_encoder_->OnBitrateUpdated(
2986 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002987 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002988 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002989 video_source_.set_adaptation_enabled(true);
2990
sprang4847ae62017-06-27 07:06:52 -07002991 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002992
2993 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002994 rtc::VideoSinkWants last_wants;
2995 do {
2996 last_wants = video_source_.sink_wants();
2997
sprangc5d62e22017-04-02 23:53:04 -07002998 // Insert frames to get a new fps estimate...
2999 for (int j = 0; j < kFramerateFps; ++j) {
3000 video_source_.IncomingCapturedFrame(
3001 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003002 if (video_source_.last_sent_width()) {
3003 sink_.WaitForEncodedFrame(timestamp_ms);
3004 }
sprangc5d62e22017-04-02 23:53:04 -07003005 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003006 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003007 }
3008 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003009 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003010 } while (video_source_.sink_wants().max_framerate_fps <
3011 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003012
Jonathan Yubc771b72017-12-08 17:04:29 -08003013 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003014
mflodmancc3d4422017-08-03 08:27:51 -07003015 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003016}
asaperssonf7e294d2017-06-13 23:25:22 -07003017
mflodmancc3d4422017-08-03 08:27:51 -07003018TEST_F(VideoStreamEncoderTest,
3019 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003020 const int kWidth = 1280;
3021 const int kHeight = 720;
3022 const int64_t kFrameIntervalMs = 150;
3023 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003024 video_stream_encoder_->OnBitrateUpdated(
3025 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003026
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003027 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003028 AdaptingFrameForwarder source;
3029 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003030 video_stream_encoder_->SetSource(&source,
3031 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003032 timestamp_ms += kFrameIntervalMs;
3033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003034 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003035 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3038 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3039
3040 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003042 timestamp_ms += kFrameIntervalMs;
3043 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003044 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003045 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3046 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3047 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3048 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3049
3050 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003051 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003052 timestamp_ms += kFrameIntervalMs;
3053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003054 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003055 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3056 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3057 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3058 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3059
3060 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003061 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003062 timestamp_ms += kFrameIntervalMs;
3063 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003064 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003065 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3066 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3067 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3068 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3069
3070 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003071 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003072 timestamp_ms += kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003074 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003075 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3078 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3079
3080 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003081 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003082 timestamp_ms += kFrameIntervalMs;
3083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003084 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003085 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3088 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003091 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003092 timestamp_ms += kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003094 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003095 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3096 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3097 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3098 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3099
3100 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003101 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003102 timestamp_ms += kFrameIntervalMs;
3103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003104 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003105 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3106 rtc::VideoSinkWants last_wants = source.sink_wants();
3107 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3108 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3109 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3110
3111 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003113 timestamp_ms += kFrameIntervalMs;
3114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003116 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3117 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3118 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3119 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3120
3121 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003122 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003123 timestamp_ms += kFrameIntervalMs;
3124 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003125 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003126 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3127 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3128 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3129 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3130
3131 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003132 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003133 timestamp_ms += kFrameIntervalMs;
3134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003135 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003136 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3137 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3138 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3139 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3140
3141 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003142 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003143 timestamp_ms += kFrameIntervalMs;
3144 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003145 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003146 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3147 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3148 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3149 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3150
3151 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003153 timestamp_ms += kFrameIntervalMs;
3154 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003156 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3157 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3158 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3159 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3160
3161 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003163 timestamp_ms += kFrameIntervalMs;
3164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003165 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003166 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3169 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003173 timestamp_ms += kFrameIntervalMs;
3174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003175 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003176 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3177 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3179 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3180
3181 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003182 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003183 timestamp_ms += kFrameIntervalMs;
3184 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003185 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003186 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003187 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003188 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3190 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3191
3192 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003194 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003195 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3196
mflodmancc3d4422017-08-03 08:27:51 -07003197 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003198}
3199
mflodmancc3d4422017-08-03 08:27:51 -07003200TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003201 const int kWidth = 1280;
3202 const int kHeight = 720;
3203 const int64_t kFrameIntervalMs = 150;
3204 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003205 video_stream_encoder_->OnBitrateUpdated(
3206 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003207
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003208 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003209 AdaptingFrameForwarder source;
3210 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003211 video_stream_encoder_->SetSource(&source,
3212 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003213 timestamp_ms += kFrameIntervalMs;
3214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003215 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003216 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003217 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3218 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3219 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3220 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3221 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3222 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3223
3224 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003225 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003226 timestamp_ms += kFrameIntervalMs;
3227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003228 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003229 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3230 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3232 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3233 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3234 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3235 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3236
3237 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003238 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003239 timestamp_ms += kFrameIntervalMs;
3240 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003241 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003242 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3243 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3245 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3246 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3247 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3248 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3249
3250 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003251 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003252 timestamp_ms += kFrameIntervalMs;
3253 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003254 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003255 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3256 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3257 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3258 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3259 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3260 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3261 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3262
3263 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003265 timestamp_ms += kFrameIntervalMs;
3266 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003267 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003268 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3269 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3271 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3273 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3274 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3275
3276 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003277 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003278 timestamp_ms += kFrameIntervalMs;
3279 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003280 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003281 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3282 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3283 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3284 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3285 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3286 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3287 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3288
3289 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003290 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003291 timestamp_ms += kFrameIntervalMs;
3292 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003293 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003294 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003295 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3298 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3299 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3300 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3301 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3302
3303 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003304 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003305 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003306 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3307 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003310}
3311
mflodmancc3d4422017-08-03 08:27:51 -07003312TEST_F(VideoStreamEncoderTest,
3313 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003314 const int kWidth = 640;
3315 const int kHeight = 360;
3316 const int kFpsLimit = 15;
3317 const int64_t kFrameIntervalMs = 150;
3318 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003319 video_stream_encoder_->OnBitrateUpdated(
3320 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003321
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003322 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003323 AdaptingFrameForwarder source;
3324 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003325 video_stream_encoder_->SetSource(&source,
3326 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003327 timestamp_ms += kFrameIntervalMs;
3328 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003329 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003330 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003331 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3332 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3333 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3334 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3335 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3336 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3337
3338 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003340 timestamp_ms += kFrameIntervalMs;
3341 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003342 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003343 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3346 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3347 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3348 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3349 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3350
3351 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003352 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003353 timestamp_ms += kFrameIntervalMs;
3354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003355 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003356 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3358 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3359 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3360 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3361 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3362 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003365 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003366 timestamp_ms += kFrameIntervalMs;
3367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003368 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003369 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3370 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3372 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3373 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3374 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3375 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3376
3377 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003378 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003379 timestamp_ms += kFrameIntervalMs;
3380 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003381 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003382 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003383 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3385 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3386 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3387 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3388 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003391 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003392 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003393 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3394 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3395
mflodmancc3d4422017-08-03 08:27:51 -07003396 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003397}
3398
mflodmancc3d4422017-08-03 08:27:51 -07003399TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003400 // Simulates simulcast behavior and makes highest stream resolutions divisible
3401 // by 4.
3402 class CroppingVideoStreamFactory
3403 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3404 public:
3405 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3406 int framerate)
3407 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3408 EXPECT_GT(num_temporal_layers, 0u);
3409 EXPECT_GT(framerate, 0);
3410 }
3411
3412 private:
3413 std::vector<VideoStream> CreateEncoderStreams(
3414 int width,
3415 int height,
3416 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003417 std::vector<VideoStream> streams = test::CreateVideoStreams(
3418 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003419 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003420 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003421 stream.max_framerate = framerate_;
3422 }
3423 return streams;
3424 }
3425
3426 const size_t num_temporal_layers_;
3427 const int framerate_;
3428 };
3429
3430 const int kFrameWidth = 1920;
3431 const int kFrameHeight = 1080;
3432 // 3/4 of 1920.
3433 const int kAdaptedFrameWidth = 1440;
3434 // 3/4 of 1080 rounded down to multiple of 4.
3435 const int kAdaptedFrameHeight = 808;
3436 const int kFramerate = 24;
3437
Erik Språng4c6ca302019-04-08 15:14:01 +02003438 video_stream_encoder_->OnBitrateUpdated(
3439 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003440 // Trigger reconfigure encoder (without resetting the entire instance).
3441 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003442 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003443 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3444 video_encoder_config.number_of_streams = 1;
3445 video_encoder_config.video_stream_factory =
3446 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003447 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003448 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003449 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003450
3451 video_source_.set_adaptation_enabled(true);
3452
3453 video_source_.IncomingCapturedFrame(
3454 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003455 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003456
3457 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003458 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003459 video_source_.IncomingCapturedFrame(
3460 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003461 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003462
mflodmancc3d4422017-08-03 08:27:51 -07003463 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003464}
3465
mflodmancc3d4422017-08-03 08:27:51 -07003466TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003467 const int kFrameWidth = 1280;
3468 const int kFrameHeight = 720;
3469 const int kLowFps = 2;
3470 const int kHighFps = 30;
3471
Erik Språng4c6ca302019-04-08 15:14:01 +02003472 video_stream_encoder_->OnBitrateUpdated(
3473 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003474
3475 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3476 max_framerate_ = kLowFps;
3477
3478 // Insert 2 seconds of 2fps video.
3479 for (int i = 0; i < kLowFps * 2; ++i) {
3480 video_source_.IncomingCapturedFrame(
3481 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3482 WaitForEncodedFrame(timestamp_ms);
3483 timestamp_ms += 1000 / kLowFps;
3484 }
3485
3486 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02003487 video_stream_encoder_->OnBitrateUpdated(
3488 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003489 video_source_.IncomingCapturedFrame(
3490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3491 WaitForEncodedFrame(timestamp_ms);
3492 timestamp_ms += 1000 / kLowFps;
3493
3494 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3495
3496 // Insert 30fps frames for just a little more than the forced update period.
3497 const int kVcmTimerIntervalFrames =
3498 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3499 const int kFrameIntervalMs = 1000 / kHighFps;
3500 max_framerate_ = kHighFps;
3501 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3502 video_source_.IncomingCapturedFrame(
3503 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3504 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3505 // be dropped if the encoder hans't been updated with the new higher target
3506 // framerate yet, causing it to overshoot the target bitrate and then
3507 // suffering the wrath of the media optimizer.
3508 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3509 timestamp_ms += kFrameIntervalMs;
3510 }
3511
3512 // Don expect correct measurement just yet, but it should be higher than
3513 // before.
3514 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3515
mflodmancc3d4422017-08-03 08:27:51 -07003516 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003517}
3518
mflodmancc3d4422017-08-03 08:27:51 -07003519TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003520 const int kFrameWidth = 1280;
3521 const int kFrameHeight = 720;
3522 const int kTargetBitrateBps = 1000000;
3523
3524 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003525 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02003526 video_stream_encoder_->OnBitrateUpdated(
3527 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003528 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003529
3530 // Insert a first video frame, causes another bitrate update.
3531 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3532 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3533 video_source_.IncomingCapturedFrame(
3534 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3535 WaitForEncodedFrame(timestamp_ms);
3536
3537 // Next, simulate video suspension due to pacer queue overrun.
Erik Språng4c6ca302019-04-08 15:14:01 +02003538 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01003539 1);
sprang4847ae62017-06-27 07:06:52 -07003540
3541 // Skip ahead until a new periodic parameter update should have occured.
3542 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003543 fake_clock_.AdvanceTime(
3544 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07003545
3546 // Bitrate observer should not be called.
3547 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3548 video_source_.IncomingCapturedFrame(
3549 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3550 ExpectDroppedFrame();
3551
mflodmancc3d4422017-08-03 08:27:51 -07003552 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003553}
ilnik6b826ef2017-06-16 06:53:48 -07003554
Niels Möller4db138e2018-04-19 09:04:13 +02003555TEST_F(VideoStreamEncoderTest,
3556 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3557 const int kFrameWidth = 1280;
3558 const int kFrameHeight = 720;
3559 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02003560 video_stream_encoder_->OnBitrateUpdated(
3561 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003562 video_source_.IncomingCapturedFrame(
3563 CreateFrame(1, kFrameWidth, kFrameHeight));
3564 WaitForEncodedFrame(1);
3565 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3566 .low_encode_usage_threshold_percent,
3567 default_options.low_encode_usage_threshold_percent);
3568 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3569 .high_encode_usage_threshold_percent,
3570 default_options.high_encode_usage_threshold_percent);
3571 video_stream_encoder_->Stop();
3572}
3573
3574TEST_F(VideoStreamEncoderTest,
3575 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3576 const int kFrameWidth = 1280;
3577 const int kFrameHeight = 720;
3578 CpuOveruseOptions hardware_options;
3579 hardware_options.low_encode_usage_threshold_percent = 150;
3580 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003581 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003582
Erik Språng4c6ca302019-04-08 15:14:01 +02003583 video_stream_encoder_->OnBitrateUpdated(
3584 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003585 video_source_.IncomingCapturedFrame(
3586 CreateFrame(1, kFrameWidth, kFrameHeight));
3587 WaitForEncodedFrame(1);
3588 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3589 .low_encode_usage_threshold_percent,
3590 hardware_options.low_encode_usage_threshold_percent);
3591 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3592 .high_encode_usage_threshold_percent,
3593 hardware_options.high_encode_usage_threshold_percent);
3594 video_stream_encoder_->Stop();
3595}
3596
Niels Möller6bb5ab92019-01-11 11:11:10 +01003597TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3598 const int kFrameWidth = 320;
3599 const int kFrameHeight = 240;
3600 const int kFps = 30;
3601 const int kTargetBitrateBps = 120000;
3602 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3603
Erik Språng4c6ca302019-04-08 15:14:01 +02003604 video_stream_encoder_->OnBitrateUpdated(
3605 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003606
3607 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3608 max_framerate_ = kFps;
3609
3610 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3611 fake_encoder_.SimulateOvershoot(1.0);
3612 int num_dropped = 0;
3613 for (int i = 0; i < kNumFramesInRun; ++i) {
3614 video_source_.IncomingCapturedFrame(
3615 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3616 // Wait up to two frame durations for a frame to arrive.
3617 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3618 ++num_dropped;
3619 }
3620 timestamp_ms += 1000 / kFps;
3621 }
3622
Erik Språnga8d48ab2019-02-08 14:17:40 +01003623 // Framerate should be measured to be near the expected target rate.
3624 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3625
3626 // Frame drops should be within 5% of expected 0%.
3627 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003628
3629 // Make encoder produce frames at double the expected bitrate during 3 seconds
3630 // of video, verify number of drops. Rate needs to be slightly changed in
3631 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003632 double overshoot_factor = 2.0;
3633 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3634 // With bitrate adjuster, when need to overshoot even more to trigger
3635 // frame dropping.
3636 overshoot_factor *= 2;
3637 }
3638 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01003639 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003640 DataRate::bps(kTargetBitrateBps + 1000),
3641 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003642 num_dropped = 0;
3643 for (int i = 0; i < kNumFramesInRun; ++i) {
3644 video_source_.IncomingCapturedFrame(
3645 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3646 // Wait up to two frame durations for a frame to arrive.
3647 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3648 ++num_dropped;
3649 }
3650 timestamp_ms += 1000 / kFps;
3651 }
3652
Erik Språng4c6ca302019-04-08 15:14:01 +02003653 video_stream_encoder_->OnBitrateUpdated(
3654 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01003655
3656 // Target framerate should be still be near the expected target, despite
3657 // the frame drops.
3658 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3659
3660 // Frame drops should be within 5% of expected 50%.
3661 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003662
3663 video_stream_encoder_->Stop();
3664}
3665
3666TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3667 const int kFrameWidth = 320;
3668 const int kFrameHeight = 240;
3669 const int kActualInputFps = 24;
3670 const int kTargetBitrateBps = 120000;
3671
3672 ASSERT_GT(max_framerate_, kActualInputFps);
3673
3674 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3675 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02003676 video_stream_encoder_->OnBitrateUpdated(
3677 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003678
3679 // Insert 3 seconds of video, with an input fps lower than configured max.
3680 for (int i = 0; i < kActualInputFps * 3; ++i) {
3681 video_source_.IncomingCapturedFrame(
3682 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3683 // Wait up to two frame durations for a frame to arrive.
3684 WaitForEncodedFrame(timestamp_ms);
3685 timestamp_ms += 1000 / kActualInputFps;
3686 }
3687
3688 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3689
3690 video_stream_encoder_->Stop();
3691}
3692
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003693TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3694 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02003695 video_stream_encoder_->OnBitrateUpdated(
3696 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003697
3698 fake_encoder_.BlockNextEncode();
3699 video_source_.IncomingCapturedFrame(
3700 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3701 WaitForEncodedFrame(1);
3702 // On the very first frame full update should be forced.
3703 rect = fake_encoder_.GetLastUpdateRect();
3704 EXPECT_EQ(rect.offset_x, 0);
3705 EXPECT_EQ(rect.offset_y, 0);
3706 EXPECT_EQ(rect.height, codec_height_);
3707 EXPECT_EQ(rect.width, codec_width_);
3708 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3709 // call to ContinueEncode.
3710 video_source_.IncomingCapturedFrame(
3711 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3712 ExpectDroppedFrame();
3713 video_source_.IncomingCapturedFrame(
3714 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3715 ExpectDroppedFrame();
3716 fake_encoder_.ContinueEncode();
3717 WaitForEncodedFrame(3);
3718 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3719 rect = fake_encoder_.GetLastUpdateRect();
3720 EXPECT_EQ(rect.offset_x, 1);
3721 EXPECT_EQ(rect.offset_y, 0);
3722 EXPECT_EQ(rect.width, 10);
3723 EXPECT_EQ(rect.height, 1);
3724
3725 video_source_.IncomingCapturedFrame(
3726 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3727 WaitForEncodedFrame(4);
3728 // Previous frame was encoded, so no accumulation should happen.
3729 rect = fake_encoder_.GetLastUpdateRect();
3730 EXPECT_EQ(rect.offset_x, 0);
3731 EXPECT_EQ(rect.offset_y, 0);
3732 EXPECT_EQ(rect.width, 1);
3733 EXPECT_EQ(rect.height, 1);
3734
3735 video_stream_encoder_->Stop();
3736}
3737
Erik Språngd7329ca2019-02-21 21:19:53 +01003738TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003739 video_stream_encoder_->OnBitrateUpdated(
3740 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003741
3742 // First frame is always keyframe.
3743 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3744 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01003745 EXPECT_THAT(
3746 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003747 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003748
3749 // Insert delta frame.
3750 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3751 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01003752 EXPECT_THAT(
3753 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003754 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003755
3756 // Request next frame be a key-frame.
3757 video_stream_encoder_->SendKeyFrame();
3758 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3759 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01003760 EXPECT_THAT(
3761 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003762 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003763
3764 video_stream_encoder_->Stop();
3765}
3766
3767TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3768 // Setup simulcast with three streams.
3769 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01003770 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003771 DataRate::bps(kSimulcastTargetBitrateBps),
3772 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003773 // Wait for all three layers before triggering event.
3774 sink_.SetNumExpectedLayers(3);
3775
3776 // First frame is always keyframe.
3777 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3778 WaitForEncodedFrame(1);
3779 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003780 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
3781 VideoFrameType::kVideoFrameKey,
3782 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003783
3784 // Insert delta frame.
3785 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3786 WaitForEncodedFrame(2);
3787 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003788 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
3789 VideoFrameType::kVideoFrameDelta,
3790 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003791
3792 // Request next frame be a key-frame.
3793 // Only first stream is configured to produce key-frame.
3794 video_stream_encoder_->SendKeyFrame();
3795 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3796 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02003797
3798 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
3799 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01003800 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003801 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02003802 VideoFrameType::kVideoFrameKey,
3803 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003804
3805 video_stream_encoder_->Stop();
3806}
3807
3808TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3809 // Configure internal source factory and setup test again.
3810 encoder_factory_.SetHasInternalSource(true);
3811 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003812 video_stream_encoder_->OnBitrateUpdated(
3813 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003814
3815 // Call encoder directly, simulating internal source where encoded frame
3816 // callback in VideoStreamEncoder is called despite no OnFrame().
3817 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3818 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003819 EXPECT_THAT(
3820 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003821 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003822
Niels Möller8f7ce222019-03-21 15:43:58 +01003823 const std::vector<VideoFrameType> kDeltaFrame = {
3824 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01003825 // Need to set timestamp manually since manually for injected frame.
3826 VideoFrame frame = CreateFrame(101, nullptr);
3827 frame.set_timestamp(101);
3828 fake_encoder_.InjectFrame(frame, false);
3829 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003830 EXPECT_THAT(
3831 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003832 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003833
3834 // Request key-frame. The forces a dummy frame down into the encoder.
3835 fake_encoder_.ExpectNullFrame();
3836 video_stream_encoder_->SendKeyFrame();
3837 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003838 EXPECT_THAT(
3839 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003840 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003841
3842 video_stream_encoder_->Stop();
3843}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003844
3845TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3846 // Configure internal source factory and setup test again.
3847 encoder_factory_.SetHasInternalSource(true);
3848 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003849 video_stream_encoder_->OnBitrateUpdated(
3850 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01003851
3852 int64_t timestamp = 1;
3853 EncodedImage image;
3854 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3855 image.capture_time_ms_ = ++timestamp;
3856 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3857 const int64_t kEncodeFinishDelayMs = 10;
3858 image.timing_.encode_start_ms = timestamp;
3859 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3860 fake_encoder_.InjectEncodedImage(image);
3861 // Wait for frame without incrementing clock.
3862 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3863 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3864 // capture timestamp should be kEncodeFinishDelayMs in the past.
3865 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3866 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3867 kEncodeFinishDelayMs);
3868
3869 video_stream_encoder_->Stop();
3870}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02003871
3872TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
3873 // Configure internal source factory and setup test again.
3874 encoder_factory_.SetHasInternalSource(true);
3875 ResetEncoder("H264", 1, 1, 1, false);
3876
3877 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
3878 image._frameType = VideoFrameType::kVideoFrameKey;
3879
3880 CodecSpecificInfo codec_specific_info;
3881 codec_specific_info.codecType = kVideoCodecH264;
3882
3883 RTPFragmentationHeader fragmentation;
3884 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3885 fragmentation.fragmentationOffset[0] = 4;
3886 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
3887
3888 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3889 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3890
3891 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3892 testing::ElementsAreArray(optimal_sps));
3893 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3894 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3895 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3896 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3897
3898 video_stream_encoder_->Stop();
3899}
3900
3901TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
3902 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
3903 0x00, 0x00, 0x03, 0x03, 0xF4,
3904 0x05, 0x03, 0xC7, 0xC0};
3905
3906 // Configure internal source factory and setup test again.
3907 encoder_factory_.SetHasInternalSource(true);
3908 ResetEncoder("H264", 1, 1, 1, false);
3909
3910 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
3911 image._frameType = VideoFrameType::kVideoFrameKey;
3912
3913 CodecSpecificInfo codec_specific_info;
3914 codec_specific_info.codecType = kVideoCodecH264;
3915
3916 RTPFragmentationHeader fragmentation;
3917 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3918 fragmentation.fragmentationOffset[0] = 4;
3919 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
3920
3921 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3922 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3923
3924 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3925 testing::ElementsAreArray(optimal_sps));
3926 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3927 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3928 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3929 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3930
3931 video_stream_encoder_->Stop();
3932}
3933
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02003934TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
3935 const int kFrameWidth = 1280;
3936 const int kFrameHeight = 720;
3937 const int kTargetBitrateBps = 300000; // To low for HD resolution.
3938
3939 video_stream_encoder_->OnBitrateUpdated(
3940 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
3941 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3942
3943 // Insert a first video frame. It should be dropped because of downscale in
3944 // resolution.
3945 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3946 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
3947 frame.set_rotation(kVideoRotation_270);
3948 video_source_.IncomingCapturedFrame(frame);
3949
3950 ExpectDroppedFrame();
3951
3952 // Second frame is downscaled.
3953 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3954 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
3955 frame.set_rotation(kVideoRotation_90);
3956 video_source_.IncomingCapturedFrame(frame);
3957
3958 WaitForEncodedFrame(timestamp_ms);
3959 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
3960
3961 // Insert another frame, also downscaled.
3962 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3963 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
3964 frame.set_rotation(kVideoRotation_180);
3965 video_source_.IncomingCapturedFrame(frame);
3966
3967 WaitForEncodedFrame(timestamp_ms);
3968 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
3969
3970 video_stream_encoder_->Stop();
3971}
3972
perkj26091b12016-09-01 01:17:40 -07003973} // namespace webrtc