blob: d20025dea54bca050cf80f87f99d34f001c29e6c [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080020#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010022#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020023#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010024#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020025#include "common_video/h264/h264_common.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020027#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010029#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080030#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010033#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020034#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "system_wrappers/include/sleep.h"
36#include "test/encoder_settings.h"
37#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020038#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "test/frame_generator.h"
40#include "test/gmock.h"
41#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020042#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020043#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070044
45namespace webrtc {
46
sprangb1ca0732017-02-01 08:38:12 -080047using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080048using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080049
perkj803d97f2016-11-01 11:45:46 -070050namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020051const int kMinPixelsPerFrame = 320 * 180;
52const int kMinFramerateFps = 2;
53const int kMinBalancedFramerateFps = 7;
54const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080055const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010056const uint32_t kTargetBitrateBps = 1000000;
57const uint32_t kSimulcastTargetBitrateBps = 3150000;
58const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080059const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070060const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020061const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080062
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020063uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
64 0x00, 0x00, 0x03, 0x03, 0xF4,
65 0x05, 0x03, 0xC7, 0xE0, 0x1B,
66 0x41, 0x10, 0x8D, 0x00};
67
perkj803d97f2016-11-01 11:45:46 -070068class TestBuffer : public webrtc::I420Buffer {
69 public:
70 TestBuffer(rtc::Event* event, int width, int height)
71 : I420Buffer(width, height), event_(event) {}
72
73 private:
74 friend class rtc::RefCountedObject<TestBuffer>;
75 ~TestBuffer() override {
76 if (event_)
77 event_->Set();
78 }
79 rtc::Event* const event_;
80};
81
Niels Möller7dc26b72017-12-06 10:27:48 +010082class CpuOveruseDetectorProxy : public OveruseFrameDetector {
83 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020084 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
85 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010086 last_target_framerate_fps_(-1) {}
87 virtual ~CpuOveruseDetectorProxy() {}
88
89 void OnTargetFramerateUpdated(int framerate_fps) override {
90 rtc::CritScope cs(&lock_);
91 last_target_framerate_fps_ = framerate_fps;
92 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
93 }
94
95 int GetLastTargetFramerate() {
96 rtc::CritScope cs(&lock_);
97 return last_target_framerate_fps_;
98 }
99
Niels Möller4db138e2018-04-19 09:04:13 +0200100 CpuOveruseOptions GetOptions() { return options_; }
101
Niels Möller7dc26b72017-12-06 10:27:48 +0100102 private:
103 rtc::CriticalSection lock_;
104 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
105};
106
mflodmancc3d4422017-08-03 08:27:51 -0700107class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700108 public:
Niels Möller213618e2018-07-24 09:29:58 +0200109 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200110 const VideoStreamEncoderSettings& settings,
111 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100112 : VideoStreamEncoder(Clock::GetRealTimeClock(),
113 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200114 stats_proxy,
115 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200116 std::unique_ptr<OveruseFrameDetector>(
117 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100118 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200119 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700120
sprangb1ca0732017-02-01 08:38:12 -0800121 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100122 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800123 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800124 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700125 event.Set();
126 });
perkj070ba852017-02-16 15:46:27 -0800127 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700128 }
129
kthelgason2fc52542017-03-03 00:24:41 -0800130 // This is used as a synchronisation mechanism, to make sure that the
131 // encoder queue is not blocked before we start sending it frames.
132 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100133 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200134 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800135 ASSERT_TRUE(event.Wait(5000));
136 }
137
sprangb1ca0732017-02-01 08:38:12 -0800138 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800139
sprangb1ca0732017-02-01 08:38:12 -0800140 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800141
sprangb1ca0732017-02-01 08:38:12 -0800142 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800143
sprangb1ca0732017-02-01 08:38:12 -0800144 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700145
Niels Möller7dc26b72017-12-06 10:27:48 +0100146 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700147};
148
asapersson5f7226f2016-11-25 04:37:00 -0800149class VideoStreamFactory
150 : public VideoEncoderConfig::VideoStreamFactoryInterface {
151 public:
sprangfda496a2017-06-15 04:21:07 -0700152 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
153 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800154 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700155 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800156 }
157
158 private:
159 std::vector<VideoStream> CreateEncoderStreams(
160 int width,
161 int height,
162 const VideoEncoderConfig& encoder_config) override {
163 std::vector<VideoStream> streams =
164 test::CreateVideoStreams(width, height, encoder_config);
165 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100166 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700167 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800168 }
169 return streams;
170 }
sprangfda496a2017-06-15 04:21:07 -0700171
asapersson5f7226f2016-11-25 04:37:00 -0800172 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700173 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800174};
175
sprangb1ca0732017-02-01 08:38:12 -0800176class AdaptingFrameForwarder : public test::FrameForwarder {
177 public:
178 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700179 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800180
181 void set_adaptation_enabled(bool enabled) {
182 rtc::CritScope cs(&crit_);
183 adaptation_enabled_ = enabled;
184 }
185
asaperssonfab67072017-04-04 05:51:49 -0700186 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800187 rtc::CritScope cs(&crit_);
188 return adaptation_enabled_;
189 }
190
asapersson09f05612017-05-15 23:40:18 -0700191 rtc::VideoSinkWants last_wants() const {
192 rtc::CritScope cs(&crit_);
193 return last_wants_;
194 }
195
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200196 absl::optional<int> last_sent_width() const { return last_width_; }
197 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800198
sprangb1ca0732017-02-01 08:38:12 -0800199 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
200 int cropped_width = 0;
201 int cropped_height = 0;
202 int out_width = 0;
203 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700204 if (adaption_enabled()) {
205 if (adapter_.AdaptFrameResolution(
206 video_frame.width(), video_frame.height(),
207 video_frame.timestamp_us() * 1000, &cropped_width,
208 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100209 VideoFrame adapted_frame =
210 VideoFrame::Builder()
211 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
212 nullptr, out_width, out_height))
213 .set_timestamp_rtp(99)
214 .set_timestamp_ms(99)
215 .set_rotation(kVideoRotation_0)
216 .build();
sprangc5d62e22017-04-02 23:53:04 -0700217 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
218 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800219 last_width_.emplace(adapted_frame.width());
220 last_height_.emplace(adapted_frame.height());
221 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200222 last_width_ = absl::nullopt;
223 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700224 }
sprangb1ca0732017-02-01 08:38:12 -0800225 } else {
226 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800227 last_width_.emplace(video_frame.width());
228 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800229 }
230 }
231
232 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
233 const rtc::VideoSinkWants& wants) override {
234 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700235 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700236 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
237 wants.max_pixel_count,
238 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800239 test::FrameForwarder::AddOrUpdateSink(sink, wants);
240 }
sprangb1ca0732017-02-01 08:38:12 -0800241 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700242 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
243 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200244 absl::optional<int> last_width_;
245 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800246};
sprangc5d62e22017-04-02 23:53:04 -0700247
Niels Möller213618e2018-07-24 09:29:58 +0200248// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700249class MockableSendStatisticsProxy : public SendStatisticsProxy {
250 public:
251 MockableSendStatisticsProxy(Clock* clock,
252 const VideoSendStream::Config& config,
253 VideoEncoderConfig::ContentType content_type)
254 : SendStatisticsProxy(clock, config, content_type) {}
255
256 VideoSendStream::Stats GetStats() override {
257 rtc::CritScope cs(&lock_);
258 if (mock_stats_)
259 return *mock_stats_;
260 return SendStatisticsProxy::GetStats();
261 }
262
Niels Möller213618e2018-07-24 09:29:58 +0200263 int GetInputFrameRate() const override {
264 rtc::CritScope cs(&lock_);
265 if (mock_stats_)
266 return mock_stats_->input_frame_rate;
267 return SendStatisticsProxy::GetInputFrameRate();
268 }
sprangc5d62e22017-04-02 23:53:04 -0700269 void SetMockStats(const VideoSendStream::Stats& stats) {
270 rtc::CritScope cs(&lock_);
271 mock_stats_.emplace(stats);
272 }
273
274 void ResetMockStats() {
275 rtc::CritScope cs(&lock_);
276 mock_stats_.reset();
277 }
278
279 private:
280 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200281 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700282};
283
sprang4847ae62017-06-27 07:06:52 -0700284class MockBitrateObserver : public VideoBitrateAllocationObserver {
285 public:
Erik Språng566124a2018-04-23 12:32:22 +0200286 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700287};
288
perkj803d97f2016-11-01 11:45:46 -0700289} // namespace
290
mflodmancc3d4422017-08-03 08:27:51 -0700291class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700292 public:
293 static const int kDefaultTimeoutMs = 30 * 1000;
294
mflodmancc3d4422017-08-03 08:27:51 -0700295 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700296 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700297 codec_width_(320),
298 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200299 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200300 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700301 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200302 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800303 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700304 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700305 Clock::GetRealTimeClock(),
306 video_send_config_,
307 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700308 sink_(&fake_encoder_) {}
309
310 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700311 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700312 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200313 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800314 video_send_config_.encoder_settings.bitrate_allocator_factory =
315 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200316 video_send_config_.rtp.payload_name = "FAKE";
317 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700318
Per512ecb32016-09-23 15:52:06 +0200319 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200320 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700321 video_encoder_config.video_stream_factory =
322 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100323 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700324
325 // Framerate limit is specified by the VideoStreamFactory.
326 std::vector<VideoStream> streams =
327 video_encoder_config.video_stream_factory->CreateEncoderStreams(
328 codec_width_, codec_height_, video_encoder_config);
329 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200330 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700331
Niels Möllerf1338562018-04-26 09:51:47 +0200332 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800333 }
334
Niels Möllerf1338562018-04-26 09:51:47 +0200335 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700336 if (video_stream_encoder_)
337 video_stream_encoder_->Stop();
338 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200339 stats_proxy_.get(), video_send_config_.encoder_settings,
340 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700341 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
342 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700343 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700344 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
345 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200346 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800348 }
349
350 void ResetEncoder(const std::string& payload_name,
351 size_t num_streams,
352 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700353 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700354 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200355 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800356
357 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200358 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800359 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100360 video_encoder_config.max_bitrate_bps =
361 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800362 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700363 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
364 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700365 video_encoder_config.content_type =
366 screenshare ? VideoEncoderConfig::ContentType::kScreen
367 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700368 if (payload_name == "VP9") {
369 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
370 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
371 video_encoder_config.encoder_specific_settings =
372 new rtc::RefCountedObject<
373 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
374 }
Niels Möllerf1338562018-04-26 09:51:47 +0200375 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700376 }
377
sprang57c2fff2017-01-16 06:24:02 -0800378 VideoFrame CreateFrame(int64_t ntp_time_ms,
379 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100380 VideoFrame frame =
381 VideoFrame::Builder()
382 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
383 destruction_event, codec_width_, codec_height_))
384 .set_timestamp_rtp(99)
385 .set_timestamp_ms(99)
386 .set_rotation(kVideoRotation_0)
387 .build();
sprang57c2fff2017-01-16 06:24:02 -0800388 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700389 return frame;
390 }
391
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100392 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
393 rtc::Event* destruction_event,
394 int offset_x) const {
395 VideoFrame frame =
396 VideoFrame::Builder()
397 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
398 destruction_event, codec_width_, codec_height_))
399 .set_timestamp_rtp(99)
400 .set_timestamp_ms(99)
401 .set_rotation(kVideoRotation_0)
402 .set_update_rect({offset_x, 0, 1, 1})
403 .build();
404 frame.set_ntp_time_ms(ntp_time_ms);
405 return frame;
406 }
407
sprang57c2fff2017-01-16 06:24:02 -0800408 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100409 VideoFrame frame =
410 VideoFrame::Builder()
411 .set_video_frame_buffer(
412 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
413 .set_timestamp_rtp(99)
414 .set_timestamp_ms(99)
415 .set_rotation(kVideoRotation_0)
416 .build();
sprang57c2fff2017-01-16 06:24:02 -0800417 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700418 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700419 return frame;
420 }
421
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100422 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
423 MockBitrateObserver bitrate_observer;
424 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
425
426 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
427 .Times(1);
428 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +0200429 DataRate::bps(kTargetBitrateBps), 0,
430 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100431
432 video_source_.IncomingCapturedFrame(
433 CreateFrame(1, codec_width_, codec_height_));
434 WaitForEncodedFrame(1);
435 }
436
asapersson02465b82017-04-10 01:12:52 -0700437 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700438 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700439 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
440 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700441 }
442
asapersson09f05612017-05-15 23:40:18 -0700443 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
444 const rtc::VideoSinkWants& wants2) {
445 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
446 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
447 }
448
Åsa Persson8c1bf952018-09-13 10:42:19 +0200449 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
450 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
451 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
452 EXPECT_FALSE(wants.target_pixel_count);
453 }
454
asapersson09f05612017-05-15 23:40:18 -0700455 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
456 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200457 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700458 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
459 EXPECT_GT(wants1.max_pixel_count, 0);
460 }
461
462 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
463 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200464 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700465 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
466 }
467
asaperssonf7e294d2017-06-13 23:25:22 -0700468 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
469 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200470 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700471 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
472 }
473
474 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
475 const rtc::VideoSinkWants& wants2) {
476 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
477 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
478 }
479
480 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
481 const rtc::VideoSinkWants& wants2) {
482 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
483 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
484 }
485
486 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
487 const rtc::VideoSinkWants& wants2) {
488 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
489 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
490 EXPECT_GT(wants1.max_pixel_count, 0);
491 }
492
493 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
494 const rtc::VideoSinkWants& wants2) {
495 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
496 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
497 }
498
asapersson09f05612017-05-15 23:40:18 -0700499 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
500 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200501 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700502 EXPECT_LT(wants.max_pixel_count, pixel_count);
503 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700504 }
505
506 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
507 EXPECT_LT(wants.max_framerate_fps, fps);
508 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
509 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700510 }
511
asaperssonf7e294d2017-06-13 23:25:22 -0700512 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
513 int expected_fps) {
514 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
515 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
516 EXPECT_FALSE(wants.target_pixel_count);
517 }
518
Jonathan Yubc771b72017-12-08 17:04:29 -0800519 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
520 int last_frame_pixels) {
521 // Balanced mode should always scale FPS to the desired range before
522 // attempting to scale resolution.
523 int fps_limit = wants.max_framerate_fps;
524 if (last_frame_pixels <= 320 * 240) {
525 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
526 } else if (last_frame_pixels <= 480 * 270) {
527 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
528 } else if (last_frame_pixels <= 640 * 480) {
529 EXPECT_LE(15, fps_limit);
530 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200531 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800532 }
533 }
534
sprang4847ae62017-06-27 07:06:52 -0700535 void WaitForEncodedFrame(int64_t expected_ntp_time) {
536 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200537 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700538 }
539
540 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
541 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200542 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700543 return ok;
544 }
545
546 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
547 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200548 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700549 }
550
551 void ExpectDroppedFrame() {
552 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200553 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700554 }
555
556 bool WaitForFrame(int64_t timeout_ms) {
557 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200558 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700559 return ok;
560 }
561
perkj26091b12016-09-01 01:17:40 -0700562 class TestEncoder : public test::FakeEncoder {
563 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100564 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700565
asaperssonfab67072017-04-04 05:51:49 -0700566 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800567 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700568 return config_;
569 }
570
571 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800572 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700573 block_next_encode_ = true;
574 }
575
Erik Språngaed30702018-11-05 12:57:17 +0100576 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800577 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100578 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100579 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100580 if (quality_scaling_) {
581 info.scaling_settings =
582 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
583 }
584 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100585 for (int i = 0; i < kMaxSpatialLayers; ++i) {
586 if (temporal_layers_supported_[i]) {
587 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
588 info.fps_allocation[i].resize(num_layers);
589 }
590 }
Erik Språngaed30702018-11-05 12:57:17 +0100591 }
592 return info;
kthelgason876222f2016-11-29 01:44:11 -0800593 }
594
Erik Språngb7cb7b52019-02-26 15:52:33 +0100595 int32_t RegisterEncodeCompleteCallback(
596 EncodedImageCallback* callback) override {
597 rtc::CritScope lock(&local_crit_sect_);
598 encoded_image_callback_ = callback;
599 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
600 }
601
perkjfa10b552016-10-02 23:45:26 -0700602 void ContinueEncode() { continue_encode_event_.Set(); }
603
604 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
605 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800606 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700607 EXPECT_EQ(timestamp_, timestamp);
608 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
609 }
610
kthelgason2fc52542017-03-03 00:24:41 -0800611 void SetQualityScaling(bool b) {
612 rtc::CritScope lock(&local_crit_sect_);
613 quality_scaling_ = b;
614 }
kthelgasonad9010c2017-02-14 00:46:51 -0800615
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100616 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
617 rtc::CritScope lock(&local_crit_sect_);
618 is_hardware_accelerated_ = is_hardware_accelerated;
619 }
620
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100621 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
622 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
623 rtc::CritScope lock(&local_crit_sect_);
624 temporal_layers_supported_[spatial_idx] = supported;
625 }
626
sprangfe627f32017-03-29 08:24:59 -0700627 void ForceInitEncodeFailure(bool force_failure) {
628 rtc::CritScope lock(&local_crit_sect_);
629 force_init_encode_failed_ = force_failure;
630 }
631
Niels Möller6bb5ab92019-01-11 11:11:10 +0100632 void SimulateOvershoot(double rate_factor) {
633 rtc::CritScope lock(&local_crit_sect_);
634 rate_factor_ = rate_factor;
635 }
636
Erik Språngd7329ca2019-02-21 21:19:53 +0100637 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100638 rtc::CritScope lock(&local_crit_sect_);
639 return last_framerate_;
640 }
641
Erik Språngd7329ca2019-02-21 21:19:53 +0100642 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100643 rtc::CritScope lock(&local_crit_sect_);
644 return last_update_rect_;
645 }
646
Niels Möller87e2d782019-03-07 10:18:23 +0100647 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100648 rtc::CritScope lock(&local_crit_sect_);
649 return last_frame_types_;
650 }
651
652 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100653 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100654 keyframe ? VideoFrameType::kVideoFrameKey
655 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100656 {
657 rtc::CritScope lock(&local_crit_sect_);
658 last_frame_types_ = frame_type;
659 }
Niels Möllerb859b322019-03-07 12:40:01 +0100660 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100661 }
662
Erik Språngb7cb7b52019-02-26 15:52:33 +0100663 void InjectEncodedImage(const EncodedImage& image) {
664 rtc::CritScope lock(&local_crit_sect_);
665 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
666 }
667
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200668 void InjectEncodedImage(const EncodedImage& image,
669 const CodecSpecificInfo* codec_specific_info,
670 const RTPFragmentationHeader* fragmentation) {
671 rtc::CritScope lock(&local_crit_sect_);
672 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
673 fragmentation);
674 }
675
Erik Språngd7329ca2019-02-21 21:19:53 +0100676 void ExpectNullFrame() {
677 rtc::CritScope lock(&local_crit_sect_);
678 expect_null_frame_ = true;
679 }
680
681 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
682 auto allocation = last_bitrate_allocation_;
683 last_bitrate_allocation_.reset();
684 return allocation;
685 }
686
perkjfa10b552016-10-02 23:45:26 -0700687 private:
perkj26091b12016-09-01 01:17:40 -0700688 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100689 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700690 bool block_encode;
691 {
brandtre78d2662017-01-16 05:57:16 -0800692 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100693 if (expect_null_frame_) {
694 EXPECT_EQ(input_image.timestamp(), 0u);
695 EXPECT_EQ(input_image.width(), 1);
696 last_frame_types_ = *frame_types;
697 expect_null_frame_ = false;
698 } else {
699 EXPECT_GT(input_image.timestamp(), timestamp_);
700 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
701 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
702 }
perkj26091b12016-09-01 01:17:40 -0700703
704 timestamp_ = input_image.timestamp();
705 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700706 last_input_width_ = input_image.width();
707 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700708 block_encode = block_next_encode_;
709 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100710 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100711 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700712 }
Niels Möllerb859b322019-03-07 12:40:01 +0100713 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700714 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700715 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700716 return result;
717 }
718
sprangfe627f32017-03-29 08:24:59 -0700719 int32_t InitEncode(const VideoCodec* config,
720 int32_t number_of_cores,
721 size_t max_payload_size) override {
722 int res =
723 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
724 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100725 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100726 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700727 // Simulate setting up temporal layers, in order to validate the life
728 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100729 Vp8TemporalLayersFactory factory;
730 frame_buffer_controller_ = factory.Create(*config);
sprangfe627f32017-03-29 08:24:59 -0700731 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100732 if (force_init_encode_failed_) {
733 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700734 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100735 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100736
Erik Språngb7cb7b52019-02-26 15:52:33 +0100737 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700738 return res;
739 }
740
Erik Språngb7cb7b52019-02-26 15:52:33 +0100741 int32_t Release() override {
742 rtc::CritScope lock(&local_crit_sect_);
743 EXPECT_NE(initialized_, EncoderState::kUninitialized);
744 initialized_ = EncoderState::kUninitialized;
745 return FakeEncoder::Release();
746 }
747
Erik Språng16cb8f52019-04-12 13:59:09 +0200748 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100749 rtc::CritScope lock(&local_crit_sect_);
750 VideoBitrateAllocation adjusted_rate_allocation;
751 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
752 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200753 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100754 adjusted_rate_allocation.SetBitrate(
755 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200756 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100757 rate_factor_));
758 }
759 }
760 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200761 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
762 last_bitrate_allocation_ = parameters.bitrate;
763 RateControlParameters adjusted_paramters = parameters;
764 adjusted_paramters.bitrate = adjusted_rate_allocation;
765 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100766 }
767
brandtre78d2662017-01-16 05:57:16 -0800768 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100769 enum class EncoderState {
770 kUninitialized,
771 kInitializationFailed,
772 kInitialized
773 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
774 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700775 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700776 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700777 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
778 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
779 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
780 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
781 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100782 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100783 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700784 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100785 absl::optional<bool>
786 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
787 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700788 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100789 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
790 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100791 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100792 VideoFrame::UpdateRect last_update_rect_
793 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100794 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100795 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100796 EncodedImageCallback* encoded_image_callback_
797 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700798 };
799
mflodmancc3d4422017-08-03 08:27:51 -0700800 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700801 public:
802 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100803 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700804
perkj26091b12016-09-01 01:17:40 -0700805 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700806 EXPECT_TRUE(
807 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
808 }
809
810 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
811 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700812 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700813 if (!encoded_frame_event_.Wait(timeout_ms))
814 return false;
perkj26091b12016-09-01 01:17:40 -0700815 {
816 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800817 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700818 }
819 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700820 return true;
perkj26091b12016-09-01 01:17:40 -0700821 }
822
sprangb1ca0732017-02-01 08:38:12 -0800823 void WaitForEncodedFrame(uint32_t expected_width,
824 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700825 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100826 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700827 }
828
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100829 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700830 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800831 uint32_t width = 0;
832 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800833 {
834 rtc::CritScope lock(&crit_);
835 width = last_width_;
836 height = last_height_;
837 }
838 EXPECT_EQ(expected_height, height);
839 EXPECT_EQ(expected_width, width);
840 }
841
kthelgason2fc52542017-03-03 00:24:41 -0800842 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800843
sprangc5d62e22017-04-02 23:53:04 -0700844 bool WaitForFrame(int64_t timeout_ms) {
845 return encoded_frame_event_.Wait(timeout_ms);
846 }
847
perkj26091b12016-09-01 01:17:40 -0700848 void SetExpectNoFrames() {
849 rtc::CritScope lock(&crit_);
850 expect_frames_ = false;
851 }
852
asaperssonfab67072017-04-04 05:51:49 -0700853 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200854 rtc::CritScope lock(&crit_);
855 return number_of_reconfigurations_;
856 }
857
asaperssonfab67072017-04-04 05:51:49 -0700858 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200859 rtc::CritScope lock(&crit_);
860 return min_transmit_bitrate_bps_;
861 }
862
Erik Språngd7329ca2019-02-21 21:19:53 +0100863 void SetNumExpectedLayers(size_t num_layers) {
864 rtc::CritScope lock(&crit_);
865 num_expected_layers_ = num_layers;
866 }
867
Erik Språngb7cb7b52019-02-26 15:52:33 +0100868 int64_t GetLastCaptureTimeMs() const {
869 rtc::CritScope lock(&crit_);
870 return last_capture_time_ms_;
871 }
872
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200873 std::vector<uint8_t> GetLastEncodedImageData() {
874 rtc::CritScope lock(&crit_);
875 return std::move(last_encoded_image_data_);
876 }
877
878 RTPFragmentationHeader GetLastFragmentation() {
879 rtc::CritScope lock(&crit_);
880 return std::move(last_fragmentation_);
881 }
882
perkj26091b12016-09-01 01:17:40 -0700883 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700884 Result OnEncodedImage(
885 const EncodedImage& encoded_image,
886 const CodecSpecificInfo* codec_specific_info,
887 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200888 rtc::CritScope lock(&crit_);
889 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200890 last_encoded_image_data_ = std::vector<uint8_t>(
891 encoded_image.data(), encoded_image.data() + encoded_image.size());
892 if (fragmentation) {
893 last_fragmentation_.CopyFrom(*fragmentation);
894 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100895 uint32_t timestamp = encoded_image.Timestamp();
896 if (last_timestamp_ != timestamp) {
897 num_received_layers_ = 1;
898 } else {
899 ++num_received_layers_;
900 }
901 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100902 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800903 last_width_ = encoded_image._encodedWidth;
904 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +0100905 if (num_received_layers_ == num_expected_layers_) {
906 encoded_frame_event_.Set();
907 }
sprangb1ca0732017-02-01 08:38:12 -0800908 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200909 }
910
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100911 void OnEncoderConfigurationChanged(
912 std::vector<VideoStream> streams,
913 VideoEncoderConfig::ContentType content_type,
914 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200915 rtc::CriticalSection crit_;
916 ++number_of_reconfigurations_;
917 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
918 }
919
perkj26091b12016-09-01 01:17:40 -0700920 rtc::CriticalSection crit_;
921 TestEncoder* test_encoder_;
922 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200923 std::vector<uint8_t> last_encoded_image_data_;
924 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -0800925 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100926 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800927 uint32_t last_height_ = 0;
928 uint32_t last_width_ = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100929 size_t num_expected_layers_ = 1;
930 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700931 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200932 int number_of_reconfigurations_ = 0;
933 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700934 };
935
936 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100937 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200938 int codec_width_;
939 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700940 int max_framerate_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200941 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -0700942 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200943 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800944 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700945 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700946 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800947 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700948 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700949 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700950};
951
mflodmancc3d4422017-08-03 08:27:51 -0700952TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200953 video_stream_encoder_->OnBitrateUpdated(
954 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100955 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700956 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700957 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700958 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700959 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700960}
961
mflodmancc3d4422017-08-03 08:27:51 -0700962TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700963 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100964 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200965 // The encoder will cache up to one frame for a short duration. Adding two
966 // frames means that the first frame will be dropped and the second frame will
967 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700968 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200969 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700970 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700971
Erik Språng4c6ca302019-04-08 15:14:01 +0200972 video_stream_encoder_->OnBitrateUpdated(
973 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -0700974
Sebastian Janssona3177052018-04-10 13:05:49 +0200975 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700976 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200977 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
978
979 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700980 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700981}
982
mflodmancc3d4422017-08-03 08:27:51 -0700983TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200984 video_stream_encoder_->OnBitrateUpdated(
985 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700986 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700987 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700988
Erik Språng4c6ca302019-04-08 15:14:01 +0200989 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +0100990 0);
Sebastian Janssona3177052018-04-10 13:05:49 +0200991 // The encoder will cache up to one frame for a short duration. Adding two
992 // frames means that the first frame will be dropped and the second frame will
993 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -0700994 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +0200995 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700996
Erik Språng4c6ca302019-04-08 15:14:01 +0200997 video_stream_encoder_->OnBitrateUpdated(
998 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -0700999 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001000 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1001 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001002 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001003}
1004
mflodmancc3d4422017-08-03 08:27:51 -07001005TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001006 video_stream_encoder_->OnBitrateUpdated(
1007 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001008 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001009 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001010
1011 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001012 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001013
perkja49cbd32016-09-16 07:53:41 -07001014 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001015 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001016 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001017}
1018
mflodmancc3d4422017-08-03 08:27:51 -07001019TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001020 video_stream_encoder_->OnBitrateUpdated(
1021 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001022
perkja49cbd32016-09-16 07:53:41 -07001023 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001024 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001025
mflodmancc3d4422017-08-03 08:27:51 -07001026 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001027 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001028 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001029 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1030 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001031}
1032
mflodmancc3d4422017-08-03 08:27:51 -07001033TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001034 video_stream_encoder_->OnBitrateUpdated(
1035 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001036
1037 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001038 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001039 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001040 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1041 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001042 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1043 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001044 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001045 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001046
mflodmancc3d4422017-08-03 08:27:51 -07001047 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001048}
1049
mflodmancc3d4422017-08-03 08:27:51 -07001050TEST_F(VideoStreamEncoderTest,
1051 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001052 video_stream_encoder_->OnBitrateUpdated(
1053 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001054 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001055
1056 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001057 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001058 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001059 // The encoder will have been configured once when the first frame is
1060 // received.
1061 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001062
1063 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001064 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001065 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001066 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001067 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001068
1069 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001070 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001071 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001072 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001073 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001074
mflodmancc3d4422017-08-03 08:27:51 -07001075 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001076}
1077
mflodmancc3d4422017-08-03 08:27:51 -07001078TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001079 video_stream_encoder_->OnBitrateUpdated(
1080 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001081
1082 // Capture a frame and wait for it to synchronize with the encoder thread.
1083 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001084 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001085 // The encoder will have been configured once.
1086 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001087 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1088 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1089
1090 codec_width_ *= 2;
1091 codec_height_ *= 2;
1092 // Capture a frame with a higher resolution and wait for it to synchronize
1093 // with the encoder thread.
1094 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001095 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001096 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1097 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001098 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001099
mflodmancc3d4422017-08-03 08:27:51 -07001100 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001101}
1102
mflodmancc3d4422017-08-03 08:27:51 -07001103TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001104 EXPECT_TRUE(video_source_.has_sinks());
1105 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001106 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001107 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001108 EXPECT_FALSE(video_source_.has_sinks());
1109 EXPECT_TRUE(new_video_source.has_sinks());
1110
mflodmancc3d4422017-08-03 08:27:51 -07001111 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001112}
1113
mflodmancc3d4422017-08-03 08:27:51 -07001114TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001115 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001116 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001117 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001118 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001119}
1120
Jonathan Yubc771b72017-12-08 17:04:29 -08001121TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1122 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001123 const int kWidth = 1280;
1124 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001125
1126 // We rely on the automatic resolution adaptation, but we handle framerate
1127 // adaptation manually by mocking the stats proxy.
1128 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001129
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001130 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001131 video_stream_encoder_->OnBitrateUpdated(
1132 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001133 video_stream_encoder_->SetSource(&video_source_,
1134 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001135 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001136 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001137 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001138 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1139
Jonathan Yubc771b72017-12-08 17:04:29 -08001140 // Adapt down as far as possible.
1141 rtc::VideoSinkWants last_wants;
1142 int64_t t = 1;
1143 int loop_count = 0;
1144 do {
1145 ++loop_count;
1146 last_wants = video_source_.sink_wants();
1147
1148 // Simulate the framerate we've been asked to adapt to.
1149 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1150 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1151 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1152 mock_stats.input_frame_rate = fps;
1153 stats_proxy_->SetMockStats(mock_stats);
1154
1155 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1156 sink_.WaitForEncodedFrame(t);
1157 t += frame_interval_ms;
1158
mflodmancc3d4422017-08-03 08:27:51 -07001159 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001160 VerifyBalancedModeFpsRange(
1161 video_source_.sink_wants(),
1162 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1163 } while (video_source_.sink_wants().max_pixel_count <
1164 last_wants.max_pixel_count ||
1165 video_source_.sink_wants().max_framerate_fps <
1166 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001167
Jonathan Yubc771b72017-12-08 17:04:29 -08001168 // Verify that we've adapted all the way down.
1169 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001170 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001171 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1172 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001173 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001174 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1175 *video_source_.last_sent_height());
1176 EXPECT_EQ(kMinBalancedFramerateFps,
1177 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001178
Jonathan Yubc771b72017-12-08 17:04:29 -08001179 // Adapt back up the same number of times we adapted down.
1180 for (int i = 0; i < loop_count - 1; ++i) {
1181 last_wants = video_source_.sink_wants();
1182
1183 // Simulate the framerate we've been asked to adapt to.
1184 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1185 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1186 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1187 mock_stats.input_frame_rate = fps;
1188 stats_proxy_->SetMockStats(mock_stats);
1189
1190 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1191 sink_.WaitForEncodedFrame(t);
1192 t += frame_interval_ms;
1193
mflodmancc3d4422017-08-03 08:27:51 -07001194 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001195 VerifyBalancedModeFpsRange(
1196 video_source_.sink_wants(),
1197 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1198 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1199 last_wants.max_pixel_count ||
1200 video_source_.sink_wants().max_framerate_fps >
1201 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001202 }
1203
Åsa Persson8c1bf952018-09-13 10:42:19 +02001204 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001205 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001206 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001207 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1208 EXPECT_EQ((loop_count - 1) * 2,
1209 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001210
mflodmancc3d4422017-08-03 08:27:51 -07001211 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001212}
mflodmancc3d4422017-08-03 08:27:51 -07001213TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001214 video_stream_encoder_->OnBitrateUpdated(
1215 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001216 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001217
sprangc5d62e22017-04-02 23:53:04 -07001218 const int kFrameWidth = 1280;
1219 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001220
Åsa Persson8c1bf952018-09-13 10:42:19 +02001221 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001222
kthelgason5e13d412016-12-01 03:59:51 -08001223 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001224 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001225 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001226 frame_timestamp += kFrameIntervalMs;
1227
perkj803d97f2016-11-01 11:45:46 -07001228 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001229 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001230 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001231 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001232 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001233 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001234
asapersson0944a802017-04-07 00:57:58 -07001235 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001236 // wanted resolution.
1237 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1238 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1239 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001240 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001241
1242 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001243 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001244 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001245 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001246
sprangc5d62e22017-04-02 23:53:04 -07001247 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001248 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001249
sprangc5d62e22017-04-02 23:53:04 -07001250 // Force an input frame rate to be available, or the adaptation call won't
1251 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001252 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001253 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001254 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001255 stats_proxy_->SetMockStats(stats);
1256
mflodmancc3d4422017-08-03 08:27:51 -07001257 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001258 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001259 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001260 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001261 frame_timestamp += kFrameIntervalMs;
1262
1263 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001264 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001265 EXPECT_EQ(std::numeric_limits<int>::max(),
1266 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001267 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001268
asapersson02465b82017-04-10 01:12:52 -07001269 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001270 video_stream_encoder_->SetSource(&new_video_source,
1271 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001272 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001273
mflodmancc3d4422017-08-03 08:27:51 -07001274 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001275 new_video_source.IncomingCapturedFrame(
1276 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001277 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001278 frame_timestamp += kFrameIntervalMs;
1279
1280 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001281 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001282
1283 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001284 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001285 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001286 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1287 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001288 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001289 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001290
1291 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001292 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001293 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001294 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1295 EXPECT_EQ(std::numeric_limits<int>::max(),
1296 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001297 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001298
mflodmancc3d4422017-08-03 08:27:51 -07001299 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001300}
1301
mflodmancc3d4422017-08-03 08:27:51 -07001302TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001303 video_stream_encoder_->OnBitrateUpdated(
1304 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001305
asaperssonfab67072017-04-04 05:51:49 -07001306 const int kWidth = 1280;
1307 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001308 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001309 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001310 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1311 EXPECT_FALSE(stats.bw_limited_resolution);
1312 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1313
1314 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001315 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001316 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001317 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001318
1319 stats = stats_proxy_->GetStats();
1320 EXPECT_TRUE(stats.bw_limited_resolution);
1321 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1322
1323 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001324 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001325 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001326 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001327
1328 stats = stats_proxy_->GetStats();
1329 EXPECT_FALSE(stats.bw_limited_resolution);
1330 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1331 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1332
mflodmancc3d4422017-08-03 08:27:51 -07001333 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001334}
1335
mflodmancc3d4422017-08-03 08:27:51 -07001336TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001337 video_stream_encoder_->OnBitrateUpdated(
1338 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001339
1340 const int kWidth = 1280;
1341 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001342 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001343 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001344 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1345 EXPECT_FALSE(stats.cpu_limited_resolution);
1346 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1347
1348 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001349 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001350 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001351 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001352
1353 stats = stats_proxy_->GetStats();
1354 EXPECT_TRUE(stats.cpu_limited_resolution);
1355 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1356
1357 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001358 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001359 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001360 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001361
1362 stats = stats_proxy_->GetStats();
1363 EXPECT_FALSE(stats.cpu_limited_resolution);
1364 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001365 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001366
mflodmancc3d4422017-08-03 08:27:51 -07001367 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001368}
1369
mflodmancc3d4422017-08-03 08:27:51 -07001370TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001371 video_stream_encoder_->OnBitrateUpdated(
1372 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001373
asaperssonfab67072017-04-04 05:51:49 -07001374 const int kWidth = 1280;
1375 const int kHeight = 720;
1376 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001377 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001378 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001379 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001380 EXPECT_FALSE(stats.cpu_limited_resolution);
1381 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1382
asaperssonfab67072017-04-04 05:51:49 -07001383 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001384 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001385 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001386 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001387 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001388 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001389 EXPECT_TRUE(stats.cpu_limited_resolution);
1390 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1391
1392 // Set new source with adaptation still enabled.
1393 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001394 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001395 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001396
asaperssonfab67072017-04-04 05:51:49 -07001397 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001398 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001399 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001400 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001401 EXPECT_TRUE(stats.cpu_limited_resolution);
1402 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1403
1404 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001405 video_stream_encoder_->SetSource(&new_video_source,
1406 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001407
asaperssonfab67072017-04-04 05:51:49 -07001408 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001409 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001410 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001411 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001412 EXPECT_FALSE(stats.cpu_limited_resolution);
1413 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1414
1415 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001416 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001417 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001418
asaperssonfab67072017-04-04 05:51:49 -07001419 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001420 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001421 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001422 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001423 EXPECT_TRUE(stats.cpu_limited_resolution);
1424 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1425
asaperssonfab67072017-04-04 05:51:49 -07001426 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001427 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001428 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001429 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001430 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001431 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001432 EXPECT_FALSE(stats.cpu_limited_resolution);
1433 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001434 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001435
mflodmancc3d4422017-08-03 08:27:51 -07001436 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001437}
1438
mflodmancc3d4422017-08-03 08:27:51 -07001439TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001440 video_stream_encoder_->OnBitrateUpdated(
1441 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001442
asaperssonfab67072017-04-04 05:51:49 -07001443 const int kWidth = 1280;
1444 const int kHeight = 720;
1445 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001446 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001447 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001448 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001449 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001450 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001451
1452 // Set new source with adaptation still enabled.
1453 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001454 video_stream_encoder_->SetSource(&new_video_source,
1455 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001456
asaperssonfab67072017-04-04 05:51:49 -07001457 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001458 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001459 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001460 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001461 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001462 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001463
asaperssonfab67072017-04-04 05:51:49 -07001464 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001465 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001466 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001468 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001469 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001470 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001471 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001472
asaperssonfab67072017-04-04 05:51:49 -07001473 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001474 video_stream_encoder_->SetSource(&new_video_source,
1475 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001476
asaperssonfab67072017-04-04 05:51:49 -07001477 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001478 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001479 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001480 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001481 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001482 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001483
asapersson02465b82017-04-10 01:12:52 -07001484 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001485 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001486 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001487
asaperssonfab67072017-04-04 05:51:49 -07001488 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001489 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001490 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001491 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001492 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001493 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1494 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001495
mflodmancc3d4422017-08-03 08:27:51 -07001496 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001497}
1498
mflodmancc3d4422017-08-03 08:27:51 -07001499TEST_F(VideoStreamEncoderTest,
1500 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001501 video_stream_encoder_->OnBitrateUpdated(
1502 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001503
1504 const int kWidth = 1280;
1505 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001506 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001507 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001508 video_source_.IncomingCapturedFrame(
1509 CreateFrame(timestamp_ms, kWidth, kHeight));
1510 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001511 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1512 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1513 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1514
1515 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001516 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001517 timestamp_ms += kFrameIntervalMs;
1518 video_source_.IncomingCapturedFrame(
1519 CreateFrame(timestamp_ms, kWidth, kHeight));
1520 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001521 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1523 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1524
1525 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001526 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001527 timestamp_ms += kFrameIntervalMs;
1528 video_source_.IncomingCapturedFrame(
1529 CreateFrame(timestamp_ms, kWidth, kHeight));
1530 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001531 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1532 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1533 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1534
Niels Möller4db138e2018-04-19 09:04:13 +02001535 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001536 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001537
1538 VideoEncoderConfig video_encoder_config;
1539 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1540 // Make format different, to force recreation of encoder.
1541 video_encoder_config.video_format.parameters["foo"] = "foo";
1542 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001543 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001544 timestamp_ms += kFrameIntervalMs;
1545 video_source_.IncomingCapturedFrame(
1546 CreateFrame(timestamp_ms, kWidth, kHeight));
1547 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001548 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1550 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1551
mflodmancc3d4422017-08-03 08:27:51 -07001552 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001553}
1554
mflodmancc3d4422017-08-03 08:27:51 -07001555TEST_F(VideoStreamEncoderTest,
1556 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001557 video_stream_encoder_->OnBitrateUpdated(
1558 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001559
asapersson0944a802017-04-07 00:57:58 -07001560 const int kWidth = 1280;
1561 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001562 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001563
asaperssonfab67072017-04-04 05:51:49 -07001564 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001565 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001566 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001567 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001568 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001569 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1570
asapersson02465b82017-04-10 01:12:52 -07001571 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001572 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001573 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001574 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001575 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001576 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001577 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001578 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1579
1580 // Set new source with adaptation still enabled.
1581 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001582 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001583 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001584
1585 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001586 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001587 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001588 stats = stats_proxy_->GetStats();
1589 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001590 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001591 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1592
sprangc5d62e22017-04-02 23:53:04 -07001593 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001594 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001595 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001596 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001597 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001598 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001599 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001600 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001601 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001602 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001603 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1604
sprangc5d62e22017-04-02 23:53:04 -07001605 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001606 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001607 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1608 mock_stats.input_frame_rate = 30;
1609 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001610 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001611 stats_proxy_->ResetMockStats();
1612
1613 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001614 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001615 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001616
1617 // Framerate now adapted.
1618 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001619 EXPECT_FALSE(stats.cpu_limited_resolution);
1620 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001621 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1622
1623 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001624 video_stream_encoder_->SetSource(&new_video_source,
1625 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001626 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001627 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001628 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001629
1630 stats = stats_proxy_->GetStats();
1631 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001632 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001633 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1634
1635 // Try to trigger overuse. Should not succeed.
1636 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001637 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001638 stats_proxy_->ResetMockStats();
1639
1640 stats = stats_proxy_->GetStats();
1641 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001642 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001643 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1644
1645 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001647 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001648 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001649 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001650 stats = stats_proxy_->GetStats();
1651 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001652 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001653 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001654
1655 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001656 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001657 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001658 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001659 stats = stats_proxy_->GetStats();
1660 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001661 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001662 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1663
1664 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001665 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001666 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001667 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001668 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001669 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001670 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001671 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001672 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001673 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001674 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1675
1676 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001677 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001678 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001679 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001680 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001681 stats = stats_proxy_->GetStats();
1682 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001683 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001684 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001685 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001686
mflodmancc3d4422017-08-03 08:27:51 -07001687 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001688}
1689
mflodmancc3d4422017-08-03 08:27:51 -07001690TEST_F(VideoStreamEncoderTest,
1691 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001692 const int kWidth = 1280;
1693 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001694 video_stream_encoder_->OnBitrateUpdated(
1695 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001696
asaperssonfab67072017-04-04 05:51:49 -07001697 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001698 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001699
asaperssonfab67072017-04-04 05:51:49 -07001700 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001701 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001702
asaperssonfab67072017-04-04 05:51:49 -07001703 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001704 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001705
asaperssonfab67072017-04-04 05:51:49 -07001706 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001707 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001708
kthelgason876222f2016-11-29 01:44:11 -08001709 // Expect a scale down.
1710 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001711 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001712
asapersson02465b82017-04-10 01:12:52 -07001713 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001714 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001715 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001716 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001717
asaperssonfab67072017-04-04 05:51:49 -07001718 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001719 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001720 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001721 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001722
asaperssonfab67072017-04-04 05:51:49 -07001723 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001724 EXPECT_EQ(std::numeric_limits<int>::max(),
1725 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001726
asaperssonfab67072017-04-04 05:51:49 -07001727 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001728 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001729 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001730 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001731
asapersson02465b82017-04-10 01:12:52 -07001732 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001733 EXPECT_EQ(std::numeric_limits<int>::max(),
1734 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001735
mflodmancc3d4422017-08-03 08:27:51 -07001736 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001737}
1738
mflodmancc3d4422017-08-03 08:27:51 -07001739TEST_F(VideoStreamEncoderTest,
1740 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001741 const int kWidth = 1280;
1742 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001743 video_stream_encoder_->OnBitrateUpdated(
1744 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001745
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001746 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001747 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001748 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001749 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001750
1751 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001752 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001753 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001754 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1755 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1756
1757 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001758 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001759 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001760 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1761 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1762 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1763
1764 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001765 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001766 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1767 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1769
mflodmancc3d4422017-08-03 08:27:51 -07001770 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001771}
1772
mflodmancc3d4422017-08-03 08:27:51 -07001773TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001774 const int kWidth = 1280;
1775 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001776 video_stream_encoder_->OnBitrateUpdated(
1777 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001778
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001779 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001780 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001781 video_stream_encoder_->SetSource(&source,
1782 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001783 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1784 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001785 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001786
1787 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001789 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1790 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1791 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1792 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1793
1794 // Trigger adapt down for same input resolution, expect no change.
1795 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1796 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001797 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001798 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1799 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1800 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1801
1802 // Trigger adapt down for larger input resolution, expect no change.
1803 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1804 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001805 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001806 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1807 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1808 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1809
mflodmancc3d4422017-08-03 08:27:51 -07001810 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001811}
1812
mflodmancc3d4422017-08-03 08:27:51 -07001813TEST_F(VideoStreamEncoderTest,
1814 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001815 const int kWidth = 1280;
1816 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001817 video_stream_encoder_->OnBitrateUpdated(
1818 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001819
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001820 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001821 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001822 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001823 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001824
1825 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001826 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001827 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001828 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1830
1831 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001832 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001833 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001834 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1835 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1836
mflodmancc3d4422017-08-03 08:27:51 -07001837 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001838}
1839
mflodmancc3d4422017-08-03 08:27:51 -07001840TEST_F(VideoStreamEncoderTest,
1841 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001842 const int kWidth = 1280;
1843 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001844 video_stream_encoder_->OnBitrateUpdated(
1845 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001846
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001847 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001848 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001849 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001850 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001851
1852 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001853 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001854 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1857
1858 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001859 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001860 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001861 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001862 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1863
mflodmancc3d4422017-08-03 08:27:51 -07001864 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001865}
1866
mflodmancc3d4422017-08-03 08:27:51 -07001867TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001868 const int kWidth = 1280;
1869 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001870 video_stream_encoder_->OnBitrateUpdated(
1871 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001872
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001873 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001874 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001875 video_stream_encoder_->SetSource(&source,
1876 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001877
1878 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1879 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001880 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001881 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1884
1885 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001887 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001888 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1889 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1890 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1891
mflodmancc3d4422017-08-03 08:27:51 -07001892 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001893}
1894
mflodmancc3d4422017-08-03 08:27:51 -07001895TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001896 const int kWidth = 1280;
1897 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001898 video_stream_encoder_->OnBitrateUpdated(
1899 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001900
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001901 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001902 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001903 video_stream_encoder_->SetSource(&source,
1904 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001905
1906 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1907 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001908 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001909 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1911 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1912
1913 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001914 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001915 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001916 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1917 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1918 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1919
mflodmancc3d4422017-08-03 08:27:51 -07001920 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001921}
1922
mflodmancc3d4422017-08-03 08:27:51 -07001923TEST_F(VideoStreamEncoderTest,
1924 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001925 const int kWidth = 1280;
1926 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001927 video_stream_encoder_->OnBitrateUpdated(
1928 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001929
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001930 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001931 AdaptingFrameForwarder source;
1932 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001933 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001934 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001935
1936 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001937 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001938 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001939 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1940 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1941
1942 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001943 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001944 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001945 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001946 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001947 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1948 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1949
1950 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001951 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001952 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001953 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1954 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1955 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1956
mflodmancc3d4422017-08-03 08:27:51 -07001957 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001958}
1959
mflodmancc3d4422017-08-03 08:27:51 -07001960TEST_F(VideoStreamEncoderTest,
1961 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001962 const int kWidth = 1280;
1963 const int kHeight = 720;
1964 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02001965 video_stream_encoder_->OnBitrateUpdated(
1966 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001967
1968 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1969 stats.input_frame_rate = kInputFps;
1970 stats_proxy_->SetMockStats(stats);
1971
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001972 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001973 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1974 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001975 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001976
1977 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001978 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001979 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1980 sink_.WaitForEncodedFrame(2);
1981 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1982
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001983 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001984 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001985 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001986 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001987 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001988
1989 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07001990 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001991 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1992 sink_.WaitForEncodedFrame(3);
1993 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1994
1995 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001996 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001997 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001998
mflodmancc3d4422017-08-03 08:27:51 -07001999 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002000}
2001
mflodmancc3d4422017-08-03 08:27:51 -07002002TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002003 const int kWidth = 1280;
2004 const int kHeight = 720;
2005 const size_t kNumFrames = 10;
2006
Erik Språng4c6ca302019-04-08 15:14:01 +02002007 video_stream_encoder_->OnBitrateUpdated(
2008 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002009
asaperssond0de2952017-04-21 01:47:31 -07002010 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002011 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002012 video_source_.set_adaptation_enabled(true);
2013
2014 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2016
2017 int downscales = 0;
2018 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002019 video_source_.IncomingCapturedFrame(
2020 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2021 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002022
asaperssonfab67072017-04-04 05:51:49 -07002023 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002024 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002025 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002026 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002027
2028 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2029 ++downscales;
2030
2031 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2032 EXPECT_EQ(downscales,
2033 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2034 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002035 }
mflodmancc3d4422017-08-03 08:27:51 -07002036 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002037}
2038
mflodmancc3d4422017-08-03 08:27:51 -07002039TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002040 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2041 const int kWidth = 1280;
2042 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002043 video_stream_encoder_->OnBitrateUpdated(
2044 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002045
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002046 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002047 AdaptingFrameForwarder source;
2048 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002049 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002050 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002051
Åsa Persson8c1bf952018-09-13 10:42:19 +02002052 int64_t timestamp_ms = kFrameIntervalMs;
2053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002054 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002055 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002056 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2057 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2058
2059 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002060 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002061 timestamp_ms += kFrameIntervalMs;
2062 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2063 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002064 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002065 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2066 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2067
2068 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002069 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002070 timestamp_ms += kFrameIntervalMs;
2071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002072 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002073 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002074 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2075 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2076
2077 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002078 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002079 timestamp_ms += kFrameIntervalMs;
2080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2081 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002082 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002083 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2084 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2085
2086 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002087 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002088 timestamp_ms += kFrameIntervalMs;
2089 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002090 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002091 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002092 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2093 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2094
mflodmancc3d4422017-08-03 08:27:51 -07002095 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002096}
2097
mflodmancc3d4422017-08-03 08:27:51 -07002098TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002099 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2100 const int kWidth = 1280;
2101 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002102 video_stream_encoder_->OnBitrateUpdated(
2103 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002104
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002105 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002106 AdaptingFrameForwarder source;
2107 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002108 video_stream_encoder_->SetSource(&source,
2109 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002110
Åsa Persson8c1bf952018-09-13 10:42:19 +02002111 int64_t timestamp_ms = kFrameIntervalMs;
2112 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002113 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002114 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2116 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2117
2118 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002119 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002120 timestamp_ms += kFrameIntervalMs;
2121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2122 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002123 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2124 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2125 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2126
2127 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002128 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002129 timestamp_ms += kFrameIntervalMs;
2130 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002131 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002132 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002133 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2134 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2135
2136 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002137 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002138 timestamp_ms += kFrameIntervalMs;
2139 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2140 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002141 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2142 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2143 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2144
2145 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002146 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002147 timestamp_ms += kFrameIntervalMs;
2148 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002149 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002150 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002151 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2152 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2153
mflodmancc3d4422017-08-03 08:27:51 -07002154 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002155}
2156
mflodmancc3d4422017-08-03 08:27:51 -07002157TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002158 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2159 const int kWidth = 1280;
2160 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002161 video_stream_encoder_->OnBitrateUpdated(
2162 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002163
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002164 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002165 AdaptingFrameForwarder source;
2166 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002168 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002169
Åsa Persson8c1bf952018-09-13 10:42:19 +02002170 int64_t timestamp_ms = kFrameIntervalMs;
2171 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002172 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002173 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002174 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2175 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2176 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2177 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2178
2179 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002180 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002181 timestamp_ms += kFrameIntervalMs;
2182 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2183 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002184 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002185 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2186 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2187 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2188 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2189
2190 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002191 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002192 timestamp_ms += kFrameIntervalMs;
2193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2194 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002195 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002196 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2197 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2198 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2199 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2200
Jonathan Yubc771b72017-12-08 17:04:29 -08002201 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002202 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002203 timestamp_ms += kFrameIntervalMs;
2204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2205 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002206 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002207 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002209 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002210 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2211
Jonathan Yubc771b72017-12-08 17:04:29 -08002212 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002214 timestamp_ms += kFrameIntervalMs;
2215 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2216 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002217 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002218 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002219 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2220 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2221 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2222 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2223
Jonathan Yubc771b72017-12-08 17:04:29 -08002224 // Trigger quality adapt down, expect no change (min resolution reached).
2225 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002226 timestamp_ms += kFrameIntervalMs;
2227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2228 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002229 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2230 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2231 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2232 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2233 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2234
2235 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002236 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002237 timestamp_ms += kFrameIntervalMs;
2238 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2239 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002240 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002241 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2242 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2243 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2244 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2245
2246 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2247 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002248 timestamp_ms += kFrameIntervalMs;
2249 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2250 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002251 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2252 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2253 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2254 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2256
2257 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2258 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002259 timestamp_ms += kFrameIntervalMs;
2260 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2261 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002262 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002263 last_wants = source.sink_wants();
2264 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2265 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002266 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002267 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2268
2269 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002270 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002271 timestamp_ms += kFrameIntervalMs;
2272 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2273 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002274 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002275 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002277 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002278 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2279
2280 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002281 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002282 timestamp_ms += kFrameIntervalMs;
2283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002284 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002285 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002286 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002287 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2288 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002289 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002290 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002291
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002293}
2294
mflodmancc3d4422017-08-03 08:27:51 -07002295TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002296 const int kWidth = 640;
2297 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002298
Erik Språng4c6ca302019-04-08 15:14:01 +02002299 video_stream_encoder_->OnBitrateUpdated(
2300 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002301
perkj803d97f2016-11-01 11:45:46 -07002302 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002303 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002304 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002305 }
2306
mflodmancc3d4422017-08-03 08:27:51 -07002307 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002308 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002309 video_source_.IncomingCapturedFrame(CreateFrame(
2310 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002311 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002312 }
2313
mflodmancc3d4422017-08-03 08:27:51 -07002314 video_stream_encoder_->Stop();
2315 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002316 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002317
perkj803d97f2016-11-01 11:45:46 -07002318 EXPECT_EQ(1,
2319 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2320 EXPECT_EQ(
2321 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2322}
2323
mflodmancc3d4422017-08-03 08:27:51 -07002324TEST_F(VideoStreamEncoderTest,
2325 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002326 video_stream_encoder_->OnBitrateUpdated(
2327 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002328 const int kWidth = 640;
2329 const int kHeight = 360;
2330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002331 video_stream_encoder_->SetSource(&video_source_,
2332 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002333
2334 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2335 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002336 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002337 }
2338
mflodmancc3d4422017-08-03 08:27:51 -07002339 video_stream_encoder_->Stop();
2340 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002341 stats_proxy_.reset();
2342
2343 EXPECT_EQ(0,
2344 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2345}
2346
mflodmancc3d4422017-08-03 08:27:51 -07002347TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002348 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002349 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002350
2351 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002352 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002353 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002354 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002355
sprang57c2fff2017-01-16 06:24:02 -08002356 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002357 .Times(1);
Erik Språng610c7632019-03-06 15:37:33 +01002358 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002359 DataRate::bps(kLowTargetBitrateBps),
2360 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002361
sprang57c2fff2017-01-16 06:24:02 -08002362 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002363 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2364 WaitForEncodedFrame(rtc::TimeMillis());
2365 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2366 fake_encoder_.GetAndResetLastBitrateAllocation();
2367 // Check that encoder has been updated too, not just allocation observer.
2368 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02002369 // TODO(srte): The use of millisecs here looks like an error, but the tests
2370 // fails using seconds, this should be investigated.
2371 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002372
2373 // Not called on second frame.
2374 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2375 .Times(0);
2376 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002377 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2378 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002379 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002380
2381 // Called after a process interval.
2382 const int64_t kProcessIntervalMs =
2383 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002384 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2385 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002386 const int64_t start_time_ms = rtc::TimeMillis();
2387 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2388 video_source_.IncomingCapturedFrame(
2389 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2390 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002391 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01002392 }
2393
2394 // Since rates are unchanged, encoder should not be reconfigured.
2395 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002396
mflodmancc3d4422017-08-03 08:27:51 -07002397 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002398}
2399
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01002400TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
2401 // 2 TLs configured, temporal layers supported by encoder.
2402 const int kNumTemporalLayers = 2;
2403 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
2404 fake_encoder_.SetTemporalLayersSupported(0, true);
2405
2406 // Bitrate allocated across temporal layers.
2407 const int kTl0Bps = kTargetBitrateBps *
2408 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2409 kNumTemporalLayers, /*temporal_id*/ 0);
2410 const int kTl1Bps = kTargetBitrateBps *
2411 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2412 kNumTemporalLayers, /*temporal_id*/ 1);
2413 VideoBitrateAllocation expected_bitrate;
2414 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
2415 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
2416
2417 VerifyAllocatedBitrate(expected_bitrate);
2418 video_stream_encoder_->Stop();
2419}
2420
2421TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
2422 // 2 TLs configured, temporal layers not supported by encoder.
2423 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2424 fake_encoder_.SetTemporalLayersSupported(0, false);
2425
2426 // Temporal layers not supported by the encoder.
2427 // Total bitrate should be at ti:0.
2428 VideoBitrateAllocation expected_bitrate;
2429 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
2430
2431 VerifyAllocatedBitrate(expected_bitrate);
2432 video_stream_encoder_->Stop();
2433}
2434
2435TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
2436 // 2 TLs configured, temporal layers only supported for first stream.
2437 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2438 fake_encoder_.SetTemporalLayersSupported(0, true);
2439 fake_encoder_.SetTemporalLayersSupported(1, false);
2440
2441 const int kS0Bps = 150000;
2442 const int kS0Tl0Bps =
2443 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2444 /*num_layers*/ 2, /*temporal_id*/ 0);
2445 const int kS0Tl1Bps =
2446 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2447 /*num_layers*/ 2, /*temporal_id*/ 1);
2448 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
2449 // Temporal layers not supported by si:1.
2450 VideoBitrateAllocation expected_bitrate;
2451 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
2452 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
2453 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
2454
2455 VerifyAllocatedBitrate(expected_bitrate);
2456 video_stream_encoder_->Stop();
2457}
2458
Niels Möller7dc26b72017-12-06 10:27:48 +01002459TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2460 const int kFrameWidth = 1280;
2461 const int kFrameHeight = 720;
2462 const int kFramerate = 24;
2463
Erik Språng4c6ca302019-04-08 15:14:01 +02002464 video_stream_encoder_->OnBitrateUpdated(
2465 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002466 test::FrameForwarder source;
2467 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002468 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002469
2470 // Insert a single frame, triggering initial configuration.
2471 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2472 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2473
2474 EXPECT_EQ(
2475 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2476 kDefaultFramerate);
2477
2478 // Trigger reconfigure encoder (without resetting the entire instance).
2479 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002480 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002481 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2482 video_encoder_config.number_of_streams = 1;
2483 video_encoder_config.video_stream_factory =
2484 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2485 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002486 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002487 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2488
2489 // Detector should be updated with fps limit from codec config.
2490 EXPECT_EQ(
2491 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2492 kFramerate);
2493
2494 // Trigger overuse, max framerate should be reduced.
2495 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2496 stats.input_frame_rate = kFramerate;
2497 stats_proxy_->SetMockStats(stats);
2498 video_stream_encoder_->TriggerCpuOveruse();
2499 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2500 int adapted_framerate =
2501 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2502 EXPECT_LT(adapted_framerate, kFramerate);
2503
2504 // Trigger underuse, max framerate should go back to codec configured fps.
2505 // Set extra low fps, to make sure it's actually reset, not just incremented.
2506 stats = stats_proxy_->GetStats();
2507 stats.input_frame_rate = adapted_framerate / 2;
2508 stats_proxy_->SetMockStats(stats);
2509 video_stream_encoder_->TriggerCpuNormalUsage();
2510 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2511 EXPECT_EQ(
2512 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2513 kFramerate);
2514
2515 video_stream_encoder_->Stop();
2516}
2517
2518TEST_F(VideoStreamEncoderTest,
2519 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2520 const int kFrameWidth = 1280;
2521 const int kFrameHeight = 720;
2522 const int kLowFramerate = 15;
2523 const int kHighFramerate = 25;
2524
Erik Språng4c6ca302019-04-08 15:14:01 +02002525 video_stream_encoder_->OnBitrateUpdated(
2526 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002527 test::FrameForwarder source;
2528 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002529 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002530
2531 // Trigger initial configuration.
2532 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002533 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002534 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2535 video_encoder_config.number_of_streams = 1;
2536 video_encoder_config.video_stream_factory =
2537 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2538 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2539 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002540 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002541 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2542
2543 EXPECT_EQ(
2544 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2545 kLowFramerate);
2546
2547 // Trigger overuse, max framerate should be reduced.
2548 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2549 stats.input_frame_rate = kLowFramerate;
2550 stats_proxy_->SetMockStats(stats);
2551 video_stream_encoder_->TriggerCpuOveruse();
2552 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2553 int adapted_framerate =
2554 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2555 EXPECT_LT(adapted_framerate, kLowFramerate);
2556
2557 // Reconfigure the encoder with a new (higher max framerate), max fps should
2558 // still respect the adaptation.
2559 video_encoder_config.video_stream_factory =
2560 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2561 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2562 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002563 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002564 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2565
2566 EXPECT_EQ(
2567 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2568 adapted_framerate);
2569
2570 // Trigger underuse, max framerate should go back to codec configured fps.
2571 stats = stats_proxy_->GetStats();
2572 stats.input_frame_rate = adapted_framerate;
2573 stats_proxy_->SetMockStats(stats);
2574 video_stream_encoder_->TriggerCpuNormalUsage();
2575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2576 EXPECT_EQ(
2577 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2578 kHighFramerate);
2579
2580 video_stream_encoder_->Stop();
2581}
2582
mflodmancc3d4422017-08-03 08:27:51 -07002583TEST_F(VideoStreamEncoderTest,
2584 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002585 const int kFrameWidth = 1280;
2586 const int kFrameHeight = 720;
2587 const int kFramerate = 24;
2588
Erik Språng4c6ca302019-04-08 15:14:01 +02002589 video_stream_encoder_->OnBitrateUpdated(
2590 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002591 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002592 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002593 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002594
2595 // Trigger initial configuration.
2596 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002597 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002598 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2599 video_encoder_config.number_of_streams = 1;
2600 video_encoder_config.video_stream_factory =
2601 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2602 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002603 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002604 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002605 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002606
Niels Möller7dc26b72017-12-06 10:27:48 +01002607 EXPECT_EQ(
2608 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2609 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002610
2611 // Trigger overuse, max framerate should be reduced.
2612 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2613 stats.input_frame_rate = kFramerate;
2614 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002615 video_stream_encoder_->TriggerCpuOveruse();
2616 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002617 int adapted_framerate =
2618 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002619 EXPECT_LT(adapted_framerate, kFramerate);
2620
2621 // Change degradation preference to not enable framerate scaling. Target
2622 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002623 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002624 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002625 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002626 EXPECT_EQ(
2627 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2628 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002629
mflodmancc3d4422017-08-03 08:27:51 -07002630 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002631}
2632
mflodmancc3d4422017-08-03 08:27:51 -07002633TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002634 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002635 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002636 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2637 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002638 const int kWidth = 640;
2639 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002640
asaperssonfab67072017-04-04 05:51:49 -07002641 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002642
2643 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002644 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002645
2646 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002647 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002648
sprangc5d62e22017-04-02 23:53:04 -07002649 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002650
asaperssonfab67072017-04-04 05:51:49 -07002651 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002652 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002653 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002654
2655 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002656 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002657
sprangc5d62e22017-04-02 23:53:04 -07002658 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002659
mflodmancc3d4422017-08-03 08:27:51 -07002660 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002661}
2662
mflodmancc3d4422017-08-03 08:27:51 -07002663TEST_F(VideoStreamEncoderTest,
2664 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002665 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002666 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002667 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2668 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002669 const int kWidth = 640;
2670 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002671
2672 // We expect the n initial frames to get dropped.
2673 int i;
2674 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002675 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002676 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002677 }
2678 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002679 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002680 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002681
2682 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002683 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002684
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002686}
2687
mflodmancc3d4422017-08-03 08:27:51 -07002688TEST_F(VideoStreamEncoderTest,
2689 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002690 const int kWidth = 640;
2691 const int kHeight = 360;
Erik Språng610c7632019-03-06 15:37:33 +01002692 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002693 DataRate::bps(kLowTargetBitrateBps),
2694 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002695
2696 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002697 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002698 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002699
asaperssonfab67072017-04-04 05:51:49 -07002700 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002701 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002702 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002703
mflodmancc3d4422017-08-03 08:27:51 -07002704 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002705}
2706
mflodmancc3d4422017-08-03 08:27:51 -07002707TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002708 const int kWidth = 640;
2709 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002710 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002711
2712 VideoEncoderConfig video_encoder_config;
2713 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2714 // Make format different, to force recreation of encoder.
2715 video_encoder_config.video_format.parameters["foo"] = "foo";
2716 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002717 kMaxPayloadLength);
Erik Språng610c7632019-03-06 15:37:33 +01002718 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002719 DataRate::bps(kLowTargetBitrateBps),
2720 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002721
kthelgasonb83797b2017-02-14 11:57:25 -08002722 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002723 video_stream_encoder_->SetSource(&video_source_,
2724 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002725
asaperssonfab67072017-04-04 05:51:49 -07002726 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002727 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002728 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002729
mflodmancc3d4422017-08-03 08:27:51 -07002730 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002731 fake_encoder_.SetQualityScaling(true);
2732}
2733
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002734TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2735 webrtc::test::ScopedFieldTrials field_trials(
2736 "WebRTC-InitialFramedrop/Enabled/");
2737 // Reset encoder for field trials to take effect.
2738 ConfigureEncoder(video_encoder_config_.Copy());
2739 const int kTooLowBitrateForFrameSizeBps = 10000;
2740 const int kWidth = 640;
2741 const int kHeight = 360;
2742
Erik Språng4c6ca302019-04-08 15:14:01 +02002743 video_stream_encoder_->OnBitrateUpdated(
2744 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002745 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2746 // Frame should not be dropped.
2747 WaitForEncodedFrame(1);
2748
Erik Språng610c7632019-03-06 15:37:33 +01002749 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002750 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2751 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002752 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2753 // Expect to drop this frame, the wait should time out.
2754 ExpectDroppedFrame();
2755
2756 // Expect the sink_wants to specify a scaled frame.
2757 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2758 video_stream_encoder_->Stop();
2759}
2760
mflodmancc3d4422017-08-03 08:27:51 -07002761TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002762 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2763 const int kTooSmallWidth = 10;
2764 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02002765 video_stream_encoder_->OnBitrateUpdated(
2766 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002767
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002768 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002769 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002770 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002771 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002772 VerifyNoLimitation(source.sink_wants());
2773 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2774
2775 // Trigger adapt down, too small frame, expect no change.
2776 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002777 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002778 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002779 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2781 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2782
mflodmancc3d4422017-08-03 08:27:51 -07002783 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002784}
2785
mflodmancc3d4422017-08-03 08:27:51 -07002786TEST_F(VideoStreamEncoderTest,
2787 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002788 const int kTooSmallWidth = 10;
2789 const int kTooSmallHeight = 10;
2790 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02002791 video_stream_encoder_->OnBitrateUpdated(
2792 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002793
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002794 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002795 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002796 video_stream_encoder_->SetSource(&source,
2797 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002798 VerifyNoLimitation(source.sink_wants());
2799 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2800 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2801
2802 // Trigger adapt down, expect limited framerate.
2803 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002804 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002805 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002806 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2808 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2809 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2810
2811 // Trigger adapt down, too small frame, expect no change.
2812 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002814 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002815 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2817 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2818 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2819
mflodmancc3d4422017-08-03 08:27:51 -07002820 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002821}
2822
mflodmancc3d4422017-08-03 08:27:51 -07002823TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002824 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02002825 video_stream_encoder_->OnBitrateUpdated(
2826 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002827 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002828 const int kFrameWidth = 1280;
2829 const int kFrameHeight = 720;
2830 video_source_.IncomingCapturedFrame(
2831 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002832 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002833 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002834}
2835
sprangb1ca0732017-02-01 08:38:12 -08002836// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002837TEST_F(VideoStreamEncoderTest,
2838 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002839 video_stream_encoder_->OnBitrateUpdated(
2840 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002841
2842 const int kFrameWidth = 1280;
2843 const int kFrameHeight = 720;
2844 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002845 // requested by
2846 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002847 video_source_.set_adaptation_enabled(true);
2848
2849 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002850 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002851 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002852
2853 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002854 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002855 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002856 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002857 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002858
asaperssonfab67072017-04-04 05:51:49 -07002859 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002860 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002861 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002862 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002863 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002864
mflodmancc3d4422017-08-03 08:27:51 -07002865 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002866}
sprangfe627f32017-03-29 08:24:59 -07002867
mflodmancc3d4422017-08-03 08:27:51 -07002868TEST_F(VideoStreamEncoderTest,
2869 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002870 const int kFrameWidth = 1280;
2871 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002872
Erik Språng4c6ca302019-04-08 15:14:01 +02002873 video_stream_encoder_->OnBitrateUpdated(
2874 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002875 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002876 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002877 video_source_.set_adaptation_enabled(true);
2878
sprang4847ae62017-06-27 07:06:52 -07002879 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002880
2881 video_source_.IncomingCapturedFrame(
2882 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002883 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002884
2885 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002886 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002887
2888 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002889 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002890 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002891 video_source_.IncomingCapturedFrame(
2892 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002893 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002894 }
2895
2896 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002898 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002899 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002900 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002901 video_source_.IncomingCapturedFrame(
2902 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002904 ++num_frames_dropped;
2905 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002906 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002907 }
2908 }
2909
sprang4847ae62017-06-27 07:06:52 -07002910 // Add some slack to account for frames dropped by the frame dropper.
2911 const int kErrorMargin = 1;
2912 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002913 kErrorMargin);
2914
2915 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002916 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002917 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002918 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002919 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002920 video_source_.IncomingCapturedFrame(
2921 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002922 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002923 ++num_frames_dropped;
2924 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002925 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002926 }
2927 }
sprang4847ae62017-06-27 07:06:52 -07002928 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002929 kErrorMargin);
2930
2931 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002932 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002933 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002934 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002935 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002936 video_source_.IncomingCapturedFrame(
2937 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002938 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002939 ++num_frames_dropped;
2940 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002941 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002942 }
2943 }
sprang4847ae62017-06-27 07:06:52 -07002944 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002945 kErrorMargin);
2946
2947 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002948 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002949 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002950 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002951 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002952 video_source_.IncomingCapturedFrame(
2953 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002955 ++num_frames_dropped;
2956 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002957 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002958 }
2959 }
2960 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2961
mflodmancc3d4422017-08-03 08:27:51 -07002962 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002963}
2964
mflodmancc3d4422017-08-03 08:27:51 -07002965TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002966 const int kFramerateFps = 5;
2967 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002968 const int kFrameWidth = 1280;
2969 const int kFrameHeight = 720;
2970
sprang4847ae62017-06-27 07:06:52 -07002971 // Reconfigure encoder with two temporal layers and screensharing, which will
2972 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002973 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002974
Erik Språng4c6ca302019-04-08 15:14:01 +02002975 video_stream_encoder_->OnBitrateUpdated(
2976 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002977 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002978 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002979 video_source_.set_adaptation_enabled(true);
2980
sprang4847ae62017-06-27 07:06:52 -07002981 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002982
2983 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002984 rtc::VideoSinkWants last_wants;
2985 do {
2986 last_wants = video_source_.sink_wants();
2987
sprangc5d62e22017-04-02 23:53:04 -07002988 // Insert frames to get a new fps estimate...
2989 for (int j = 0; j < kFramerateFps; ++j) {
2990 video_source_.IncomingCapturedFrame(
2991 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08002992 if (video_source_.last_sent_width()) {
2993 sink_.WaitForEncodedFrame(timestamp_ms);
2994 }
sprangc5d62e22017-04-02 23:53:04 -07002995 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02002996 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07002997 }
2998 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07002999 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003000 } while (video_source_.sink_wants().max_framerate_fps <
3001 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003002
Jonathan Yubc771b72017-12-08 17:04:29 -08003003 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003004
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003006}
asaperssonf7e294d2017-06-13 23:25:22 -07003007
mflodmancc3d4422017-08-03 08:27:51 -07003008TEST_F(VideoStreamEncoderTest,
3009 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003010 const int kWidth = 1280;
3011 const int kHeight = 720;
3012 const int64_t kFrameIntervalMs = 150;
3013 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003014 video_stream_encoder_->OnBitrateUpdated(
3015 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003016
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003017 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003018 AdaptingFrameForwarder source;
3019 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003020 video_stream_encoder_->SetSource(&source,
3021 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003022 timestamp_ms += kFrameIntervalMs;
3023 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003024 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003025 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003026 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3027 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3029
3030 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003031 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003032 timestamp_ms += kFrameIntervalMs;
3033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003034 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003035 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3036 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3038 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3039
3040 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003042 timestamp_ms += kFrameIntervalMs;
3043 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003044 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003045 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3046 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3047 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3048 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3049
3050 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003051 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003052 timestamp_ms += kFrameIntervalMs;
3053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003054 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003055 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3056 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3057 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3058 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3059
3060 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003061 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003062 timestamp_ms += kFrameIntervalMs;
3063 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003064 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003065 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3066 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3067 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3068 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3069
3070 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003071 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003072 timestamp_ms += kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003074 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003075 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3078 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3079
3080 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003081 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003082 timestamp_ms += kFrameIntervalMs;
3083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003084 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003085 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3088 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003091 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003092 timestamp_ms += kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003094 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003095 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3096 rtc::VideoSinkWants last_wants = source.sink_wants();
3097 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3098 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3099 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3100
3101 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003102 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003103 timestamp_ms += kFrameIntervalMs;
3104 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003105 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003106 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3107 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3108 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3109 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3110
3111 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003113 timestamp_ms += kFrameIntervalMs;
3114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003116 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3117 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3118 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3119 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3120
3121 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003122 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003123 timestamp_ms += kFrameIntervalMs;
3124 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003125 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003126 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3127 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3128 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3129 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3130
3131 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003132 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003133 timestamp_ms += kFrameIntervalMs;
3134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003135 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003136 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3137 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3138 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3139 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3140
3141 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003142 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003143 timestamp_ms += kFrameIntervalMs;
3144 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003145 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003146 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3147 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3148 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3149 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3150
3151 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003153 timestamp_ms += kFrameIntervalMs;
3154 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003156 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3157 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3158 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3159 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3160
3161 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003163 timestamp_ms += kFrameIntervalMs;
3164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003165 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003166 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3169 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003173 timestamp_ms += kFrameIntervalMs;
3174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003175 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003176 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003177 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3179 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3180 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3181
3182 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003183 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003184 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003185 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3186
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003188}
3189
mflodmancc3d4422017-08-03 08:27:51 -07003190TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003191 const int kWidth = 1280;
3192 const int kHeight = 720;
3193 const int64_t kFrameIntervalMs = 150;
3194 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003195 video_stream_encoder_->OnBitrateUpdated(
3196 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003197
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003198 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003199 AdaptingFrameForwarder source;
3200 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003201 video_stream_encoder_->SetSource(&source,
3202 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003203 timestamp_ms += kFrameIntervalMs;
3204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003205 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003206 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003207 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3209 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3210 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3211 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3212 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3213
3214 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003216 timestamp_ms += kFrameIntervalMs;
3217 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003218 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003219 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3220 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3221 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3222 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3223 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3224 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3225 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3226
3227 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003228 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003229 timestamp_ms += kFrameIntervalMs;
3230 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003231 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003232 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3234 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3235 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3236 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3237 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3238 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3239
3240 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003241 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003242 timestamp_ms += kFrameIntervalMs;
3243 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003244 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003245 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3247 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3248 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3249 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3250 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3251 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3252
3253 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003254 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003255 timestamp_ms += kFrameIntervalMs;
3256 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003257 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003258 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3259 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3260 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3261 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3262 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3263 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3264 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3265
3266 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003267 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003268 timestamp_ms += kFrameIntervalMs;
3269 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003271 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3272 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3273 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3274 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3275 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3276 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3277 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3278
3279 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003280 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003281 timestamp_ms += kFrameIntervalMs;
3282 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003283 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003284 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003285 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3288 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3289 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3290 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3291 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3292
3293 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003295 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003296 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3297 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003300}
3301
mflodmancc3d4422017-08-03 08:27:51 -07003302TEST_F(VideoStreamEncoderTest,
3303 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003304 const int kWidth = 640;
3305 const int kHeight = 360;
3306 const int kFpsLimit = 15;
3307 const int64_t kFrameIntervalMs = 150;
3308 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003309 video_stream_encoder_->OnBitrateUpdated(
3310 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003311
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003312 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003313 AdaptingFrameForwarder source;
3314 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003315 video_stream_encoder_->SetSource(&source,
3316 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003317 timestamp_ms += kFrameIntervalMs;
3318 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003319 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003320 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003321 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3322 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3323 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3324 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3325 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3326 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3327
3328 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003330 timestamp_ms += kFrameIntervalMs;
3331 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003332 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003333 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3334 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3335 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3336 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3337 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3338 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3340
3341 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003343 timestamp_ms += kFrameIntervalMs;
3344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003345 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003346 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3349 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3350 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3351 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3352 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3353
3354 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003356 timestamp_ms += kFrameIntervalMs;
3357 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003358 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003359 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3360 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3361 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3362 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3363 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3364 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3365 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3366
3367 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003368 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003369 timestamp_ms += kFrameIntervalMs;
3370 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003371 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003372 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003373 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3374 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3375 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3376 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3377 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3378 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3379
3380 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003381 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003382 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003383 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3384 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3385
mflodmancc3d4422017-08-03 08:27:51 -07003386 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003387}
3388
mflodmancc3d4422017-08-03 08:27:51 -07003389TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003390 // Simulates simulcast behavior and makes highest stream resolutions divisible
3391 // by 4.
3392 class CroppingVideoStreamFactory
3393 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3394 public:
3395 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3396 int framerate)
3397 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3398 EXPECT_GT(num_temporal_layers, 0u);
3399 EXPECT_GT(framerate, 0);
3400 }
3401
3402 private:
3403 std::vector<VideoStream> CreateEncoderStreams(
3404 int width,
3405 int height,
3406 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003407 std::vector<VideoStream> streams = test::CreateVideoStreams(
3408 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003409 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003410 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003411 stream.max_framerate = framerate_;
3412 }
3413 return streams;
3414 }
3415
3416 const size_t num_temporal_layers_;
3417 const int framerate_;
3418 };
3419
3420 const int kFrameWidth = 1920;
3421 const int kFrameHeight = 1080;
3422 // 3/4 of 1920.
3423 const int kAdaptedFrameWidth = 1440;
3424 // 3/4 of 1080 rounded down to multiple of 4.
3425 const int kAdaptedFrameHeight = 808;
3426 const int kFramerate = 24;
3427
Erik Språng4c6ca302019-04-08 15:14:01 +02003428 video_stream_encoder_->OnBitrateUpdated(
3429 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003430 // Trigger reconfigure encoder (without resetting the entire instance).
3431 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003432 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003433 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3434 video_encoder_config.number_of_streams = 1;
3435 video_encoder_config.video_stream_factory =
3436 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003437 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003438 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003439 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003440
3441 video_source_.set_adaptation_enabled(true);
3442
3443 video_source_.IncomingCapturedFrame(
3444 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003445 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003446
3447 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003448 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003449 video_source_.IncomingCapturedFrame(
3450 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003451 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003452
mflodmancc3d4422017-08-03 08:27:51 -07003453 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003454}
3455
mflodmancc3d4422017-08-03 08:27:51 -07003456TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003457 const int kFrameWidth = 1280;
3458 const int kFrameHeight = 720;
3459 const int kLowFps = 2;
3460 const int kHighFps = 30;
3461
Erik Språng4c6ca302019-04-08 15:14:01 +02003462 video_stream_encoder_->OnBitrateUpdated(
3463 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003464
3465 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3466 max_framerate_ = kLowFps;
3467
3468 // Insert 2 seconds of 2fps video.
3469 for (int i = 0; i < kLowFps * 2; ++i) {
3470 video_source_.IncomingCapturedFrame(
3471 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3472 WaitForEncodedFrame(timestamp_ms);
3473 timestamp_ms += 1000 / kLowFps;
3474 }
3475
3476 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02003477 video_stream_encoder_->OnBitrateUpdated(
3478 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003479 video_source_.IncomingCapturedFrame(
3480 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3481 WaitForEncodedFrame(timestamp_ms);
3482 timestamp_ms += 1000 / kLowFps;
3483
3484 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3485
3486 // Insert 30fps frames for just a little more than the forced update period.
3487 const int kVcmTimerIntervalFrames =
3488 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3489 const int kFrameIntervalMs = 1000 / kHighFps;
3490 max_framerate_ = kHighFps;
3491 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3492 video_source_.IncomingCapturedFrame(
3493 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3494 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3495 // be dropped if the encoder hans't been updated with the new higher target
3496 // framerate yet, causing it to overshoot the target bitrate and then
3497 // suffering the wrath of the media optimizer.
3498 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3499 timestamp_ms += kFrameIntervalMs;
3500 }
3501
3502 // Don expect correct measurement just yet, but it should be higher than
3503 // before.
3504 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3505
mflodmancc3d4422017-08-03 08:27:51 -07003506 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003507}
3508
mflodmancc3d4422017-08-03 08:27:51 -07003509TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003510 const int kFrameWidth = 1280;
3511 const int kFrameHeight = 720;
3512 const int kTargetBitrateBps = 1000000;
3513
3514 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003515 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02003516 video_stream_encoder_->OnBitrateUpdated(
3517 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003518 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003519
3520 // Insert a first video frame, causes another bitrate update.
3521 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3522 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3523 video_source_.IncomingCapturedFrame(
3524 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3525 WaitForEncodedFrame(timestamp_ms);
3526
3527 // Next, simulate video suspension due to pacer queue overrun.
Erik Språng4c6ca302019-04-08 15:14:01 +02003528 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01003529 1);
sprang4847ae62017-06-27 07:06:52 -07003530
3531 // Skip ahead until a new periodic parameter update should have occured.
3532 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003533 fake_clock_.AdvanceTime(
3534 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07003535
3536 // Bitrate observer should not be called.
3537 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3538 video_source_.IncomingCapturedFrame(
3539 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3540 ExpectDroppedFrame();
3541
mflodmancc3d4422017-08-03 08:27:51 -07003542 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003543}
ilnik6b826ef2017-06-16 06:53:48 -07003544
Niels Möller4db138e2018-04-19 09:04:13 +02003545TEST_F(VideoStreamEncoderTest,
3546 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3547 const int kFrameWidth = 1280;
3548 const int kFrameHeight = 720;
3549 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02003550 video_stream_encoder_->OnBitrateUpdated(
3551 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003552 video_source_.IncomingCapturedFrame(
3553 CreateFrame(1, kFrameWidth, kFrameHeight));
3554 WaitForEncodedFrame(1);
3555 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3556 .low_encode_usage_threshold_percent,
3557 default_options.low_encode_usage_threshold_percent);
3558 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3559 .high_encode_usage_threshold_percent,
3560 default_options.high_encode_usage_threshold_percent);
3561 video_stream_encoder_->Stop();
3562}
3563
3564TEST_F(VideoStreamEncoderTest,
3565 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3566 const int kFrameWidth = 1280;
3567 const int kFrameHeight = 720;
3568 CpuOveruseOptions hardware_options;
3569 hardware_options.low_encode_usage_threshold_percent = 150;
3570 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003571 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003572
Erik Språng4c6ca302019-04-08 15:14:01 +02003573 video_stream_encoder_->OnBitrateUpdated(
3574 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003575 video_source_.IncomingCapturedFrame(
3576 CreateFrame(1, kFrameWidth, kFrameHeight));
3577 WaitForEncodedFrame(1);
3578 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3579 .low_encode_usage_threshold_percent,
3580 hardware_options.low_encode_usage_threshold_percent);
3581 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3582 .high_encode_usage_threshold_percent,
3583 hardware_options.high_encode_usage_threshold_percent);
3584 video_stream_encoder_->Stop();
3585}
3586
Niels Möller6bb5ab92019-01-11 11:11:10 +01003587TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3588 const int kFrameWidth = 320;
3589 const int kFrameHeight = 240;
3590 const int kFps = 30;
3591 const int kTargetBitrateBps = 120000;
3592 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3593
Erik Språng4c6ca302019-04-08 15:14:01 +02003594 video_stream_encoder_->OnBitrateUpdated(
3595 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003596
3597 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3598 max_framerate_ = kFps;
3599
3600 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3601 fake_encoder_.SimulateOvershoot(1.0);
3602 int num_dropped = 0;
3603 for (int i = 0; i < kNumFramesInRun; ++i) {
3604 video_source_.IncomingCapturedFrame(
3605 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3606 // Wait up to two frame durations for a frame to arrive.
3607 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3608 ++num_dropped;
3609 }
3610 timestamp_ms += 1000 / kFps;
3611 }
3612
Erik Språnga8d48ab2019-02-08 14:17:40 +01003613 // Framerate should be measured to be near the expected target rate.
3614 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3615
3616 // Frame drops should be within 5% of expected 0%.
3617 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003618
3619 // Make encoder produce frames at double the expected bitrate during 3 seconds
3620 // of video, verify number of drops. Rate needs to be slightly changed in
3621 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003622 double overshoot_factor = 2.0;
3623 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3624 // With bitrate adjuster, when need to overshoot even more to trigger
3625 // frame dropping.
3626 overshoot_factor *= 2;
3627 }
3628 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01003629 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003630 DataRate::bps(kTargetBitrateBps + 1000),
3631 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003632 num_dropped = 0;
3633 for (int i = 0; i < kNumFramesInRun; ++i) {
3634 video_source_.IncomingCapturedFrame(
3635 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3636 // Wait up to two frame durations for a frame to arrive.
3637 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3638 ++num_dropped;
3639 }
3640 timestamp_ms += 1000 / kFps;
3641 }
3642
Erik Språng4c6ca302019-04-08 15:14:01 +02003643 video_stream_encoder_->OnBitrateUpdated(
3644 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01003645
3646 // Target framerate should be still be near the expected target, despite
3647 // the frame drops.
3648 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3649
3650 // Frame drops should be within 5% of expected 50%.
3651 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003652
3653 video_stream_encoder_->Stop();
3654}
3655
3656TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3657 const int kFrameWidth = 320;
3658 const int kFrameHeight = 240;
3659 const int kActualInputFps = 24;
3660 const int kTargetBitrateBps = 120000;
3661
3662 ASSERT_GT(max_framerate_, kActualInputFps);
3663
3664 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3665 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02003666 video_stream_encoder_->OnBitrateUpdated(
3667 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003668
3669 // Insert 3 seconds of video, with an input fps lower than configured max.
3670 for (int i = 0; i < kActualInputFps * 3; ++i) {
3671 video_source_.IncomingCapturedFrame(
3672 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3673 // Wait up to two frame durations for a frame to arrive.
3674 WaitForEncodedFrame(timestamp_ms);
3675 timestamp_ms += 1000 / kActualInputFps;
3676 }
3677
3678 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3679
3680 video_stream_encoder_->Stop();
3681}
3682
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003683TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3684 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02003685 video_stream_encoder_->OnBitrateUpdated(
3686 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003687
3688 fake_encoder_.BlockNextEncode();
3689 video_source_.IncomingCapturedFrame(
3690 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3691 WaitForEncodedFrame(1);
3692 // On the very first frame full update should be forced.
3693 rect = fake_encoder_.GetLastUpdateRect();
3694 EXPECT_EQ(rect.offset_x, 0);
3695 EXPECT_EQ(rect.offset_y, 0);
3696 EXPECT_EQ(rect.height, codec_height_);
3697 EXPECT_EQ(rect.width, codec_width_);
3698 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3699 // call to ContinueEncode.
3700 video_source_.IncomingCapturedFrame(
3701 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3702 ExpectDroppedFrame();
3703 video_source_.IncomingCapturedFrame(
3704 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3705 ExpectDroppedFrame();
3706 fake_encoder_.ContinueEncode();
3707 WaitForEncodedFrame(3);
3708 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3709 rect = fake_encoder_.GetLastUpdateRect();
3710 EXPECT_EQ(rect.offset_x, 1);
3711 EXPECT_EQ(rect.offset_y, 0);
3712 EXPECT_EQ(rect.width, 10);
3713 EXPECT_EQ(rect.height, 1);
3714
3715 video_source_.IncomingCapturedFrame(
3716 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3717 WaitForEncodedFrame(4);
3718 // Previous frame was encoded, so no accumulation should happen.
3719 rect = fake_encoder_.GetLastUpdateRect();
3720 EXPECT_EQ(rect.offset_x, 0);
3721 EXPECT_EQ(rect.offset_y, 0);
3722 EXPECT_EQ(rect.width, 1);
3723 EXPECT_EQ(rect.height, 1);
3724
3725 video_stream_encoder_->Stop();
3726}
3727
Erik Språngd7329ca2019-02-21 21:19:53 +01003728TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003729 video_stream_encoder_->OnBitrateUpdated(
3730 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003731
3732 // First frame is always keyframe.
3733 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3734 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01003735 EXPECT_THAT(
3736 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003737 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003738
3739 // Insert delta frame.
3740 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3741 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01003742 EXPECT_THAT(
3743 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003744 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003745
3746 // Request next frame be a key-frame.
3747 video_stream_encoder_->SendKeyFrame();
3748 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3749 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01003750 EXPECT_THAT(
3751 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003752 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003753
3754 video_stream_encoder_->Stop();
3755}
3756
3757TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3758 // Setup simulcast with three streams.
3759 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01003760 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003761 DataRate::bps(kSimulcastTargetBitrateBps),
3762 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003763 // Wait for all three layers before triggering event.
3764 sink_.SetNumExpectedLayers(3);
3765
3766 // First frame is always keyframe.
3767 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3768 WaitForEncodedFrame(1);
3769 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003770 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
3771 VideoFrameType::kVideoFrameKey,
3772 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003773
3774 // Insert delta frame.
3775 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3776 WaitForEncodedFrame(2);
3777 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003778 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
3779 VideoFrameType::kVideoFrameDelta,
3780 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003781
3782 // Request next frame be a key-frame.
3783 // Only first stream is configured to produce key-frame.
3784 video_stream_encoder_->SendKeyFrame();
3785 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3786 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02003787
3788 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
3789 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01003790 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003791 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02003792 VideoFrameType::kVideoFrameKey,
3793 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003794
3795 video_stream_encoder_->Stop();
3796}
3797
3798TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3799 // Configure internal source factory and setup test again.
3800 encoder_factory_.SetHasInternalSource(true);
3801 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003802 video_stream_encoder_->OnBitrateUpdated(
3803 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003804
3805 // Call encoder directly, simulating internal source where encoded frame
3806 // callback in VideoStreamEncoder is called despite no OnFrame().
3807 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3808 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003809 EXPECT_THAT(
3810 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003811 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003812
Niels Möller8f7ce222019-03-21 15:43:58 +01003813 const std::vector<VideoFrameType> kDeltaFrame = {
3814 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01003815 // Need to set timestamp manually since manually for injected frame.
3816 VideoFrame frame = CreateFrame(101, nullptr);
3817 frame.set_timestamp(101);
3818 fake_encoder_.InjectFrame(frame, false);
3819 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003820 EXPECT_THAT(
3821 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003822 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003823
3824 // Request key-frame. The forces a dummy frame down into the encoder.
3825 fake_encoder_.ExpectNullFrame();
3826 video_stream_encoder_->SendKeyFrame();
3827 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003828 EXPECT_THAT(
3829 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003830 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003831
3832 video_stream_encoder_->Stop();
3833}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003834
3835TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3836 // Configure internal source factory and setup test again.
3837 encoder_factory_.SetHasInternalSource(true);
3838 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003839 video_stream_encoder_->OnBitrateUpdated(
3840 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01003841
3842 int64_t timestamp = 1;
3843 EncodedImage image;
3844 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3845 image.capture_time_ms_ = ++timestamp;
3846 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3847 const int64_t kEncodeFinishDelayMs = 10;
3848 image.timing_.encode_start_ms = timestamp;
3849 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3850 fake_encoder_.InjectEncodedImage(image);
3851 // Wait for frame without incrementing clock.
3852 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3853 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3854 // capture timestamp should be kEncodeFinishDelayMs in the past.
3855 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3856 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3857 kEncodeFinishDelayMs);
3858
3859 video_stream_encoder_->Stop();
3860}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02003861
3862TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
3863 // Configure internal source factory and setup test again.
3864 encoder_factory_.SetHasInternalSource(true);
3865 ResetEncoder("H264", 1, 1, 1, false);
3866
3867 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
3868 image._frameType = VideoFrameType::kVideoFrameKey;
3869
3870 CodecSpecificInfo codec_specific_info;
3871 codec_specific_info.codecType = kVideoCodecH264;
3872
3873 RTPFragmentationHeader fragmentation;
3874 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3875 fragmentation.fragmentationOffset[0] = 4;
3876 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
3877
3878 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3879 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3880
3881 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3882 testing::ElementsAreArray(optimal_sps));
3883 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3884 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3885 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3886 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3887
3888 video_stream_encoder_->Stop();
3889}
3890
3891TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
3892 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
3893 0x00, 0x00, 0x03, 0x03, 0xF4,
3894 0x05, 0x03, 0xC7, 0xC0};
3895
3896 // Configure internal source factory and setup test again.
3897 encoder_factory_.SetHasInternalSource(true);
3898 ResetEncoder("H264", 1, 1, 1, false);
3899
3900 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
3901 image._frameType = VideoFrameType::kVideoFrameKey;
3902
3903 CodecSpecificInfo codec_specific_info;
3904 codec_specific_info.codecType = kVideoCodecH264;
3905
3906 RTPFragmentationHeader fragmentation;
3907 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3908 fragmentation.fragmentationOffset[0] = 4;
3909 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
3910
3911 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3912 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3913
3914 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3915 testing::ElementsAreArray(optimal_sps));
3916 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3917 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3918 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3919 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3920
3921 video_stream_encoder_->Stop();
3922}
3923
perkj26091b12016-09-01 01:17:40 -07003924} // namespace webrtc