blob: 4a26772d730ae87f50b035fda39ee3ca93bd7da2 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Per512ecb32016-09-23 15:52:06 +020015#include <utility>
16
Jiawei Ouc2ebe212018-11-08 10:02:56 -080017#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010019#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020020#include "api/video_codecs/create_vp8_temporal_layers.h"
21#include "api/video_codecs/vp8_temporal_layers.h"
Steve Anton10542f22019-01-11 09:11:00 -080022#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020023#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080025#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080027#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010028#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020029#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/sleep.h"
31#include "test/encoder_settings.h"
32#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020033#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "test/frame_generator.h"
35#include "test/gmock.h"
36#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020037#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070039
40namespace webrtc {
41
sprangb1ca0732017-02-01 08:38:12 -080042using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080043using ::testing::_;
44using ::testing::Return;
kthelgason876222f2016-11-29 01:44:11 -080045
perkj803d97f2016-11-01 11:45:46 -070046namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020047const int kMinPixelsPerFrame = 320 * 180;
48const int kMinFramerateFps = 2;
49const int kMinBalancedFramerateFps = 7;
50const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080051const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010052const uint32_t kTargetBitrateBps = 1000000;
53const uint32_t kSimulcastTargetBitrateBps = 3150000;
54const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080055const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070056const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020057const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080058
perkj803d97f2016-11-01 11:45:46 -070059class TestBuffer : public webrtc::I420Buffer {
60 public:
61 TestBuffer(rtc::Event* event, int width, int height)
62 : I420Buffer(width, height), event_(event) {}
63
64 private:
65 friend class rtc::RefCountedObject<TestBuffer>;
66 ~TestBuffer() override {
67 if (event_)
68 event_->Set();
69 }
70 rtc::Event* const event_;
71};
72
Niels Möller7dc26b72017-12-06 10:27:48 +010073class CpuOveruseDetectorProxy : public OveruseFrameDetector {
74 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020075 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
76 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010077 last_target_framerate_fps_(-1) {}
78 virtual ~CpuOveruseDetectorProxy() {}
79
80 void OnTargetFramerateUpdated(int framerate_fps) override {
81 rtc::CritScope cs(&lock_);
82 last_target_framerate_fps_ = framerate_fps;
83 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
84 }
85
86 int GetLastTargetFramerate() {
87 rtc::CritScope cs(&lock_);
88 return last_target_framerate_fps_;
89 }
90
Niels Möller4db138e2018-04-19 09:04:13 +020091 CpuOveruseOptions GetOptions() { return options_; }
92
Niels Möller7dc26b72017-12-06 10:27:48 +010093 private:
94 rtc::CriticalSection lock_;
95 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
96};
97
mflodmancc3d4422017-08-03 08:27:51 -070098class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -070099 public:
Niels Möller213618e2018-07-24 09:29:58 +0200100 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
101 const VideoStreamEncoderSettings& settings)
Yves Gerey665174f2018-06-19 15:03:05 +0200102 : VideoStreamEncoder(1 /* number_of_cores */,
103 stats_proxy,
104 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200105 std::unique_ptr<OveruseFrameDetector>(
106 overuse_detector_proxy_ =
107 new CpuOveruseDetectorProxy(stats_proxy))) {}
perkj803d97f2016-11-01 11:45:46 -0700108
sprangb1ca0732017-02-01 08:38:12 -0800109 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100110 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800111 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800112 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700113 event.Set();
114 });
perkj070ba852017-02-16 15:46:27 -0800115 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700116 }
117
kthelgason2fc52542017-03-03 00:24:41 -0800118 // This is used as a synchronisation mechanism, to make sure that the
119 // encoder queue is not blocked before we start sending it frames.
120 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100121 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200122 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800123 ASSERT_TRUE(event.Wait(5000));
124 }
125
sprangb1ca0732017-02-01 08:38:12 -0800126 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800127
sprangb1ca0732017-02-01 08:38:12 -0800128 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800129
sprangb1ca0732017-02-01 08:38:12 -0800130 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800131
sprangb1ca0732017-02-01 08:38:12 -0800132 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700133
Niels Möller7dc26b72017-12-06 10:27:48 +0100134 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700135};
136
asapersson5f7226f2016-11-25 04:37:00 -0800137class VideoStreamFactory
138 : public VideoEncoderConfig::VideoStreamFactoryInterface {
139 public:
sprangfda496a2017-06-15 04:21:07 -0700140 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
141 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800142 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700143 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800144 }
145
146 private:
147 std::vector<VideoStream> CreateEncoderStreams(
148 int width,
149 int height,
150 const VideoEncoderConfig& encoder_config) override {
151 std::vector<VideoStream> streams =
152 test::CreateVideoStreams(width, height, encoder_config);
153 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100154 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700155 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800156 }
157 return streams;
158 }
sprangfda496a2017-06-15 04:21:07 -0700159
asapersson5f7226f2016-11-25 04:37:00 -0800160 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700161 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800162};
163
sprangb1ca0732017-02-01 08:38:12 -0800164class AdaptingFrameForwarder : public test::FrameForwarder {
165 public:
166 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700167 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800168
169 void set_adaptation_enabled(bool enabled) {
170 rtc::CritScope cs(&crit_);
171 adaptation_enabled_ = enabled;
172 }
173
asaperssonfab67072017-04-04 05:51:49 -0700174 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800175 rtc::CritScope cs(&crit_);
176 return adaptation_enabled_;
177 }
178
asapersson09f05612017-05-15 23:40:18 -0700179 rtc::VideoSinkWants last_wants() const {
180 rtc::CritScope cs(&crit_);
181 return last_wants_;
182 }
183
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200184 absl::optional<int> last_sent_width() const { return last_width_; }
185 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800186
sprangb1ca0732017-02-01 08:38:12 -0800187 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
188 int cropped_width = 0;
189 int cropped_height = 0;
190 int out_width = 0;
191 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700192 if (adaption_enabled()) {
193 if (adapter_.AdaptFrameResolution(
194 video_frame.width(), video_frame.height(),
195 video_frame.timestamp_us() * 1000, &cropped_width,
196 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100197 VideoFrame adapted_frame =
198 VideoFrame::Builder()
199 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
200 nullptr, out_width, out_height))
201 .set_timestamp_rtp(99)
202 .set_timestamp_ms(99)
203 .set_rotation(kVideoRotation_0)
204 .build();
sprangc5d62e22017-04-02 23:53:04 -0700205 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
206 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800207 last_width_.emplace(adapted_frame.width());
208 last_height_.emplace(adapted_frame.height());
209 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200210 last_width_ = absl::nullopt;
211 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700212 }
sprangb1ca0732017-02-01 08:38:12 -0800213 } else {
214 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800215 last_width_.emplace(video_frame.width());
216 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800217 }
218 }
219
220 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
221 const rtc::VideoSinkWants& wants) override {
222 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700223 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700224 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
225 wants.max_pixel_count,
226 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800227 test::FrameForwarder::AddOrUpdateSink(sink, wants);
228 }
sprangb1ca0732017-02-01 08:38:12 -0800229 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700230 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
231 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200232 absl::optional<int> last_width_;
233 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800234};
sprangc5d62e22017-04-02 23:53:04 -0700235
Niels Möller213618e2018-07-24 09:29:58 +0200236// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700237class MockableSendStatisticsProxy : public SendStatisticsProxy {
238 public:
239 MockableSendStatisticsProxy(Clock* clock,
240 const VideoSendStream::Config& config,
241 VideoEncoderConfig::ContentType content_type)
242 : SendStatisticsProxy(clock, config, content_type) {}
243
244 VideoSendStream::Stats GetStats() override {
245 rtc::CritScope cs(&lock_);
246 if (mock_stats_)
247 return *mock_stats_;
248 return SendStatisticsProxy::GetStats();
249 }
250
Niels Möller213618e2018-07-24 09:29:58 +0200251 int GetInputFrameRate() const override {
252 rtc::CritScope cs(&lock_);
253 if (mock_stats_)
254 return mock_stats_->input_frame_rate;
255 return SendStatisticsProxy::GetInputFrameRate();
256 }
sprangc5d62e22017-04-02 23:53:04 -0700257 void SetMockStats(const VideoSendStream::Stats& stats) {
258 rtc::CritScope cs(&lock_);
259 mock_stats_.emplace(stats);
260 }
261
262 void ResetMockStats() {
263 rtc::CritScope cs(&lock_);
264 mock_stats_.reset();
265 }
266
267 private:
268 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200269 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700270};
271
sprang4847ae62017-06-27 07:06:52 -0700272class MockBitrateObserver : public VideoBitrateAllocationObserver {
273 public:
Erik Språng566124a2018-04-23 12:32:22 +0200274 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700275};
276
perkj803d97f2016-11-01 11:45:46 -0700277} // namespace
278
mflodmancc3d4422017-08-03 08:27:51 -0700279class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700280 public:
281 static const int kDefaultTimeoutMs = 30 * 1000;
282
mflodmancc3d4422017-08-03 08:27:51 -0700283 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700284 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700285 codec_width_(320),
286 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200287 max_framerate_(kDefaultFramerate),
perkj26091b12016-09-01 01:17:40 -0700288 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200289 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800290 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700291 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700292 Clock::GetRealTimeClock(),
293 video_send_config_,
294 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700295 sink_(&fake_encoder_) {}
296
297 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700298 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700299 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200300 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800301 video_send_config_.encoder_settings.bitrate_allocator_factory =
302 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200303 video_send_config_.rtp.payload_name = "FAKE";
304 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700305
Per512ecb32016-09-23 15:52:06 +0200306 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200307 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700308 video_encoder_config.video_stream_factory =
309 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100310 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700311
312 // Framerate limit is specified by the VideoStreamFactory.
313 std::vector<VideoStream> streams =
314 video_encoder_config.video_stream_factory->CreateEncoderStreams(
315 codec_width_, codec_height_, video_encoder_config);
316 max_framerate_ = streams[0].max_framerate;
317 fake_clock_.SetTimeMicros(1234);
318
Niels Möllerf1338562018-04-26 09:51:47 +0200319 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800320 }
321
Niels Möllerf1338562018-04-26 09:51:47 +0200322 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700323 if (video_stream_encoder_)
324 video_stream_encoder_->Stop();
325 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
perkj803d97f2016-11-01 11:45:46 -0700326 stats_proxy_.get(), video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700327 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
328 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700329 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700330 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
331 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200332 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700333 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800334 }
335
336 void ResetEncoder(const std::string& payload_name,
337 size_t num_streams,
338 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700339 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700340 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200341 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800342
343 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200344 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800345 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100346 video_encoder_config.max_bitrate_bps =
347 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800348 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700349 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
350 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700351 video_encoder_config.content_type =
352 screenshare ? VideoEncoderConfig::ContentType::kScreen
353 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700354 if (payload_name == "VP9") {
355 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
356 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
357 video_encoder_config.encoder_specific_settings =
358 new rtc::RefCountedObject<
359 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
360 }
Niels Möllerf1338562018-04-26 09:51:47 +0200361 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700362 }
363
sprang57c2fff2017-01-16 06:24:02 -0800364 VideoFrame CreateFrame(int64_t ntp_time_ms,
365 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100366 VideoFrame frame =
367 VideoFrame::Builder()
368 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
369 destruction_event, codec_width_, codec_height_))
370 .set_timestamp_rtp(99)
371 .set_timestamp_ms(99)
372 .set_rotation(kVideoRotation_0)
373 .build();
sprang57c2fff2017-01-16 06:24:02 -0800374 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700375 return frame;
376 }
377
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100378 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
379 rtc::Event* destruction_event,
380 int offset_x) const {
381 VideoFrame frame =
382 VideoFrame::Builder()
383 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
384 destruction_event, codec_width_, codec_height_))
385 .set_timestamp_rtp(99)
386 .set_timestamp_ms(99)
387 .set_rotation(kVideoRotation_0)
388 .set_update_rect({offset_x, 0, 1, 1})
389 .build();
390 frame.set_ntp_time_ms(ntp_time_ms);
391 return frame;
392 }
393
sprang57c2fff2017-01-16 06:24:02 -0800394 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100395 VideoFrame frame =
396 VideoFrame::Builder()
397 .set_video_frame_buffer(
398 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
399 .set_timestamp_rtp(99)
400 .set_timestamp_ms(99)
401 .set_rotation(kVideoRotation_0)
402 .build();
sprang57c2fff2017-01-16 06:24:02 -0800403 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700404 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700405 return frame;
406 }
407
asapersson02465b82017-04-10 01:12:52 -0700408 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700409 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700410 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
411 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700412 }
413
asapersson09f05612017-05-15 23:40:18 -0700414 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
415 const rtc::VideoSinkWants& wants2) {
416 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
417 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
418 }
419
Åsa Persson8c1bf952018-09-13 10:42:19 +0200420 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
421 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
422 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
423 EXPECT_FALSE(wants.target_pixel_count);
424 }
425
asapersson09f05612017-05-15 23:40:18 -0700426 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
427 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200428 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700429 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
430 EXPECT_GT(wants1.max_pixel_count, 0);
431 }
432
433 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
434 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200435 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700436 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
437 }
438
asaperssonf7e294d2017-06-13 23:25:22 -0700439 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
440 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200441 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700442 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
443 }
444
445 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
446 const rtc::VideoSinkWants& wants2) {
447 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
448 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
449 }
450
451 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
452 const rtc::VideoSinkWants& wants2) {
453 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
454 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
455 }
456
457 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
458 const rtc::VideoSinkWants& wants2) {
459 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
460 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
461 EXPECT_GT(wants1.max_pixel_count, 0);
462 }
463
464 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
465 const rtc::VideoSinkWants& wants2) {
466 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
467 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
468 }
469
asapersson09f05612017-05-15 23:40:18 -0700470 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
471 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200472 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700473 EXPECT_LT(wants.max_pixel_count, pixel_count);
474 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700475 }
476
477 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
478 EXPECT_LT(wants.max_framerate_fps, fps);
479 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
480 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700481 }
482
asaperssonf7e294d2017-06-13 23:25:22 -0700483 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
484 int expected_fps) {
485 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
486 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
487 EXPECT_FALSE(wants.target_pixel_count);
488 }
489
Jonathan Yubc771b72017-12-08 17:04:29 -0800490 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
491 int last_frame_pixels) {
492 // Balanced mode should always scale FPS to the desired range before
493 // attempting to scale resolution.
494 int fps_limit = wants.max_framerate_fps;
495 if (last_frame_pixels <= 320 * 240) {
496 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
497 } else if (last_frame_pixels <= 480 * 270) {
498 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
499 } else if (last_frame_pixels <= 640 * 480) {
500 EXPECT_LE(15, fps_limit);
501 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200502 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800503 }
504 }
505
sprang4847ae62017-06-27 07:06:52 -0700506 void WaitForEncodedFrame(int64_t expected_ntp_time) {
507 sink_.WaitForEncodedFrame(expected_ntp_time);
508 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
509 }
510
511 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
512 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
513 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
514 return ok;
515 }
516
517 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
518 sink_.WaitForEncodedFrame(expected_width, expected_height);
519 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
520 }
521
522 void ExpectDroppedFrame() {
523 sink_.ExpectDroppedFrame();
524 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
525 }
526
527 bool WaitForFrame(int64_t timeout_ms) {
528 bool ok = sink_.WaitForFrame(timeout_ms);
529 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
530 return ok;
531 }
532
perkj26091b12016-09-01 01:17:40 -0700533 class TestEncoder : public test::FakeEncoder {
534 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100535 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700536
asaperssonfab67072017-04-04 05:51:49 -0700537 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800538 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700539 return config_;
540 }
541
542 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800543 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700544 block_next_encode_ = true;
545 }
546
Erik Språngaed30702018-11-05 12:57:17 +0100547 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800548 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100549 EncoderInfo info;
Erik Språng715c4762019-02-25 11:18:24 +0100550 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100551 if (quality_scaling_) {
552 info.scaling_settings =
553 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
554 }
555 info.is_hardware_accelerated = is_hardware_accelerated_;
Erik Språngaed30702018-11-05 12:57:17 +0100556 }
557 return info;
kthelgason876222f2016-11-29 01:44:11 -0800558 }
559
perkjfa10b552016-10-02 23:45:26 -0700560 void ContinueEncode() { continue_encode_event_.Set(); }
561
562 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
563 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800564 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700565 EXPECT_EQ(timestamp_, timestamp);
566 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
567 }
568
kthelgason2fc52542017-03-03 00:24:41 -0800569 void SetQualityScaling(bool b) {
570 rtc::CritScope lock(&local_crit_sect_);
571 quality_scaling_ = b;
572 }
kthelgasonad9010c2017-02-14 00:46:51 -0800573
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100574 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
575 rtc::CritScope lock(&local_crit_sect_);
576 is_hardware_accelerated_ = is_hardware_accelerated;
577 }
578
sprangfe627f32017-03-29 08:24:59 -0700579 void ForceInitEncodeFailure(bool force_failure) {
580 rtc::CritScope lock(&local_crit_sect_);
581 force_init_encode_failed_ = force_failure;
582 }
583
Niels Möller6bb5ab92019-01-11 11:11:10 +0100584 void SimulateOvershoot(double rate_factor) {
585 rtc::CritScope lock(&local_crit_sect_);
586 rate_factor_ = rate_factor;
587 }
588
Erik Språngd7329ca2019-02-21 21:19:53 +0100589 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100590 rtc::CritScope lock(&local_crit_sect_);
591 return last_framerate_;
592 }
593
Erik Språngd7329ca2019-02-21 21:19:53 +0100594 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100595 rtc::CritScope lock(&local_crit_sect_);
596 return last_update_rect_;
597 }
598
Erik Språngd7329ca2019-02-21 21:19:53 +0100599 const std::vector<FrameType>& LastFrameTypes() const {
600 rtc::CritScope lock(&local_crit_sect_);
601 return last_frame_types_;
602 }
603
604 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
605 const std::vector<FrameType> frame_type = {keyframe ? kVideoFrameKey
606 : kVideoFrameDelta};
607 {
608 rtc::CritScope lock(&local_crit_sect_);
609 last_frame_types_ = frame_type;
610 }
611 FakeEncoder::Encode(input_image, nullptr, &frame_type);
612 }
613
614 void ExpectNullFrame() {
615 rtc::CritScope lock(&local_crit_sect_);
616 expect_null_frame_ = true;
617 }
618
619 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
620 auto allocation = last_bitrate_allocation_;
621 last_bitrate_allocation_.reset();
622 return allocation;
623 }
624
perkjfa10b552016-10-02 23:45:26 -0700625 private:
perkj26091b12016-09-01 01:17:40 -0700626 int32_t Encode(const VideoFrame& input_image,
627 const CodecSpecificInfo* codec_specific_info,
628 const std::vector<FrameType>* frame_types) override {
629 bool block_encode;
630 {
brandtre78d2662017-01-16 05:57:16 -0800631 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100632 if (expect_null_frame_) {
633 EXPECT_EQ(input_image.timestamp(), 0u);
634 EXPECT_EQ(input_image.width(), 1);
635 last_frame_types_ = *frame_types;
636 expect_null_frame_ = false;
637 } else {
638 EXPECT_GT(input_image.timestamp(), timestamp_);
639 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
640 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
641 }
perkj26091b12016-09-01 01:17:40 -0700642
643 timestamp_ = input_image.timestamp();
644 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700645 last_input_width_ = input_image.width();
646 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700647 block_encode = block_next_encode_;
648 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100649 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100650 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700651 }
652 int32_t result =
653 FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
654 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700655 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700656 return result;
657 }
658
sprangfe627f32017-03-29 08:24:59 -0700659 int32_t InitEncode(const VideoCodec* config,
660 int32_t number_of_cores,
661 size_t max_payload_size) override {
662 int res =
663 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
664 rtc::CritScope lock(&local_crit_sect_);
Erik Språng715c4762019-02-25 11:18:24 +0100665 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100666 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700667 // Simulate setting up temporal layers, in order to validate the life
668 // cycle of these objects.
669 int num_streams = std::max<int>(1, config->numberOfSimulcastStreams);
sprangfe627f32017-03-29 08:24:59 -0700670 for (int i = 0; i < num_streams; ++i) {
671 allocated_temporal_layers_.emplace_back(
Erik Språng4529fbc2018-10-12 10:30:31 +0200672 CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern,
673 config->VP8().numberOfTemporalLayers));
sprangfe627f32017-03-29 08:24:59 -0700674 }
675 }
Erik Språng715c4762019-02-25 11:18:24 +0100676 if (force_init_encode_failed_) {
677 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700678 return -1;
Erik Språng715c4762019-02-25 11:18:24 +0100679 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100680
Erik Språng715c4762019-02-25 11:18:24 +0100681 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700682 return res;
683 }
684
Erik Språng715c4762019-02-25 11:18:24 +0100685 int32_t Release() override {
686 rtc::CritScope lock(&local_crit_sect_);
687 EXPECT_NE(initialized_, EncoderState::kUninitialized);
688 initialized_ = EncoderState::kUninitialized;
689 return FakeEncoder::Release();
690 }
691
Niels Möller6bb5ab92019-01-11 11:11:10 +0100692 int32_t SetRateAllocation(const VideoBitrateAllocation& rate_allocation,
693 uint32_t framerate) {
694 rtc::CritScope lock(&local_crit_sect_);
695 VideoBitrateAllocation adjusted_rate_allocation;
696 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
697 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
698 if (rate_allocation.HasBitrate(si, ti)) {
699 adjusted_rate_allocation.SetBitrate(
700 si, ti,
701 static_cast<uint32_t>(rate_allocation.GetBitrate(si, ti) *
702 rate_factor_));
703 }
704 }
705 }
706 last_framerate_ = framerate;
Erik Språngd7329ca2019-02-21 21:19:53 +0100707 last_bitrate_allocation_ = rate_allocation;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100708 return FakeEncoder::SetRateAllocation(adjusted_rate_allocation,
709 framerate);
710 }
711
brandtre78d2662017-01-16 05:57:16 -0800712 rtc::CriticalSection local_crit_sect_;
Erik Språng715c4762019-02-25 11:18:24 +0100713 enum class EncoderState {
714 kUninitialized,
715 kInitializationFailed,
716 kInitialized
717 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
718 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700719 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700720 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700721 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
722 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
723 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
724 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
725 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100726 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Erik Språng4529fbc2018-10-12 10:30:31 +0200727 std::vector<std::unique_ptr<Vp8TemporalLayers>> allocated_temporal_layers_
danilchapa37de392017-09-09 04:17:22 -0700728 RTC_GUARDED_BY(local_crit_sect_);
729 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100730 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
731 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100732 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100733 VideoFrame::UpdateRect last_update_rect_
734 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Erik Språngd7329ca2019-02-21 21:19:53 +0100735 std::vector<FrameType> last_frame_types_;
736 bool expect_null_frame_ = false;
perkj26091b12016-09-01 01:17:40 -0700737 };
738
mflodmancc3d4422017-08-03 08:27:51 -0700739 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700740 public:
741 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100742 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700743
perkj26091b12016-09-01 01:17:40 -0700744 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700745 EXPECT_TRUE(
746 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
747 }
748
749 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
750 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700751 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700752 if (!encoded_frame_event_.Wait(timeout_ms))
753 return false;
perkj26091b12016-09-01 01:17:40 -0700754 {
755 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800756 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700757 }
758 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700759 return true;
perkj26091b12016-09-01 01:17:40 -0700760 }
761
sprangb1ca0732017-02-01 08:38:12 -0800762 void WaitForEncodedFrame(uint32_t expected_width,
763 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700764 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100765 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700766 }
767
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100768 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700769 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800770 uint32_t width = 0;
771 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800772 {
773 rtc::CritScope lock(&crit_);
774 width = last_width_;
775 height = last_height_;
776 }
777 EXPECT_EQ(expected_height, height);
778 EXPECT_EQ(expected_width, width);
779 }
780
kthelgason2fc52542017-03-03 00:24:41 -0800781 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800782
sprangc5d62e22017-04-02 23:53:04 -0700783 bool WaitForFrame(int64_t timeout_ms) {
784 return encoded_frame_event_.Wait(timeout_ms);
785 }
786
perkj26091b12016-09-01 01:17:40 -0700787 void SetExpectNoFrames() {
788 rtc::CritScope lock(&crit_);
789 expect_frames_ = false;
790 }
791
asaperssonfab67072017-04-04 05:51:49 -0700792 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200793 rtc::CritScope lock(&crit_);
794 return number_of_reconfigurations_;
795 }
796
asaperssonfab67072017-04-04 05:51:49 -0700797 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200798 rtc::CritScope lock(&crit_);
799 return min_transmit_bitrate_bps_;
800 }
801
Erik Språngd7329ca2019-02-21 21:19:53 +0100802 void SetNumExpectedLayers(size_t num_layers) {
803 rtc::CritScope lock(&crit_);
804 num_expected_layers_ = num_layers;
805 }
806
perkj26091b12016-09-01 01:17:40 -0700807 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700808 Result OnEncodedImage(
809 const EncodedImage& encoded_image,
810 const CodecSpecificInfo* codec_specific_info,
811 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200812 rtc::CritScope lock(&crit_);
813 EXPECT_TRUE(expect_frames_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100814 uint32_t timestamp = encoded_image.Timestamp();
815 if (last_timestamp_ != timestamp) {
816 num_received_layers_ = 1;
817 } else {
818 ++num_received_layers_;
819 }
820 last_timestamp_ = timestamp;
sprangb1ca0732017-02-01 08:38:12 -0800821 last_width_ = encoded_image._encodedWidth;
822 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +0100823 if (num_received_layers_ == num_expected_layers_) {
824 encoded_frame_event_.Set();
825 }
sprangb1ca0732017-02-01 08:38:12 -0800826 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200827 }
828
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100829 void OnEncoderConfigurationChanged(
830 std::vector<VideoStream> streams,
831 VideoEncoderConfig::ContentType content_type,
832 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200833 rtc::CriticalSection crit_;
834 ++number_of_reconfigurations_;
835 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
836 }
837
perkj26091b12016-09-01 01:17:40 -0700838 rtc::CriticalSection crit_;
839 TestEncoder* test_encoder_;
840 rtc::Event encoded_frame_event_;
sprangb1ca0732017-02-01 08:38:12 -0800841 uint32_t last_timestamp_ = 0;
842 uint32_t last_height_ = 0;
843 uint32_t last_width_ = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100844 size_t num_expected_layers_ = 1;
845 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700846 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200847 int number_of_reconfigurations_ = 0;
848 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700849 };
850
851 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100852 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200853 int codec_width_;
854 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700855 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -0700856 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200857 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800858 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700859 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700860 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800861 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700862 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700863 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700864};
865
mflodmancc3d4422017-08-03 08:27:51 -0700866TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
867 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100868 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700869 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700870 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700871 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700872 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700873}
874
mflodmancc3d4422017-08-03 08:27:51 -0700875TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700876 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100877 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200878 // The encoder will cache up to one frame for a short duration. Adding two
879 // frames means that the first frame will be dropped and the second frame will
880 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700881 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200882 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700883 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700884
mflodmancc3d4422017-08-03 08:27:51 -0700885 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700886
Sebastian Janssona3177052018-04-10 13:05:49 +0200887 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700888 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200889 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
890
891 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700892 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700893}
894
mflodmancc3d4422017-08-03 08:27:51 -0700895TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
896 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700897 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700898 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700899
mflodmancc3d4422017-08-03 08:27:51 -0700900 video_stream_encoder_->OnBitrateUpdated(0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +0200901 // The encoder will cache up to one frame for a short duration. Adding two
902 // frames means that the first frame will be dropped and the second frame will
903 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -0700904 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +0200905 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700906
mflodmancc3d4422017-08-03 08:27:51 -0700907 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -0700908 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +0200909 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
910 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -0700911 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700912}
913
mflodmancc3d4422017-08-03 08:27:51 -0700914TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
915 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700916 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700917 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700918
919 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -0700920 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -0700921
perkja49cbd32016-09-16 07:53:41 -0700922 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700923 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -0700924 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700925}
926
mflodmancc3d4422017-08-03 08:27:51 -0700927TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
928 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700929
perkja49cbd32016-09-16 07:53:41 -0700930 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700931 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700932
mflodmancc3d4422017-08-03 08:27:51 -0700933 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700934 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +0100935 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700936 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
937 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700938}
939
mflodmancc3d4422017-08-03 08:27:51 -0700940TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
941 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700942
943 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -0700944 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700945 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700946 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
947 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -0700948 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
949 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700950 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -0700951 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -0700952
mflodmancc3d4422017-08-03 08:27:51 -0700953 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700954}
955
mflodmancc3d4422017-08-03 08:27:51 -0700956TEST_F(VideoStreamEncoderTest,
957 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
958 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Per21d45d22016-10-30 21:37:57 +0100959 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200960
961 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200962 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700963 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +0100964 // The encoder will have been configured once when the first frame is
965 // received.
966 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200967
968 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200969 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +0200970 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -0700971 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200972 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +0200973
974 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200975 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700976 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +0100977 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -0700978 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -0700979
mflodmancc3d4422017-08-03 08:27:51 -0700980 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -0700981}
982
mflodmancc3d4422017-08-03 08:27:51 -0700983TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
984 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkjfa10b552016-10-02 23:45:26 -0700985
986 // Capture a frame and wait for it to synchronize with the encoder thread.
987 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700988 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +0100989 // The encoder will have been configured once.
990 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -0700991 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
992 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
993
994 codec_width_ *= 2;
995 codec_height_ *= 2;
996 // Capture a frame with a higher resolution and wait for it to synchronize
997 // with the encoder thread.
998 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700999 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001000 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1001 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001002 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001003
mflodmancc3d4422017-08-03 08:27:51 -07001004 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001005}
1006
mflodmancc3d4422017-08-03 08:27:51 -07001007TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001008 EXPECT_TRUE(video_source_.has_sinks());
1009 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001010 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001011 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001012 EXPECT_FALSE(video_source_.has_sinks());
1013 EXPECT_TRUE(new_video_source.has_sinks());
1014
mflodmancc3d4422017-08-03 08:27:51 -07001015 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001016}
1017
mflodmancc3d4422017-08-03 08:27:51 -07001018TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001019 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001020 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001021 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001022 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001023}
1024
Jonathan Yubc771b72017-12-08 17:04:29 -08001025TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1026 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001027 const int kWidth = 1280;
1028 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001029
1030 // We rely on the automatic resolution adaptation, but we handle framerate
1031 // adaptation manually by mocking the stats proxy.
1032 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001033
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001034 // Enable BALANCED preference, no initial limitation.
Jonathan Yubc771b72017-12-08 17:04:29 -08001035 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001036 video_stream_encoder_->SetSource(&video_source_,
1037 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001038 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001039 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001040 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001041 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1042
Jonathan Yubc771b72017-12-08 17:04:29 -08001043 // Adapt down as far as possible.
1044 rtc::VideoSinkWants last_wants;
1045 int64_t t = 1;
1046 int loop_count = 0;
1047 do {
1048 ++loop_count;
1049 last_wants = video_source_.sink_wants();
1050
1051 // Simulate the framerate we've been asked to adapt to.
1052 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1053 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1054 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1055 mock_stats.input_frame_rate = fps;
1056 stats_proxy_->SetMockStats(mock_stats);
1057
1058 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1059 sink_.WaitForEncodedFrame(t);
1060 t += frame_interval_ms;
1061
mflodmancc3d4422017-08-03 08:27:51 -07001062 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001063 VerifyBalancedModeFpsRange(
1064 video_source_.sink_wants(),
1065 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1066 } while (video_source_.sink_wants().max_pixel_count <
1067 last_wants.max_pixel_count ||
1068 video_source_.sink_wants().max_framerate_fps <
1069 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001070
Jonathan Yubc771b72017-12-08 17:04:29 -08001071 // Verify that we've adapted all the way down.
1072 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001073 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001074 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1075 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001076 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001077 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1078 *video_source_.last_sent_height());
1079 EXPECT_EQ(kMinBalancedFramerateFps,
1080 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001081
Jonathan Yubc771b72017-12-08 17:04:29 -08001082 // Adapt back up the same number of times we adapted down.
1083 for (int i = 0; i < loop_count - 1; ++i) {
1084 last_wants = video_source_.sink_wants();
1085
1086 // Simulate the framerate we've been asked to adapt to.
1087 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1088 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1089 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1090 mock_stats.input_frame_rate = fps;
1091 stats_proxy_->SetMockStats(mock_stats);
1092
1093 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1094 sink_.WaitForEncodedFrame(t);
1095 t += frame_interval_ms;
1096
mflodmancc3d4422017-08-03 08:27:51 -07001097 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001098 VerifyBalancedModeFpsRange(
1099 video_source_.sink_wants(),
1100 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1101 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1102 last_wants.max_pixel_count ||
1103 video_source_.sink_wants().max_framerate_fps >
1104 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001105 }
1106
Åsa Persson8c1bf952018-09-13 10:42:19 +02001107 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001108 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001109 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001110 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1111 EXPECT_EQ((loop_count - 1) * 2,
1112 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001113
mflodmancc3d4422017-08-03 08:27:51 -07001114 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001115}
mflodmancc3d4422017-08-03 08:27:51 -07001116TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
1117 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001118 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001119
sprangc5d62e22017-04-02 23:53:04 -07001120 const int kFrameWidth = 1280;
1121 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001122
Åsa Persson8c1bf952018-09-13 10:42:19 +02001123 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001124
kthelgason5e13d412016-12-01 03:59:51 -08001125 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001126 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001127 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001128 frame_timestamp += kFrameIntervalMs;
1129
perkj803d97f2016-11-01 11:45:46 -07001130 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001131 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001132 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001133 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001134 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001135 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001136
asapersson0944a802017-04-07 00:57:58 -07001137 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001138 // wanted resolution.
1139 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1140 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1141 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001142 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001143
1144 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001145 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001146 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001147 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001148
sprangc5d62e22017-04-02 23:53:04 -07001149 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001150 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001151
sprangc5d62e22017-04-02 23:53:04 -07001152 // Force an input frame rate to be available, or the adaptation call won't
1153 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001154 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001155 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001156 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001157 stats_proxy_->SetMockStats(stats);
1158
mflodmancc3d4422017-08-03 08:27:51 -07001159 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001160 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001161 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001162 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001163 frame_timestamp += kFrameIntervalMs;
1164
1165 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001166 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001167 EXPECT_EQ(std::numeric_limits<int>::max(),
1168 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001169 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001170
asapersson02465b82017-04-10 01:12:52 -07001171 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001172 video_stream_encoder_->SetSource(&new_video_source,
1173 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001174 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001175
mflodmancc3d4422017-08-03 08:27:51 -07001176 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001177 new_video_source.IncomingCapturedFrame(
1178 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001179 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001180 frame_timestamp += kFrameIntervalMs;
1181
1182 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001183 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001184
1185 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001186 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001187 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001188 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1189 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001190 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001191 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001192
1193 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001194 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001195 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001196 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1197 EXPECT_EQ(std::numeric_limits<int>::max(),
1198 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001199 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001200
mflodmancc3d4422017-08-03 08:27:51 -07001201 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001202}
1203
mflodmancc3d4422017-08-03 08:27:51 -07001204TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
1205 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001206
asaperssonfab67072017-04-04 05:51:49 -07001207 const int kWidth = 1280;
1208 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001209 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001210 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001211 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1212 EXPECT_FALSE(stats.bw_limited_resolution);
1213 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1214
1215 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001216 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001217 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001218 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001219
1220 stats = stats_proxy_->GetStats();
1221 EXPECT_TRUE(stats.bw_limited_resolution);
1222 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1223
1224 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001225 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001226 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001227 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001228
1229 stats = stats_proxy_->GetStats();
1230 EXPECT_FALSE(stats.bw_limited_resolution);
1231 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1232 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1233
mflodmancc3d4422017-08-03 08:27:51 -07001234 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001235}
1236
mflodmancc3d4422017-08-03 08:27:51 -07001237TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
1238 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001239
1240 const int kWidth = 1280;
1241 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001242 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001243 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001244 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1245 EXPECT_FALSE(stats.cpu_limited_resolution);
1246 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1247
1248 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001249 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001250 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001251 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001252
1253 stats = stats_proxy_->GetStats();
1254 EXPECT_TRUE(stats.cpu_limited_resolution);
1255 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1256
1257 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001258 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001259 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001260 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001261
1262 stats = stats_proxy_->GetStats();
1263 EXPECT_FALSE(stats.cpu_limited_resolution);
1264 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001265 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001266
mflodmancc3d4422017-08-03 08:27:51 -07001267 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001268}
1269
mflodmancc3d4422017-08-03 08:27:51 -07001270TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
1271 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001272
asaperssonfab67072017-04-04 05:51:49 -07001273 const int kWidth = 1280;
1274 const int kHeight = 720;
1275 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001276 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001277 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001278 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001279 EXPECT_FALSE(stats.cpu_limited_resolution);
1280 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1281
asaperssonfab67072017-04-04 05:51:49 -07001282 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001283 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001284 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001285 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001286 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001287 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001288 EXPECT_TRUE(stats.cpu_limited_resolution);
1289 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1290
1291 // Set new source with adaptation still enabled.
1292 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001293 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001294 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001295
asaperssonfab67072017-04-04 05:51:49 -07001296 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001297 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001298 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001299 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001300 EXPECT_TRUE(stats.cpu_limited_resolution);
1301 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1302
1303 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001304 video_stream_encoder_->SetSource(&new_video_source,
1305 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001306
asaperssonfab67072017-04-04 05:51:49 -07001307 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001308 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001309 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001310 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001311 EXPECT_FALSE(stats.cpu_limited_resolution);
1312 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1313
1314 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001315 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001316 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001317
asaperssonfab67072017-04-04 05:51:49 -07001318 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001319 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001320 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001321 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001322 EXPECT_TRUE(stats.cpu_limited_resolution);
1323 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1324
asaperssonfab67072017-04-04 05:51:49 -07001325 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001326 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001327 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001328 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001329 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001330 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001331 EXPECT_FALSE(stats.cpu_limited_resolution);
1332 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001333 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001334
mflodmancc3d4422017-08-03 08:27:51 -07001335 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001336}
1337
mflodmancc3d4422017-08-03 08:27:51 -07001338TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
1339 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001340
asaperssonfab67072017-04-04 05:51:49 -07001341 const int kWidth = 1280;
1342 const int kHeight = 720;
1343 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001344 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001345 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001346 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001347 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001348 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001349
1350 // Set new source with adaptation still enabled.
1351 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001352 video_stream_encoder_->SetSource(&new_video_source,
1353 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001354
asaperssonfab67072017-04-04 05:51:49 -07001355 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001356 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001357 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001358 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001359 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001360 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001361
asaperssonfab67072017-04-04 05:51:49 -07001362 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001363 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001364 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001365 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001366 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001367 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001368 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001369 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001370
asaperssonfab67072017-04-04 05:51:49 -07001371 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001372 video_stream_encoder_->SetSource(&new_video_source,
1373 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001374
asaperssonfab67072017-04-04 05:51:49 -07001375 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001376 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001377 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001378 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001379 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001380 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001381
asapersson02465b82017-04-10 01:12:52 -07001382 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001383 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001384 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001385
asaperssonfab67072017-04-04 05:51:49 -07001386 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001387 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001388 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001389 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001390 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001391 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1392 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001393
mflodmancc3d4422017-08-03 08:27:51 -07001394 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001395}
1396
mflodmancc3d4422017-08-03 08:27:51 -07001397TEST_F(VideoStreamEncoderTest,
1398 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
1399 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001400
1401 const int kWidth = 1280;
1402 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001403 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001404 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001405 video_source_.IncomingCapturedFrame(
1406 CreateFrame(timestamp_ms, kWidth, kHeight));
1407 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001408 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1409 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1410 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1411
1412 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001413 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001414 timestamp_ms += kFrameIntervalMs;
1415 video_source_.IncomingCapturedFrame(
1416 CreateFrame(timestamp_ms, kWidth, kHeight));
1417 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001418 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1420 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1421
1422 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001423 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001424 timestamp_ms += kFrameIntervalMs;
1425 video_source_.IncomingCapturedFrame(
1426 CreateFrame(timestamp_ms, kWidth, kHeight));
1427 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001428 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1429 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1430 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1431
Niels Möller4db138e2018-04-19 09:04:13 +02001432 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001433 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001434
1435 VideoEncoderConfig video_encoder_config;
1436 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1437 // Make format different, to force recreation of encoder.
1438 video_encoder_config.video_format.parameters["foo"] = "foo";
1439 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001440 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001441 timestamp_ms += kFrameIntervalMs;
1442 video_source_.IncomingCapturedFrame(
1443 CreateFrame(timestamp_ms, kWidth, kHeight));
1444 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001445 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1447 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1448
mflodmancc3d4422017-08-03 08:27:51 -07001449 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001450}
1451
mflodmancc3d4422017-08-03 08:27:51 -07001452TEST_F(VideoStreamEncoderTest,
1453 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
1454 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001455
asapersson0944a802017-04-07 00:57:58 -07001456 const int kWidth = 1280;
1457 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001458 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001459
asaperssonfab67072017-04-04 05:51:49 -07001460 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001462 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001463 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001464 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001465 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1466
asapersson02465b82017-04-10 01:12:52 -07001467 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001468 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001469 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001470 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001471 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001472 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001473 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001474 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1475
1476 // Set new source with adaptation still enabled.
1477 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001478 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001479 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001480
1481 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001482 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001483 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001484 stats = stats_proxy_->GetStats();
1485 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001486 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001487 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1488
sprangc5d62e22017-04-02 23:53:04 -07001489 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001490 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001491 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001492 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001493 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001494 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001495 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001496 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001497 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001498 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001499 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1500
sprangc5d62e22017-04-02 23:53:04 -07001501 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001502 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001503 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1504 mock_stats.input_frame_rate = 30;
1505 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001506 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001507 stats_proxy_->ResetMockStats();
1508
1509 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001510 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001511 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001512
1513 // Framerate now adapted.
1514 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001515 EXPECT_FALSE(stats.cpu_limited_resolution);
1516 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001517 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1518
1519 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001520 video_stream_encoder_->SetSource(&new_video_source,
1521 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001522 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001523 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001524 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001525
1526 stats = stats_proxy_->GetStats();
1527 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001528 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001529 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1530
1531 // Try to trigger overuse. Should not succeed.
1532 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001533 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001534 stats_proxy_->ResetMockStats();
1535
1536 stats = stats_proxy_->GetStats();
1537 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001538 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001539 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1540
1541 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001542 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001543 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001544 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001545 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001546 stats = stats_proxy_->GetStats();
1547 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001548 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001549 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001550
1551 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001552 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001553 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001554 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001555 stats = stats_proxy_->GetStats();
1556 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001557 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001558 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1559
1560 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001561 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001562 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001563 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001564 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001565 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001566 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001567 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001568 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001569 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001570 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1571
1572 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001573 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001574 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001575 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001576 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001577 stats = stats_proxy_->GetStats();
1578 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001579 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001580 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001581 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001582
mflodmancc3d4422017-08-03 08:27:51 -07001583 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001584}
1585
mflodmancc3d4422017-08-03 08:27:51 -07001586TEST_F(VideoStreamEncoderTest,
1587 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001588 const int kWidth = 1280;
1589 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001590 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001591
asaperssonfab67072017-04-04 05:51:49 -07001592 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001593 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001594
asaperssonfab67072017-04-04 05:51:49 -07001595 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001596 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001597
asaperssonfab67072017-04-04 05:51:49 -07001598 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001599 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001600
asaperssonfab67072017-04-04 05:51:49 -07001601 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001602 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001603
kthelgason876222f2016-11-29 01:44:11 -08001604 // Expect a scale down.
1605 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001606 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001607
asapersson02465b82017-04-10 01:12:52 -07001608 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001609 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001610 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001611 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001612
asaperssonfab67072017-04-04 05:51:49 -07001613 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001614 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001615 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001616 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001617
asaperssonfab67072017-04-04 05:51:49 -07001618 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001619 EXPECT_EQ(std::numeric_limits<int>::max(),
1620 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001621
asaperssonfab67072017-04-04 05:51:49 -07001622 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001623 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001624 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001625 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001626
asapersson02465b82017-04-10 01:12:52 -07001627 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001628 EXPECT_EQ(std::numeric_limits<int>::max(),
1629 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001630
mflodmancc3d4422017-08-03 08:27:51 -07001631 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001632}
1633
mflodmancc3d4422017-08-03 08:27:51 -07001634TEST_F(VideoStreamEncoderTest,
1635 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001636 const int kWidth = 1280;
1637 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001638 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001639
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001640 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001641 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001642 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001643 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001644
1645 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001646 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001647 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1649 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1650
1651 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001652 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001653 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001654 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1655 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1656 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1657
1658 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001659 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001660 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1661 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1662 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1663
mflodmancc3d4422017-08-03 08:27:51 -07001664 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001665}
1666
mflodmancc3d4422017-08-03 08:27:51 -07001667TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001668 const int kWidth = 1280;
1669 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001670 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001671
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001672 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001673 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001674 video_stream_encoder_->SetSource(&source,
1675 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001676 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1677 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001678 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001679
1680 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001681 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001682 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1683 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1684 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1685 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1686
1687 // Trigger adapt down for same input resolution, expect no change.
1688 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1689 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001690 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001691 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1692 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1694
1695 // Trigger adapt down for larger input resolution, expect no change.
1696 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1697 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001698 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001699 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1700 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1701 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1702
mflodmancc3d4422017-08-03 08:27:51 -07001703 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001704}
1705
mflodmancc3d4422017-08-03 08:27:51 -07001706TEST_F(VideoStreamEncoderTest,
1707 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001708 const int kWidth = 1280;
1709 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001710 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001711
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001712 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001713 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001714 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001715 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001716
1717 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001718 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001719 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1721 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1722
1723 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001724 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001725 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001726 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1727 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1728
mflodmancc3d4422017-08-03 08:27:51 -07001729 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001730}
1731
mflodmancc3d4422017-08-03 08:27:51 -07001732TEST_F(VideoStreamEncoderTest,
1733 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001734 const int kWidth = 1280;
1735 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001736 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001737
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001738 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001739 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001740 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001741 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001742
1743 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001744 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001745 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001746 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1748
1749 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001750 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001751 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001753 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1754
mflodmancc3d4422017-08-03 08:27:51 -07001755 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001756}
1757
mflodmancc3d4422017-08-03 08:27:51 -07001758TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001759 const int kWidth = 1280;
1760 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001761 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001762
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001763 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001764 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001765 video_stream_encoder_->SetSource(&source,
1766 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001767
1768 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1769 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001770 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001771 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1772 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1773 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1774
1775 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001776 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001777 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1779 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1780 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1781
mflodmancc3d4422017-08-03 08:27:51 -07001782 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001783}
1784
mflodmancc3d4422017-08-03 08:27:51 -07001785TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001786 const int kWidth = 1280;
1787 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001789
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001790 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001791 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001792 video_stream_encoder_->SetSource(&source,
1793 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001794
1795 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1796 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001797 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1801
1802 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001803 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001804 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1806 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1807 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1808
mflodmancc3d4422017-08-03 08:27:51 -07001809 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001810}
1811
mflodmancc3d4422017-08-03 08:27:51 -07001812TEST_F(VideoStreamEncoderTest,
1813 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001814 const int kWidth = 1280;
1815 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001816 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001817
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001818 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001819 AdaptingFrameForwarder source;
1820 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001821 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001822 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001823
1824 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001825 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001826 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1828 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1829
1830 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001831 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001832 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001833 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001834 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001835 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1836 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1837
1838 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001839 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001840 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001841 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1842 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1843 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1844
mflodmancc3d4422017-08-03 08:27:51 -07001845 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001846}
1847
mflodmancc3d4422017-08-03 08:27:51 -07001848TEST_F(VideoStreamEncoderTest,
1849 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001850 const int kWidth = 1280;
1851 const int kHeight = 720;
1852 const int kInputFps = 30;
mflodmancc3d4422017-08-03 08:27:51 -07001853 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001854
1855 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1856 stats.input_frame_rate = kInputFps;
1857 stats_proxy_->SetMockStats(stats);
1858
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001859 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001860 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1861 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001862 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001863
1864 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001865 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001866 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1867 sink_.WaitForEncodedFrame(2);
1868 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1869
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001870 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001871 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001872 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001873 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001874 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001875
1876 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07001877 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001878 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1879 sink_.WaitForEncodedFrame(3);
1880 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1881
1882 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001883 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001884 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001885
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001887}
1888
mflodmancc3d4422017-08-03 08:27:51 -07001889TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07001890 const int kWidth = 1280;
1891 const int kHeight = 720;
1892 const size_t kNumFrames = 10;
1893
mflodmancc3d4422017-08-03 08:27:51 -07001894 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001895
asaperssond0de2952017-04-21 01:47:31 -07001896 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07001897 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07001898 video_source_.set_adaptation_enabled(true);
1899
1900 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1901 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1902
1903 int downscales = 0;
1904 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02001905 video_source_.IncomingCapturedFrame(
1906 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
1907 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07001908
asaperssonfab67072017-04-04 05:51:49 -07001909 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07001910 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07001911 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07001912 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07001913
1914 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
1915 ++downscales;
1916
1917 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1918 EXPECT_EQ(downscales,
1919 stats_proxy_->GetStats().number_of_quality_adapt_changes);
1920 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001921 }
mflodmancc3d4422017-08-03 08:27:51 -07001922 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07001923}
1924
mflodmancc3d4422017-08-03 08:27:51 -07001925TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07001926 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
1927 const int kWidth = 1280;
1928 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001929 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07001930
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001931 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07001932 AdaptingFrameForwarder source;
1933 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001934 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001935 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07001936
Åsa Persson8c1bf952018-09-13 10:42:19 +02001937 int64_t timestamp_ms = kFrameIntervalMs;
1938 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001939 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001940 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001941 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1942 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1943
1944 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001945 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001946 timestamp_ms += kFrameIntervalMs;
1947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1948 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001949 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001950 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1951 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1952
1953 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001954 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001955 timestamp_ms += kFrameIntervalMs;
1956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001957 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001958 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1960 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1961
1962 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001963 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001964 timestamp_ms += kFrameIntervalMs;
1965 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1966 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001967 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001968 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1969 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1970
1971 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001972 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001973 timestamp_ms += kFrameIntervalMs;
1974 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07001975 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001976 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001977 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1978 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1979
mflodmancc3d4422017-08-03 08:27:51 -07001980 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07001981}
1982
mflodmancc3d4422017-08-03 08:27:51 -07001983TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07001984 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
1985 const int kWidth = 1280;
1986 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001987 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001988
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001989 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001990 AdaptingFrameForwarder source;
1991 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001992 video_stream_encoder_->SetSource(&source,
1993 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001994
Åsa Persson8c1bf952018-09-13 10:42:19 +02001995 int64_t timestamp_ms = kFrameIntervalMs;
1996 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07001997 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001998 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001999 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2000 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2001
2002 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002003 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002004 timestamp_ms += kFrameIntervalMs;
2005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2006 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002007 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2008 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2009 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2010
2011 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002012 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002013 timestamp_ms += kFrameIntervalMs;
2014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002015 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002016 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2018 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2019
2020 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002021 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002022 timestamp_ms += kFrameIntervalMs;
2023 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2024 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002025 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2026 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2027 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2028
2029 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002030 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002031 timestamp_ms += kFrameIntervalMs;
2032 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002033 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002034 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2036 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2037
mflodmancc3d4422017-08-03 08:27:51 -07002038 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002039}
2040
mflodmancc3d4422017-08-03 08:27:51 -07002041TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002042 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2043 const int kWidth = 1280;
2044 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002045 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002046
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002047 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002048 AdaptingFrameForwarder source;
2049 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002050 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002051 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002052
Åsa Persson8c1bf952018-09-13 10:42:19 +02002053 int64_t timestamp_ms = kFrameIntervalMs;
2054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002055 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002056 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002057 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2059 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2060 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2061
2062 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002063 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002064 timestamp_ms += kFrameIntervalMs;
2065 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2066 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002067 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002068 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2069 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2070 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2071 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2072
2073 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002074 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002075 timestamp_ms += kFrameIntervalMs;
2076 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2077 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002078 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002079 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2080 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2081 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2082 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2083
Jonathan Yubc771b72017-12-08 17:04:29 -08002084 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002085 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002086 timestamp_ms += kFrameIntervalMs;
2087 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2088 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002089 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002090 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2091 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002092 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002093 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2094
Jonathan Yubc771b72017-12-08 17:04:29 -08002095 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002096 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002097 timestamp_ms += kFrameIntervalMs;
2098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2099 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002100 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002101 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002102 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2103 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2104 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2105 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2106
Jonathan Yubc771b72017-12-08 17:04:29 -08002107 // Trigger quality adapt down, expect no change (min resolution reached).
2108 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002109 timestamp_ms += kFrameIntervalMs;
2110 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2111 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002112 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2113 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2114 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2115 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2116 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2117
2118 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002119 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002120 timestamp_ms += kFrameIntervalMs;
2121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2122 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002123 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002124 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2125 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2126 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2127 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2128
2129 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2130 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002131 timestamp_ms += kFrameIntervalMs;
2132 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2133 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002134 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2135 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2136 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2137 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2138 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2139
2140 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2141 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002142 timestamp_ms += kFrameIntervalMs;
2143 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2144 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002145 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002146 last_wants = source.sink_wants();
2147 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2148 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002149 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002150 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2151
2152 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002153 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002154 timestamp_ms += kFrameIntervalMs;
2155 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2156 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002157 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002158 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2159 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002160 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002161 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2162
2163 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002164 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002165 timestamp_ms += kFrameIntervalMs;
2166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002167 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002168 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002169 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002170 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2171 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002172 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002173 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002174
mflodmancc3d4422017-08-03 08:27:51 -07002175 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002176}
2177
mflodmancc3d4422017-08-03 08:27:51 -07002178TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002179 const int kWidth = 640;
2180 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002181
mflodmancc3d4422017-08-03 08:27:51 -07002182 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002183
perkj803d97f2016-11-01 11:45:46 -07002184 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002185 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002186 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002187 }
2188
mflodmancc3d4422017-08-03 08:27:51 -07002189 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002190 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002191 video_source_.IncomingCapturedFrame(CreateFrame(
2192 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002193 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002194 }
2195
mflodmancc3d4422017-08-03 08:27:51 -07002196 video_stream_encoder_->Stop();
2197 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002198 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002199
perkj803d97f2016-11-01 11:45:46 -07002200 EXPECT_EQ(1,
2201 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2202 EXPECT_EQ(
2203 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2204}
2205
mflodmancc3d4422017-08-03 08:27:51 -07002206TEST_F(VideoStreamEncoderTest,
2207 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
2208 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002209 const int kWidth = 640;
2210 const int kHeight = 360;
2211
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002212 video_stream_encoder_->SetSource(&video_source_,
2213 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002214
2215 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2216 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002217 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002218 }
2219
mflodmancc3d4422017-08-03 08:27:51 -07002220 video_stream_encoder_->Stop();
2221 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002222 stats_proxy_.reset();
2223
2224 EXPECT_EQ(0,
2225 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2226}
2227
mflodmancc3d4422017-08-03 08:27:51 -07002228TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002229 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002230 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002231
2232 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002233 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002234 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002235 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002236
sprang57c2fff2017-01-16 06:24:02 -08002237 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002238 .Times(1);
mflodmancc3d4422017-08-03 08:27:51 -07002239 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002240
sprang57c2fff2017-01-16 06:24:02 -08002241 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002242 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2243 WaitForEncodedFrame(rtc::TimeMillis());
2244 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2245 fake_encoder_.GetAndResetLastBitrateAllocation();
2246 // Check that encoder has been updated too, not just allocation observer.
2247 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
2248 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002249
2250 // Not called on second frame.
2251 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2252 .Times(0);
2253 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002254 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2255 WaitForEncodedFrame(rtc::TimeMillis());
2256 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002257
2258 // Called after a process interval.
2259 const int64_t kProcessIntervalMs =
2260 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002261 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2262 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002263 const int64_t start_time_ms = rtc::TimeMillis();
2264 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2265 video_source_.IncomingCapturedFrame(
2266 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2267 WaitForEncodedFrame(rtc::TimeMillis());
2268 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
2269 }
2270
2271 // Since rates are unchanged, encoder should not be reconfigured.
2272 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002273
mflodmancc3d4422017-08-03 08:27:51 -07002274 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002275}
2276
Niels Möller7dc26b72017-12-06 10:27:48 +01002277TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2278 const int kFrameWidth = 1280;
2279 const int kFrameHeight = 720;
2280 const int kFramerate = 24;
2281
2282 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2283 test::FrameForwarder source;
2284 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002285 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002286
2287 // Insert a single frame, triggering initial configuration.
2288 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2289 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2290
2291 EXPECT_EQ(
2292 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2293 kDefaultFramerate);
2294
2295 // Trigger reconfigure encoder (without resetting the entire instance).
2296 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002297 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002298 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2299 video_encoder_config.number_of_streams = 1;
2300 video_encoder_config.video_stream_factory =
2301 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2302 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002303 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002304 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2305
2306 // Detector should be updated with fps limit from codec config.
2307 EXPECT_EQ(
2308 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2309 kFramerate);
2310
2311 // Trigger overuse, max framerate should be reduced.
2312 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2313 stats.input_frame_rate = kFramerate;
2314 stats_proxy_->SetMockStats(stats);
2315 video_stream_encoder_->TriggerCpuOveruse();
2316 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2317 int adapted_framerate =
2318 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2319 EXPECT_LT(adapted_framerate, kFramerate);
2320
2321 // Trigger underuse, max framerate should go back to codec configured fps.
2322 // Set extra low fps, to make sure it's actually reset, not just incremented.
2323 stats = stats_proxy_->GetStats();
2324 stats.input_frame_rate = adapted_framerate / 2;
2325 stats_proxy_->SetMockStats(stats);
2326 video_stream_encoder_->TriggerCpuNormalUsage();
2327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2328 EXPECT_EQ(
2329 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2330 kFramerate);
2331
2332 video_stream_encoder_->Stop();
2333}
2334
2335TEST_F(VideoStreamEncoderTest,
2336 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2337 const int kFrameWidth = 1280;
2338 const int kFrameHeight = 720;
2339 const int kLowFramerate = 15;
2340 const int kHighFramerate = 25;
2341
2342 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2343 test::FrameForwarder source;
2344 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002345 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002346
2347 // Trigger initial configuration.
2348 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002349 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002350 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2351 video_encoder_config.number_of_streams = 1;
2352 video_encoder_config.video_stream_factory =
2353 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2354 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2355 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002356 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2358
2359 EXPECT_EQ(
2360 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2361 kLowFramerate);
2362
2363 // Trigger overuse, max framerate should be reduced.
2364 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2365 stats.input_frame_rate = kLowFramerate;
2366 stats_proxy_->SetMockStats(stats);
2367 video_stream_encoder_->TriggerCpuOveruse();
2368 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2369 int adapted_framerate =
2370 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2371 EXPECT_LT(adapted_framerate, kLowFramerate);
2372
2373 // Reconfigure the encoder with a new (higher max framerate), max fps should
2374 // still respect the adaptation.
2375 video_encoder_config.video_stream_factory =
2376 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2377 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2378 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002379 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2381
2382 EXPECT_EQ(
2383 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2384 adapted_framerate);
2385
2386 // Trigger underuse, max framerate should go back to codec configured fps.
2387 stats = stats_proxy_->GetStats();
2388 stats.input_frame_rate = adapted_framerate;
2389 stats_proxy_->SetMockStats(stats);
2390 video_stream_encoder_->TriggerCpuNormalUsage();
2391 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2392 EXPECT_EQ(
2393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2394 kHighFramerate);
2395
2396 video_stream_encoder_->Stop();
2397}
2398
mflodmancc3d4422017-08-03 08:27:51 -07002399TEST_F(VideoStreamEncoderTest,
2400 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002401 const int kFrameWidth = 1280;
2402 const int kFrameHeight = 720;
2403 const int kFramerate = 24;
2404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002406 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002408 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002409
2410 // Trigger initial configuration.
2411 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002412 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002413 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2414 video_encoder_config.number_of_streams = 1;
2415 video_encoder_config.video_stream_factory =
2416 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2417 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002418 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002419 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002420 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002421
Niels Möller7dc26b72017-12-06 10:27:48 +01002422 EXPECT_EQ(
2423 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2424 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002425
2426 // Trigger overuse, max framerate should be reduced.
2427 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2428 stats.input_frame_rate = kFramerate;
2429 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002430 video_stream_encoder_->TriggerCpuOveruse();
2431 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002432 int adapted_framerate =
2433 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002434 EXPECT_LT(adapted_framerate, kFramerate);
2435
2436 // Change degradation preference to not enable framerate scaling. Target
2437 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002439 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002441 EXPECT_EQ(
2442 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2443 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002444
mflodmancc3d4422017-08-03 08:27:51 -07002445 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002446}
2447
mflodmancc3d4422017-08-03 08:27:51 -07002448TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002449 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002451 const int kWidth = 640;
2452 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002453
asaperssonfab67072017-04-04 05:51:49 -07002454 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002455
2456 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002457 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002458
2459 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002460 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002461
sprangc5d62e22017-04-02 23:53:04 -07002462 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002463
asaperssonfab67072017-04-04 05:51:49 -07002464 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002465 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002466 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002467
2468 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002469 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002470
sprangc5d62e22017-04-02 23:53:04 -07002471 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002472
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002474}
2475
mflodmancc3d4422017-08-03 08:27:51 -07002476TEST_F(VideoStreamEncoderTest,
2477 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002478 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002479 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002480 const int kWidth = 640;
2481 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002482
2483 // We expect the n initial frames to get dropped.
2484 int i;
2485 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002486 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002487 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002488 }
2489 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002490 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002491 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002492
2493 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002494 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002495
mflodmancc3d4422017-08-03 08:27:51 -07002496 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002497}
2498
mflodmancc3d4422017-08-03 08:27:51 -07002499TEST_F(VideoStreamEncoderTest,
2500 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002501 const int kWidth = 640;
2502 const int kHeight = 360;
mflodmancc3d4422017-08-03 08:27:51 -07002503 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002504
2505 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002506 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002507 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002508
asaperssonfab67072017-04-04 05:51:49 -07002509 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002510 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002511 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002512
mflodmancc3d4422017-08-03 08:27:51 -07002513 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002514}
2515
mflodmancc3d4422017-08-03 08:27:51 -07002516TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002517 const int kWidth = 640;
2518 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002519 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002520
2521 VideoEncoderConfig video_encoder_config;
2522 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2523 // Make format different, to force recreation of encoder.
2524 video_encoder_config.video_format.parameters["foo"] = "foo";
2525 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002526 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002528
kthelgasonb83797b2017-02-14 11:57:25 -08002529 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002530 video_stream_encoder_->SetSource(&video_source_,
2531 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002532
asaperssonfab67072017-04-04 05:51:49 -07002533 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002534 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002535 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002536
mflodmancc3d4422017-08-03 08:27:51 -07002537 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002538 fake_encoder_.SetQualityScaling(true);
2539}
2540
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002541TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2542 webrtc::test::ScopedFieldTrials field_trials(
2543 "WebRTC-InitialFramedrop/Enabled/");
2544 // Reset encoder for field trials to take effect.
2545 ConfigureEncoder(video_encoder_config_.Copy());
2546 const int kTooLowBitrateForFrameSizeBps = 10000;
2547 const int kWidth = 640;
2548 const int kHeight = 360;
2549
2550 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2551 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2552 // Frame should not be dropped.
2553 WaitForEncodedFrame(1);
2554
2555 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2556 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2557 // Expect to drop this frame, the wait should time out.
2558 ExpectDroppedFrame();
2559
2560 // Expect the sink_wants to specify a scaled frame.
2561 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2562 video_stream_encoder_->Stop();
2563}
2564
mflodmancc3d4422017-08-03 08:27:51 -07002565TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002566 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2567 const int kTooSmallWidth = 10;
2568 const int kTooSmallHeight = 10;
mflodmancc3d4422017-08-03 08:27:51 -07002569 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002570
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002571 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002572 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002573 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002574 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002575 VerifyNoLimitation(source.sink_wants());
2576 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2577
2578 // Trigger adapt down, too small frame, expect no change.
2579 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002580 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002581 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002582 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002583 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2584 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2585
mflodmancc3d4422017-08-03 08:27:51 -07002586 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002587}
2588
mflodmancc3d4422017-08-03 08:27:51 -07002589TEST_F(VideoStreamEncoderTest,
2590 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002591 const int kTooSmallWidth = 10;
2592 const int kTooSmallHeight = 10;
2593 const int kFpsLimit = 7;
mflodmancc3d4422017-08-03 08:27:51 -07002594 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002595
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002596 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002597 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002598 video_stream_encoder_->SetSource(&source,
2599 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002600 VerifyNoLimitation(source.sink_wants());
2601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2602 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2603
2604 // Trigger adapt down, expect limited framerate.
2605 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002606 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002607 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002608 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2611 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2612
2613 // Trigger adapt down, too small frame, expect no change.
2614 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002615 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002616 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002617 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2618 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2621
mflodmancc3d4422017-08-03 08:27:51 -07002622 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002623}
2624
mflodmancc3d4422017-08-03 08:27:51 -07002625TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002626 fake_encoder_.ForceInitEncodeFailure(true);
mflodmancc3d4422017-08-03 08:27:51 -07002627 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002628 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002629 const int kFrameWidth = 1280;
2630 const int kFrameHeight = 720;
2631 video_source_.IncomingCapturedFrame(
2632 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002633 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002635}
2636
sprangb1ca0732017-02-01 08:38:12 -08002637// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002638TEST_F(VideoStreamEncoderTest,
2639 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
2640 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002641
2642 const int kFrameWidth = 1280;
2643 const int kFrameHeight = 720;
2644 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002645 // requested by
2646 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002647 video_source_.set_adaptation_enabled(true);
2648
2649 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002650 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002651 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002652
2653 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002654 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002655 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002656 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002657 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002658
asaperssonfab67072017-04-04 05:51:49 -07002659 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002660 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002661 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002662 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002663 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002664
mflodmancc3d4422017-08-03 08:27:51 -07002665 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002666}
sprangfe627f32017-03-29 08:24:59 -07002667
mflodmancc3d4422017-08-03 08:27:51 -07002668TEST_F(VideoStreamEncoderTest,
2669 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002670 const int kFrameWidth = 1280;
2671 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002672
mflodmancc3d4422017-08-03 08:27:51 -07002673 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2674 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002675 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002676 video_source_.set_adaptation_enabled(true);
2677
sprang4847ae62017-06-27 07:06:52 -07002678 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002679
2680 video_source_.IncomingCapturedFrame(
2681 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002682 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002683
2684 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002686
2687 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002688 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002689 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002690 video_source_.IncomingCapturedFrame(
2691 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002692 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002693 }
2694
2695 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002697 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002698 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002699 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002700 video_source_.IncomingCapturedFrame(
2701 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002702 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002703 ++num_frames_dropped;
2704 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002705 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002706 }
2707 }
2708
sprang4847ae62017-06-27 07:06:52 -07002709 // Add some slack to account for frames dropped by the frame dropper.
2710 const int kErrorMargin = 1;
2711 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002712 kErrorMargin);
2713
2714 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002715 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002716 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002717 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002718 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002719 video_source_.IncomingCapturedFrame(
2720 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002721 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002722 ++num_frames_dropped;
2723 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002724 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002725 }
2726 }
sprang4847ae62017-06-27 07:06:52 -07002727 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002728 kErrorMargin);
2729
2730 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002732 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002733 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002734 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002735 video_source_.IncomingCapturedFrame(
2736 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002737 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002738 ++num_frames_dropped;
2739 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002740 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002741 }
2742 }
sprang4847ae62017-06-27 07:06:52 -07002743 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002744 kErrorMargin);
2745
2746 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002747 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002748 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002749 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002750 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002751 video_source_.IncomingCapturedFrame(
2752 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002753 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002754 ++num_frames_dropped;
2755 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002756 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002757 }
2758 }
2759 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2760
mflodmancc3d4422017-08-03 08:27:51 -07002761 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002762}
2763
mflodmancc3d4422017-08-03 08:27:51 -07002764TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002765 const int kFramerateFps = 5;
2766 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002767 const int kFrameWidth = 1280;
2768 const int kFrameHeight = 720;
2769
sprang4847ae62017-06-27 07:06:52 -07002770 // Reconfigure encoder with two temporal layers and screensharing, which will
2771 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002772 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002773
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2775 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002776 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002777 video_source_.set_adaptation_enabled(true);
2778
sprang4847ae62017-06-27 07:06:52 -07002779 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002780
2781 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002782 rtc::VideoSinkWants last_wants;
2783 do {
2784 last_wants = video_source_.sink_wants();
2785
sprangc5d62e22017-04-02 23:53:04 -07002786 // Insert frames to get a new fps estimate...
2787 for (int j = 0; j < kFramerateFps; ++j) {
2788 video_source_.IncomingCapturedFrame(
2789 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08002790 if (video_source_.last_sent_width()) {
2791 sink_.WaitForEncodedFrame(timestamp_ms);
2792 }
sprangc5d62e22017-04-02 23:53:04 -07002793 timestamp_ms += kFrameIntervalMs;
Yves Gerey665174f2018-06-19 15:03:05 +02002794 fake_clock_.AdvanceTimeMicros(kFrameIntervalMs *
2795 rtc::kNumMicrosecsPerMillisec);
sprangc5d62e22017-04-02 23:53:04 -07002796 }
2797 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07002798 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08002799 } while (video_source_.sink_wants().max_framerate_fps <
2800 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002801
Jonathan Yubc771b72017-12-08 17:04:29 -08002802 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07002803
mflodmancc3d4422017-08-03 08:27:51 -07002804 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002805}
asaperssonf7e294d2017-06-13 23:25:22 -07002806
mflodmancc3d4422017-08-03 08:27:51 -07002807TEST_F(VideoStreamEncoderTest,
2808 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002809 const int kWidth = 1280;
2810 const int kHeight = 720;
2811 const int64_t kFrameIntervalMs = 150;
2812 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07002813 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002814
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002816 AdaptingFrameForwarder source;
2817 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002818 video_stream_encoder_->SetSource(&source,
2819 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002820 timestamp_ms += kFrameIntervalMs;
2821 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002822 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002823 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002824 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2825 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2826 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2827
2828 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002829 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002830 timestamp_ms += kFrameIntervalMs;
2831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002832 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002833 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2834 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2835 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2836 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2837
2838 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002839 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002840 timestamp_ms += kFrameIntervalMs;
2841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002842 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002843 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2844 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2845 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2846 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2847
2848 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002849 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002850 timestamp_ms += kFrameIntervalMs;
2851 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002852 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002853 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2854 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2856 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2857
2858 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002860 timestamp_ms += kFrameIntervalMs;
2861 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002862 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002863 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2864 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2866 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2867
2868 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002870 timestamp_ms += kFrameIntervalMs;
2871 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002872 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002873 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2874 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2876 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2877
2878 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002879 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002880 timestamp_ms += kFrameIntervalMs;
2881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002883 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2884 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2885 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2886 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2887
2888 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07002889 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002890 timestamp_ms += kFrameIntervalMs;
2891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002892 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002893 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2894 rtc::VideoSinkWants last_wants = source.sink_wants();
2895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2897 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2898
2899 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002901 timestamp_ms += kFrameIntervalMs;
2902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002904 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2907 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2908
2909 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002910 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002911 timestamp_ms += kFrameIntervalMs;
2912 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002914 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2915 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2916 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2917 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2918
2919 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002920 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002921 timestamp_ms += kFrameIntervalMs;
2922 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002923 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002924 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2925 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2926 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2927 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2928
2929 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002930 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002931 timestamp_ms += kFrameIntervalMs;
2932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002933 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002934 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2935 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2936 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2937 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2938
2939 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002940 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002941 timestamp_ms += kFrameIntervalMs;
2942 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002943 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002944 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2945 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2946 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2947 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2948
2949 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002951 timestamp_ms += kFrameIntervalMs;
2952 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002953 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002954 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2955 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2956 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2957 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2958
2959 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002960 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002961 timestamp_ms += kFrameIntervalMs;
2962 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002963 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002964 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2965 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2966 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2967 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2968
2969 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002971 timestamp_ms += kFrameIntervalMs;
2972 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002973 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07002974 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002975 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002976 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2978 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2979
2980 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002982 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002983 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2984
mflodmancc3d4422017-08-03 08:27:51 -07002985 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002986}
2987
mflodmancc3d4422017-08-03 08:27:51 -07002988TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07002989 const int kWidth = 1280;
2990 const int kHeight = 720;
2991 const int64_t kFrameIntervalMs = 150;
2992 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07002993 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002994
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002995 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002996 AdaptingFrameForwarder source;
2997 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002998 video_stream_encoder_->SetSource(&source,
2999 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003000 timestamp_ms += kFrameIntervalMs;
3001 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003002 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003003 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3005 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3006 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3007 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3008 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3009 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3010
3011 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003012 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003013 timestamp_ms += kFrameIntervalMs;
3014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003015 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003016 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3018 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3019 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3020 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3021 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3022 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3023
3024 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003026 timestamp_ms += kFrameIntervalMs;
3027 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003028 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003029 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3030 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3031 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3032 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3033 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3034 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3035 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3036
3037 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003038 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003039 timestamp_ms += kFrameIntervalMs;
3040 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003041 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003042 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3043 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3044 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3045 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3046 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3047 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3048 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3049
3050 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003051 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003052 timestamp_ms += kFrameIntervalMs;
3053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003054 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003055 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3056 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3057 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3058 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3059 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3060 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3061 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3062
3063 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003064 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003065 timestamp_ms += kFrameIntervalMs;
3066 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003067 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003068 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3069 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3070 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3071 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3072 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3073 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3074 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3075
3076 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003077 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003078 timestamp_ms += kFrameIntervalMs;
3079 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003080 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003081 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003082 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003083 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3084 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3085 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3086 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3087 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3088 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003091 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003093 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3094 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3095
mflodmancc3d4422017-08-03 08:27:51 -07003096 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003097}
3098
mflodmancc3d4422017-08-03 08:27:51 -07003099TEST_F(VideoStreamEncoderTest,
3100 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003101 const int kWidth = 640;
3102 const int kHeight = 360;
3103 const int kFpsLimit = 15;
3104 const int64_t kFrameIntervalMs = 150;
3105 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003106 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003107
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003108 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003109 AdaptingFrameForwarder source;
3110 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003111 video_stream_encoder_->SetSource(&source,
3112 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003113 timestamp_ms += kFrameIntervalMs;
3114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003116 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3118 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3119 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3120 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3121 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3122 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3123
3124 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003125 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003126 timestamp_ms += kFrameIntervalMs;
3127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003128 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003129 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3130 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3131 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3132 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3133 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3134 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3135 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3136
3137 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003138 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003139 timestamp_ms += kFrameIntervalMs;
3140 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003141 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003142 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3143 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3144 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3145 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3146 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3147 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3148 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3149
3150 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003151 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003152 timestamp_ms += kFrameIntervalMs;
3153 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003154 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003155 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3156 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3157 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3158 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3159 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3160 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3161 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3162
3163 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003165 timestamp_ms += kFrameIntervalMs;
3166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003167 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003168 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003169 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3170 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3171 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3172 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3173 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3174 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3175
3176 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003177 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003178 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003179 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3180 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3181
mflodmancc3d4422017-08-03 08:27:51 -07003182 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003183}
3184
mflodmancc3d4422017-08-03 08:27:51 -07003185TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003186 // Simulates simulcast behavior and makes highest stream resolutions divisible
3187 // by 4.
3188 class CroppingVideoStreamFactory
3189 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3190 public:
3191 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3192 int framerate)
3193 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3194 EXPECT_GT(num_temporal_layers, 0u);
3195 EXPECT_GT(framerate, 0);
3196 }
3197
3198 private:
3199 std::vector<VideoStream> CreateEncoderStreams(
3200 int width,
3201 int height,
3202 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003203 std::vector<VideoStream> streams = test::CreateVideoStreams(
3204 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003205 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003206 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003207 stream.max_framerate = framerate_;
3208 }
3209 return streams;
3210 }
3211
3212 const size_t num_temporal_layers_;
3213 const int framerate_;
3214 };
3215
3216 const int kFrameWidth = 1920;
3217 const int kFrameHeight = 1080;
3218 // 3/4 of 1920.
3219 const int kAdaptedFrameWidth = 1440;
3220 // 3/4 of 1080 rounded down to multiple of 4.
3221 const int kAdaptedFrameHeight = 808;
3222 const int kFramerate = 24;
3223
mflodmancc3d4422017-08-03 08:27:51 -07003224 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003225 // Trigger reconfigure encoder (without resetting the entire instance).
3226 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003227 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003228 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3229 video_encoder_config.number_of_streams = 1;
3230 video_encoder_config.video_stream_factory =
3231 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003232 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003233 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003234 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003235
3236 video_source_.set_adaptation_enabled(true);
3237
3238 video_source_.IncomingCapturedFrame(
3239 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003240 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003241
3242 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003243 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003244 video_source_.IncomingCapturedFrame(
3245 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003246 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003247
mflodmancc3d4422017-08-03 08:27:51 -07003248 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003249}
3250
mflodmancc3d4422017-08-03 08:27:51 -07003251TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003252 const int kFrameWidth = 1280;
3253 const int kFrameHeight = 720;
3254 const int kLowFps = 2;
3255 const int kHighFps = 30;
3256
mflodmancc3d4422017-08-03 08:27:51 -07003257 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003258
3259 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3260 max_framerate_ = kLowFps;
3261
3262 // Insert 2 seconds of 2fps video.
3263 for (int i = 0; i < kLowFps * 2; ++i) {
3264 video_source_.IncomingCapturedFrame(
3265 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3266 WaitForEncodedFrame(timestamp_ms);
3267 timestamp_ms += 1000 / kLowFps;
3268 }
3269
3270 // Make sure encoder is updated with new target.
mflodmancc3d4422017-08-03 08:27:51 -07003271 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003272 video_source_.IncomingCapturedFrame(
3273 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3274 WaitForEncodedFrame(timestamp_ms);
3275 timestamp_ms += 1000 / kLowFps;
3276
3277 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3278
3279 // Insert 30fps frames for just a little more than the forced update period.
3280 const int kVcmTimerIntervalFrames =
3281 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3282 const int kFrameIntervalMs = 1000 / kHighFps;
3283 max_framerate_ = kHighFps;
3284 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3285 video_source_.IncomingCapturedFrame(
3286 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3287 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3288 // be dropped if the encoder hans't been updated with the new higher target
3289 // framerate yet, causing it to overshoot the target bitrate and then
3290 // suffering the wrath of the media optimizer.
3291 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3292 timestamp_ms += kFrameIntervalMs;
3293 }
3294
3295 // Don expect correct measurement just yet, but it should be higher than
3296 // before.
3297 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003300}
3301
mflodmancc3d4422017-08-03 08:27:51 -07003302TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003303 const int kFrameWidth = 1280;
3304 const int kFrameHeight = 720;
3305 const int kTargetBitrateBps = 1000000;
3306
3307 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003308 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
mflodmancc3d4422017-08-03 08:27:51 -07003309 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3310 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003311
3312 // Insert a first video frame, causes another bitrate update.
3313 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3314 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3315 video_source_.IncomingCapturedFrame(
3316 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3317 WaitForEncodedFrame(timestamp_ms);
3318
3319 // Next, simulate video suspension due to pacer queue overrun.
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->OnBitrateUpdated(0, 0, 1);
sprang4847ae62017-06-27 07:06:52 -07003321
3322 // Skip ahead until a new periodic parameter update should have occured.
3323 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
3324 fake_clock_.AdvanceTimeMicros(
3325 vcm::VCMProcessTimer::kDefaultProcessIntervalMs *
3326 rtc::kNumMicrosecsPerMillisec);
3327
3328 // Bitrate observer should not be called.
3329 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3330 video_source_.IncomingCapturedFrame(
3331 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3332 ExpectDroppedFrame();
3333
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003335}
ilnik6b826ef2017-06-16 06:53:48 -07003336
Niels Möller4db138e2018-04-19 09:04:13 +02003337TEST_F(VideoStreamEncoderTest,
3338 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3339 const int kFrameWidth = 1280;
3340 const int kFrameHeight = 720;
3341 const CpuOveruseOptions default_options;
3342 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3343 video_source_.IncomingCapturedFrame(
3344 CreateFrame(1, kFrameWidth, kFrameHeight));
3345 WaitForEncodedFrame(1);
3346 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3347 .low_encode_usage_threshold_percent,
3348 default_options.low_encode_usage_threshold_percent);
3349 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3350 .high_encode_usage_threshold_percent,
3351 default_options.high_encode_usage_threshold_percent);
3352 video_stream_encoder_->Stop();
3353}
3354
3355TEST_F(VideoStreamEncoderTest,
3356 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3357 const int kFrameWidth = 1280;
3358 const int kFrameHeight = 720;
3359 CpuOveruseOptions hardware_options;
3360 hardware_options.low_encode_usage_threshold_percent = 150;
3361 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003362 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003363
3364 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3365 video_source_.IncomingCapturedFrame(
3366 CreateFrame(1, kFrameWidth, kFrameHeight));
3367 WaitForEncodedFrame(1);
3368 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3369 .low_encode_usage_threshold_percent,
3370 hardware_options.low_encode_usage_threshold_percent);
3371 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3372 .high_encode_usage_threshold_percent,
3373 hardware_options.high_encode_usage_threshold_percent);
3374 video_stream_encoder_->Stop();
3375}
3376
Niels Möller6bb5ab92019-01-11 11:11:10 +01003377TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3378 const int kFrameWidth = 320;
3379 const int kFrameHeight = 240;
3380 const int kFps = 30;
3381 const int kTargetBitrateBps = 120000;
3382 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3383
3384 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3385
3386 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3387 max_framerate_ = kFps;
3388
3389 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3390 fake_encoder_.SimulateOvershoot(1.0);
3391 int num_dropped = 0;
3392 for (int i = 0; i < kNumFramesInRun; ++i) {
3393 video_source_.IncomingCapturedFrame(
3394 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3395 // Wait up to two frame durations for a frame to arrive.
3396 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3397 ++num_dropped;
3398 }
3399 timestamp_ms += 1000 / kFps;
3400 }
3401
Erik Språnga8d48ab2019-02-08 14:17:40 +01003402 // Framerate should be measured to be near the expected target rate.
3403 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3404
3405 // Frame drops should be within 5% of expected 0%.
3406 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003407
3408 // Make encoder produce frames at double the expected bitrate during 3 seconds
3409 // of video, verify number of drops. Rate needs to be slightly changed in
3410 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003411 double overshoot_factor = 2.0;
3412 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3413 // With bitrate adjuster, when need to overshoot even more to trigger
3414 // frame dropping.
3415 overshoot_factor *= 2;
3416 }
3417 fake_encoder_.SimulateOvershoot(overshoot_factor);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003418 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0);
3419 num_dropped = 0;
3420 for (int i = 0; i < kNumFramesInRun; ++i) {
3421 video_source_.IncomingCapturedFrame(
3422 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3423 // Wait up to two frame durations for a frame to arrive.
3424 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3425 ++num_dropped;
3426 }
3427 timestamp_ms += 1000 / kFps;
3428 }
3429
Erik Språnga8d48ab2019-02-08 14:17:40 +01003430 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3431
3432 // Target framerate should be still be near the expected target, despite
3433 // the frame drops.
3434 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3435
3436 // Frame drops should be within 5% of expected 50%.
3437 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003438
3439 video_stream_encoder_->Stop();
3440}
3441
3442TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3443 const int kFrameWidth = 320;
3444 const int kFrameHeight = 240;
3445 const int kActualInputFps = 24;
3446 const int kTargetBitrateBps = 120000;
3447
3448 ASSERT_GT(max_framerate_, kActualInputFps);
3449
3450 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3451 max_framerate_ = kActualInputFps;
3452 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3453
3454 // Insert 3 seconds of video, with an input fps lower than configured max.
3455 for (int i = 0; i < kActualInputFps * 3; ++i) {
3456 video_source_.IncomingCapturedFrame(
3457 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3458 // Wait up to two frame durations for a frame to arrive.
3459 WaitForEncodedFrame(timestamp_ms);
3460 timestamp_ms += 1000 / kActualInputFps;
3461 }
3462
3463 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3464
3465 video_stream_encoder_->Stop();
3466}
3467
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003468TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3469 VideoFrame::UpdateRect rect;
3470 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3471
3472 fake_encoder_.BlockNextEncode();
3473 video_source_.IncomingCapturedFrame(
3474 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3475 WaitForEncodedFrame(1);
3476 // On the very first frame full update should be forced.
3477 rect = fake_encoder_.GetLastUpdateRect();
3478 EXPECT_EQ(rect.offset_x, 0);
3479 EXPECT_EQ(rect.offset_y, 0);
3480 EXPECT_EQ(rect.height, codec_height_);
3481 EXPECT_EQ(rect.width, codec_width_);
3482 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3483 // call to ContinueEncode.
3484 video_source_.IncomingCapturedFrame(
3485 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3486 ExpectDroppedFrame();
3487 video_source_.IncomingCapturedFrame(
3488 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3489 ExpectDroppedFrame();
3490 fake_encoder_.ContinueEncode();
3491 WaitForEncodedFrame(3);
3492 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3493 rect = fake_encoder_.GetLastUpdateRect();
3494 EXPECT_EQ(rect.offset_x, 1);
3495 EXPECT_EQ(rect.offset_y, 0);
3496 EXPECT_EQ(rect.width, 10);
3497 EXPECT_EQ(rect.height, 1);
3498
3499 video_source_.IncomingCapturedFrame(
3500 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3501 WaitForEncodedFrame(4);
3502 // Previous frame was encoded, so no accumulation should happen.
3503 rect = fake_encoder_.GetLastUpdateRect();
3504 EXPECT_EQ(rect.offset_x, 0);
3505 EXPECT_EQ(rect.offset_y, 0);
3506 EXPECT_EQ(rect.width, 1);
3507 EXPECT_EQ(rect.height, 1);
3508
3509 video_stream_encoder_->Stop();
3510}
3511
Erik Språngd7329ca2019-02-21 21:19:53 +01003512TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
3513 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3514
3515 // First frame is always keyframe.
3516 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3517 WaitForEncodedFrame(1);
3518 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3519 testing::ElementsAre(FrameType{kVideoFrameKey}));
3520
3521 // Insert delta frame.
3522 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3523 WaitForEncodedFrame(2);
3524 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3525 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3526
3527 // Request next frame be a key-frame.
3528 video_stream_encoder_->SendKeyFrame();
3529 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3530 WaitForEncodedFrame(3);
3531 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3532 testing::ElementsAre(FrameType{kVideoFrameKey}));
3533
3534 video_stream_encoder_->Stop();
3535}
3536
3537TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3538 // Setup simulcast with three streams.
3539 ResetEncoder("VP8", 3, 1, 1, false);
3540 video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0);
3541 // Wait for all three layers before triggering event.
3542 sink_.SetNumExpectedLayers(3);
3543
3544 // First frame is always keyframe.
3545 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3546 WaitForEncodedFrame(1);
3547 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3548 testing::ElementsAreArray(
3549 {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey}));
3550
3551 // Insert delta frame.
3552 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3553 WaitForEncodedFrame(2);
3554 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3555 testing::ElementsAreArray(
3556 {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta}));
3557
3558 // Request next frame be a key-frame.
3559 // Only first stream is configured to produce key-frame.
3560 video_stream_encoder_->SendKeyFrame();
3561 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3562 WaitForEncodedFrame(3);
3563 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3564 testing::ElementsAreArray(
3565 {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta}));
3566
3567 video_stream_encoder_->Stop();
3568}
3569
3570TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3571 // Configure internal source factory and setup test again.
3572 encoder_factory_.SetHasInternalSource(true);
3573 ResetEncoder("VP8", 1, 1, 1, false);
3574 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3575
3576 // Call encoder directly, simulating internal source where encoded frame
3577 // callback in VideoStreamEncoder is called despite no OnFrame().
3578 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3579 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3580 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3581 testing::ElementsAre(FrameType{kVideoFrameKey}));
3582
3583 const std::vector<FrameType> kDeltaFrame = {kVideoFrameDelta};
3584 // Need to set timestamp manually since manually for injected frame.
3585 VideoFrame frame = CreateFrame(101, nullptr);
3586 frame.set_timestamp(101);
3587 fake_encoder_.InjectFrame(frame, false);
3588 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3589 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3590 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3591
3592 // Request key-frame. The forces a dummy frame down into the encoder.
3593 fake_encoder_.ExpectNullFrame();
3594 video_stream_encoder_->SendKeyFrame();
3595 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3596 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3597 testing::ElementsAre(FrameType{kVideoFrameKey}));
3598
3599 video_stream_encoder_->Stop();
3600}
perkj26091b12016-09-01 01:17:40 -07003601} // namespace webrtc