blob: da7095cbe6721d4f04c5f93cb36132db989511a4 [file] [log] [blame]
pbos@webrtc.org744fbc72013-09-10 09:26:25 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000010
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "call/rampup_tests.h"
stefanff483612015-12-21 03:14:00 -080012
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020013#include "rtc_base/checks.h"
14#include "rtc_base/logging.h"
15#include "rtc_base/platform_thread.h"
16#include "test/encoder_settings.h"
17#include "test/gtest.h"
18#include "test/testsupport/perf_test.h"
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000019
20namespace webrtc {
pbos@webrtc.org29023282013-09-11 10:14:56 +000021namespace {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000022
Stefan Holmer723dff12015-10-05 14:59:41 +020023static const int64_t kPollIntervalMs = 20;
philipel5ef2bc12017-02-21 07:28:31 -080024static const int kExpectedHighVideoBitrateBps = 80000;
stefandb752f92016-12-05 08:23:40 -080025static const int kExpectedHighAudioBitrateBps = 30000;
26static const int kLowBandwidthLimitBps = 20000;
27static const int kExpectedLowBitrateBps = 20000;
pbos@webrtc.org29023282013-09-11 10:14:56 +000028
stefanff483612015-12-21 03:14:00 -080029std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000030 std::vector<uint32_t> ssrcs;
31 for (size_t i = 0; i != num_streams; ++i)
32 ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
33 return ssrcs;
34}
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +000035} // namespace
henrik.lundin@webrtc.org845862f2014-03-06 07:19:28 +000036
stefanff483612015-12-21 03:14:00 -080037RampUpTester::RampUpTester(size_t num_video_streams,
38 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -080039 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -070040 unsigned int start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -080041 int64_t min_run_time_ms,
stefan4fbd1452015-09-28 03:57:14 -070042 const std::string& extension_type,
43 bool rtx,
stefan5a2c5062017-01-27 06:43:18 -080044 bool red,
45 bool report_perf_stats)
stefan4fbd1452015-09-28 03:57:14 -070046 : EndToEndTest(test::CallTest::kLongTimeoutMs),
tommi0f8b4032017-02-22 11:22:05 -080047 stop_event_(false, false),
stefan4fbd1452015-09-28 03:57:14 -070048 clock_(Clock::GetRealTimeClock()),
stefanff483612015-12-21 03:14:00 -080049 num_video_streams_(num_video_streams),
50 num_audio_streams_(num_audio_streams),
philipel5ef2bc12017-02-21 07:28:31 -080051 num_flexfec_streams_(num_flexfec_streams),
stefan4fbd1452015-09-28 03:57:14 -070052 rtx_(rtx),
53 red_(red),
stefan45b5fe52017-03-09 06:27:02 -080054 report_perf_stats_(report_perf_stats),
mflodman86cc6ff2016-07-26 04:44:06 -070055 sender_call_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070056 send_stream_(nullptr),
sprang4847ae62017-06-27 07:06:52 -070057 send_transport_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070058 start_bitrate_bps_(start_bitrate_bps),
stefan5a2c5062017-01-27 06:43:18 -080059 min_run_time_ms_(min_run_time_ms),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000060 expected_bitrate_bps_(0),
Erik Språngf3a7c9d2015-10-05 14:03:22 +020061 test_start_ms_(-1),
stefan4fbd1452015-09-28 03:57:14 -070062 ramp_up_finished_ms_(-1),
63 extension_type_(extension_type),
stefanff483612015-12-21 03:14:00 -080064 video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)),
65 video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)),
66 audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)),
Peter Boström8c38e8b2015-11-26 17:45:47 +010067 poller_thread_(&BitrateStatsPollingThread,
68 this,
mflodman86cc6ff2016-07-26 04:44:06 -070069 "BitrateStatsPollingThread") {
philipel5ef2bc12017-02-21 07:28:31 -080070 if (red_)
71 EXPECT_EQ(0u, num_flexfec_streams_);
Stefan Holmerff2a6352016-01-14 10:00:21 +010072 EXPECT_LE(num_audio_streams_, 1u);
stefan4fbd1452015-09-28 03:57:14 -070073}
74
75RampUpTester::~RampUpTester() {
stefan4fbd1452015-09-28 03:57:14 -070076}
77
78Call::Config RampUpTester::GetSenderCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -070079 Call::Config call_config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -070080 if (start_bitrate_bps_ != 0) {
81 call_config.bitrate_config.start_bitrate_bps = start_bitrate_bps_;
82 }
83 call_config.bitrate_config.min_bitrate_bps = 10000;
84 return call_config;
85}
86
stefanff483612015-12-21 03:14:00 -080087void RampUpTester::OnVideoStreamsCreated(
stefan4fbd1452015-09-28 03:57:14 -070088 VideoSendStream* send_stream,
89 const std::vector<VideoReceiveStream*>& receive_streams) {
90 send_stream_ = send_stream;
91}
92
eladalon413ee9a2017-08-22 04:02:52 -070093test::PacketTransport* RampUpTester::CreateSendTransport(
94 test::SingleThreadedTaskQueueForTesting* task_queue,
95 Call* sender_call) {
minyue20c84cc2017-04-10 16:57:57 -070096 send_transport_ = new test::PacketTransport(
eladalon413ee9a2017-08-22 04:02:52 -070097 task_queue, sender_call, this, test::PacketTransport::kSender,
minyue20c84cc2017-04-10 16:57:57 -070098 test::CallTest::payload_type_map_, forward_transport_config_);
stefane74eef12016-01-08 06:47:13 -080099 return send_transport_;
stefanf116bd02015-10-27 08:29:42 -0700100}
101
Stefan Holmerd20e6512016-01-12 15:51:22 +0100102size_t RampUpTester::GetNumVideoStreams() const {
103 return num_video_streams_;
104}
105
Stefan Holmerff2a6352016-01-14 10:00:21 +0100106size_t RampUpTester::GetNumAudioStreams() const {
107 return num_audio_streams_;
108}
109
philipel5ef2bc12017-02-21 07:28:31 -0800110size_t RampUpTester::GetNumFlexfecStreams() const {
111 return num_flexfec_streams_;
112}
113
perkjfa10b552016-10-02 23:45:26 -0700114class RampUpTester::VideoStreamFactory
115 : public VideoEncoderConfig::VideoStreamFactoryInterface {
116 public:
117 VideoStreamFactory() {}
118
119 private:
120 std::vector<VideoStream> CreateEncoderStreams(
121 int width,
122 int height,
123 const VideoEncoderConfig& encoder_config) override {
124 std::vector<VideoStream> streams =
125 test::CreateVideoStreams(width, height, encoder_config);
126 if (encoder_config.number_of_streams == 1) {
127 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
128 }
129 return streams;
130 }
131};
132
stefanff483612015-12-21 03:14:00 -0800133void RampUpTester::ModifyVideoConfigs(
stefan4fbd1452015-09-28 03:57:14 -0700134 VideoSendStream::Config* send_config,
135 std::vector<VideoReceiveStream::Config>* receive_configs,
136 VideoEncoderConfig* encoder_config) {
137 send_config->suspend_below_min_bitrate = true;
perkjfa10b552016-10-02 23:45:26 -0700138 encoder_config->number_of_streams = num_video_streams_;
139 encoder_config->max_bitrate_bps = 2000000;
140 encoder_config->video_stream_factory =
141 new rtc::RefCountedObject<RampUpTester::VideoStreamFactory>();
stefanff483612015-12-21 03:14:00 -0800142 if (num_video_streams_ == 1) {
stefan4fbd1452015-09-28 03:57:14 -0700143 // For single stream rampup until 1mbps
144 expected_bitrate_bps_ = kSingleStreamTargetBps;
145 } else {
146 // For multi stream rampup until all streams are being sent. That means
perkjfa10b552016-10-02 23:45:26 -0700147 // enough bitrate to send all the target streams plus the min bitrate of
stefan4fbd1452015-09-28 03:57:14 -0700148 // the last one.
perkjfa10b552016-10-02 23:45:26 -0700149 std::vector<VideoStream> streams = test::CreateVideoStreams(
150 test::CallTest::kDefaultWidth, test::CallTest::kDefaultHeight,
151 *encoder_config);
152 expected_bitrate_bps_ = streams.back().min_bitrate_bps;
153 for (size_t i = 0; i < streams.size() - 1; ++i) {
154 expected_bitrate_bps_ += streams[i].target_bitrate_bps;
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000155 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000156 }
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000157
stefan4fbd1452015-09-28 03:57:14 -0700158 send_config->rtp.extensions.clear();
159
160 bool remb;
stefan43edf0f2015-11-20 18:05:48 -0800161 bool transport_cc;
isheriff6f8d6862016-05-26 11:24:55 -0700162 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
stefan4fbd1452015-09-28 03:57:14 -0700163 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800164 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700165 send_config->rtp.extensions.push_back(
166 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
isheriff6f8d6862016-05-26 11:24:55 -0700167 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
stefan4fbd1452015-09-28 03:57:14 -0700168 remb = false;
stefan43edf0f2015-11-20 18:05:48 -0800169 transport_cc = true;
stefan4fbd1452015-09-28 03:57:14 -0700170 send_config->rtp.extensions.push_back(RtpExtension(
171 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000172 } else {
stefan4fbd1452015-09-28 03:57:14 -0700173 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800174 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700175 send_config->rtp.extensions.push_back(RtpExtension(
176 extension_type_.c_str(), kTransmissionTimeOffsetExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000177 }
stefan4fbd1452015-09-28 03:57:14 -0700178
179 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
stefanff483612015-12-21 03:14:00 -0800180 send_config->rtp.ssrcs = video_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700181 if (rtx_) {
182 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
stefanff483612015-12-21 03:14:00 -0800183 send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700184 }
185 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700186 send_config->rtp.ulpfec.ulpfec_payload_type =
stefan4fbd1452015-09-28 03:57:14 -0700187 test::CallTest::kUlpfecPayloadType;
brandtrb5f2c3f2016-10-04 23:28:39 -0700188 send_config->rtp.ulpfec.red_payload_type = test::CallTest::kRedPayloadType;
brandtrfbfb5362016-11-17 04:18:37 -0800189 if (rtx_) {
190 send_config->rtp.ulpfec.red_rtx_payload_type =
191 test::CallTest::kRtxRedPayloadType;
192 }
stefan4fbd1452015-09-28 03:57:14 -0700193 }
194
195 size_t i = 0;
196 for (VideoReceiveStream::Config& recv_config : *receive_configs) {
197 recv_config.rtp.remb = remb;
stefan43edf0f2015-11-20 18:05:48 -0800198 recv_config.rtp.transport_cc = transport_cc;
stefan4fbd1452015-09-28 03:57:14 -0700199 recv_config.rtp.extensions = send_config->rtp.extensions;
200
stefanff483612015-12-21 03:14:00 -0800201 recv_config.rtp.remote_ssrc = video_ssrcs_[i];
stefan4fbd1452015-09-28 03:57:14 -0700202 recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms;
203
204 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700205 recv_config.rtp.ulpfec.red_payload_type =
206 send_config->rtp.ulpfec.red_payload_type;
207 recv_config.rtp.ulpfec.ulpfec_payload_type =
208 send_config->rtp.ulpfec.ulpfec_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800209 if (rtx_) {
nisseca5706d2017-09-11 02:32:16 -0700210 recv_config.rtp.rtx_associated_payload_types
211 [send_config->rtp.ulpfec.red_rtx_payload_type] =
212 send_config->rtp.ulpfec.red_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800213 }
stefan4fbd1452015-09-28 03:57:14 -0700214 }
215
216 if (rtx_) {
brandtr14742122017-01-27 04:53:07 -0800217 recv_config.rtp.rtx_ssrc = video_rtx_ssrcs_[i];
218 recv_config.rtp
nisse26e3abb2017-08-25 04:44:25 -0700219 .rtx_associated_payload_types[send_config->rtp.rtx.payload_type] =
220 send_config->encoder_settings.payload_type;
stefan4fbd1452015-09-28 03:57:14 -0700221 }
222 ++i;
223 }
philipel5ef2bc12017-02-21 07:28:31 -0800224
225 RTC_DCHECK_LE(num_flexfec_streams_, 1);
226 if (num_flexfec_streams_ == 1) {
227 send_config->rtp.flexfec.payload_type = test::CallTest::kFlexfecPayloadType;
228 send_config->rtp.flexfec.ssrc = test::CallTest::kFlexfecSendSsrc;
229 send_config->rtp.flexfec.protected_media_ssrcs = {video_ssrcs_[0]};
230 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000231}
232
Stefan Holmerff2a6352016-01-14 10:00:21 +0100233void RampUpTester::ModifyAudioConfigs(
234 AudioSendStream::Config* send_config,
235 std::vector<AudioReceiveStream::Config>* receive_configs) {
236 if (num_audio_streams_ == 0)
237 return;
238
isheriff6f8d6862016-05-26 11:24:55 -0700239 EXPECT_NE(RtpExtension::kTimestampOffsetUri, extension_type_)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100240 << "Audio BWE not supported with toffset.";
philipel5ef2bc12017-02-21 07:28:31 -0800241 EXPECT_NE(RtpExtension::kAbsSendTimeUri, extension_type_)
242 << "Audio BWE not supported with abs-send-time.";
Stefan Holmerff2a6352016-01-14 10:00:21 +0100243
244 send_config->rtp.ssrc = audio_ssrcs_[0];
245 send_config->rtp.extensions.clear();
246
minyue10cbb462016-11-07 09:29:22 -0800247 send_config->min_bitrate_bps = 6000;
248 send_config->max_bitrate_bps = 60000;
mflodman86cc6ff2016-07-26 04:44:06 -0700249
Stefan Holmerff2a6352016-01-14 10:00:21 +0100250 bool transport_cc = false;
philipel5ef2bc12017-02-21 07:28:31 -0800251 if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100252 transport_cc = true;
253 send_config->rtp.extensions.push_back(RtpExtension(
254 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
255 }
256
257 for (AudioReceiveStream::Config& recv_config : *receive_configs) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100258 recv_config.rtp.transport_cc = transport_cc;
259 recv_config.rtp.extensions = send_config->rtp.extensions;
260 recv_config.rtp.remote_ssrc = send_config->rtp.ssrc;
261 }
262}
263
philipel5ef2bc12017-02-21 07:28:31 -0800264void RampUpTester::ModifyFlexfecConfigs(
265 std::vector<FlexfecReceiveStream::Config>* receive_configs) {
266 if (num_flexfec_streams_ == 0)
267 return;
268 RTC_DCHECK_EQ(1, num_flexfec_streams_);
269 (*receive_configs)[0].payload_type = test::CallTest::kFlexfecPayloadType;
270 (*receive_configs)[0].remote_ssrc = test::CallTest::kFlexfecSendSsrc;
271 (*receive_configs)[0].protected_media_ssrcs = {video_ssrcs_[0]};
272 (*receive_configs)[0].local_ssrc = video_ssrcs_[0];
273 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
274 (*receive_configs)[0].transport_cc = false;
275 (*receive_configs)[0].rtp_header_extensions.push_back(
276 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
277 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
278 (*receive_configs)[0].transport_cc = true;
279 (*receive_configs)[0].rtp_header_extensions.push_back(RtpExtension(
280 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
281 }
282}
283
stefan4fbd1452015-09-28 03:57:14 -0700284void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) {
285 sender_call_ = sender_call;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000286}
287
tommi0f8b4032017-02-22 11:22:05 -0800288void RampUpTester::BitrateStatsPollingThread(void* obj) {
289 static_cast<RampUpTester*>(obj)->PollStats();
pbos@webrtc.org32452b22014-10-22 12:15:24 +0000290}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000291
tommi0f8b4032017-02-22 11:22:05 -0800292void RampUpTester::PollStats() {
293 do {
294 if (sender_call_) {
295 Call::Stats stats = sender_call_->GetStats();
stefan4fbd1452015-09-28 03:57:14 -0700296
tommi0f8b4032017-02-22 11:22:05 -0800297 EXPECT_GE(stats.send_bandwidth_bps, start_bitrate_bps_);
298 EXPECT_GE(expected_bitrate_bps_, 0);
299 if (stats.send_bandwidth_bps >= expected_bitrate_bps_ &&
300 (min_run_time_ms_ == -1 ||
301 clock_->TimeInMilliseconds() - test_start_ms_ >= min_run_time_ms_)) {
302 ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
303 observation_complete_.Set();
304 }
stefan4fbd1452015-09-28 03:57:14 -0700305 }
tommi0f8b4032017-02-22 11:22:05 -0800306 } while (!stop_event_.Wait(kPollIntervalMs));
Erik Språng468e62a2015-07-06 10:50:47 +0200307}
308
stefan4fbd1452015-09-28 03:57:14 -0700309void RampUpTester::ReportResult(const std::string& measurement,
310 size_t value,
311 const std::string& units) const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000312 webrtc::test::PrintResult(
313 measurement, "",
stefanff483612015-12-21 03:14:00 -0800314 ::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
315 units, false);
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000316}
317
stefan092508a2015-09-29 02:26:42 -0700318void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
319 size_t* total_packets_sent,
320 size_t* total_sent,
321 size_t* padding_sent,
322 size_t* media_sent) const {
stefan4fbd1452015-09-28 03:57:14 -0700323 *total_packets_sent += stream.rtp_stats.transmitted.packets +
324 stream.rtp_stats.retransmitted.packets +
325 stream.rtp_stats.fec.packets;
326 *total_sent += stream.rtp_stats.transmitted.TotalBytes() +
327 stream.rtp_stats.retransmitted.TotalBytes() +
328 stream.rtp_stats.fec.TotalBytes();
329 *padding_sent += stream.rtp_stats.transmitted.padding_bytes +
330 stream.rtp_stats.retransmitted.padding_bytes +
331 stream.rtp_stats.fec.padding_bytes;
332 *media_sent += stream.rtp_stats.MediaPayloadBytes();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000333}
334
stefan4fbd1452015-09-28 03:57:14 -0700335void RampUpTester::TriggerTestDone() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200336 RTC_DCHECK_GE(test_start_ms_, 0);
337
Stefan Holmerff2a6352016-01-14 10:00:21 +0100338 // TODO(holmer): Add audio send stats here too when those APIs are available.
mflodman86cc6ff2016-07-26 04:44:06 -0700339 if (!send_stream_)
340 return;
341
stefan4fbd1452015-09-28 03:57:14 -0700342 VideoSendStream::Stats send_stats = send_stream_->GetStats();
343
344 size_t total_packets_sent = 0;
345 size_t total_sent = 0;
346 size_t padding_sent = 0;
347 size_t media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800348 for (uint32_t ssrc : video_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700349 AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent,
350 &total_sent, &padding_sent, &media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700351 }
352
353 size_t rtx_total_packets_sent = 0;
354 size_t rtx_total_sent = 0;
355 size_t rtx_padding_sent = 0;
356 size_t rtx_media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800357 for (uint32_t rtx_ssrc : video_rtx_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700358 AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent,
359 &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700360 }
361
stefan5a2c5062017-01-27 06:43:18 -0800362 if (report_perf_stats_) {
stefan5a2c5062017-01-27 06:43:18 -0800363 ReportResult("ramp-up-media-sent", media_sent, "bytes");
364 ReportResult("ramp-up-padding-sent", padding_sent, "bytes");
stefan5a2c5062017-01-27 06:43:18 -0800365 ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes");
366 ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes");
367 if (ramp_up_finished_ms_ >= 0) {
368 ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
369 "milliseconds");
370 }
371 ReportResult("ramp-up-average-network-latency",
372 send_transport_->GetAverageDelayMs(), "milliseconds");
stefan4fbd1452015-09-28 03:57:14 -0700373 }
374}
375
376void RampUpTester::PerformTest() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200377 test_start_ms_ = clock_->TimeInMilliseconds();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100378 poller_thread_.Start();
Peter Boström5811a392015-12-10 13:02:50 +0100379 EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete.";
stefan4fbd1452015-09-28 03:57:14 -0700380 TriggerTestDone();
tommi0f8b4032017-02-22 11:22:05 -0800381 stop_event_.Set();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100382 poller_thread_.Stop();
stefan4fbd1452015-09-28 03:57:14 -0700383}
384
Stefan Holmerff2a6352016-01-14 10:00:21 +0100385RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams,
386 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800387 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -0700388 unsigned int start_bitrate_bps,
389 const std::string& extension_type,
390 bool rtx,
philipel5ef2bc12017-02-21 07:28:31 -0800391 bool red,
stefan45b5fe52017-03-09 06:27:02 -0800392 const std::vector<int>& loss_rates,
393 bool report_perf_stats)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100394 : RampUpTester(num_video_streams,
395 num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800396 num_flexfec_streams,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100397 start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -0800398 0,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100399 extension_type,
400 rtx,
stefan5a2c5062017-01-27 06:43:18 -0800401 red,
stefan45b5fe52017-03-09 06:27:02 -0800402 report_perf_stats),
stefan76d9c9c2017-04-01 06:51:09 -0700403 link_rates_({4 * GetExpectedHighBitrate() / (3 * 1000),
404 kLowBandwidthLimitBps / 1000,
405 4 * GetExpectedHighBitrate() / (3 * 1000), 0}),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000406 test_state_(kFirstRampup),
philipel5ef2bc12017-02-21 07:28:31 -0800407 next_state_(kTransitionToNextState),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000408 state_start_ms_(clock_->TimeInMilliseconds()),
stefan4fbd1452015-09-28 03:57:14 -0700409 interval_start_ms_(clock_->TimeInMilliseconds()),
philipel5ef2bc12017-02-21 07:28:31 -0800410 sent_bytes_(0),
411 loss_rates_(loss_rates) {
412 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
413 forward_transport_config_.queue_delay_ms = 100;
414 forward_transport_config_.loss_percent = loss_rates_[test_state_];
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000415}
416
stefan4fbd1452015-09-28 03:57:14 -0700417RampUpDownUpTester::~RampUpDownUpTester() {}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000418
tommi0f8b4032017-02-22 11:22:05 -0800419void RampUpDownUpTester::PollStats() {
420 do {
421 if (send_stream_) {
422 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
423 int transmit_bitrate_bps = 0;
424 for (auto it : stats.substreams) {
425 transmit_bitrate_bps += it.second.total_bitrate_bps;
426 }
427 EvolveTestState(transmit_bitrate_bps, stats.suspended);
428 } else if (num_audio_streams_ > 0 && sender_call_ != nullptr) {
429 // An audio send stream doesn't have bitrate stats, so the call send BW is
430 // currently used instead.
431 int transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps;
432 EvolveTestState(transmit_bitrate_bps, false);
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +0000433 }
tommi0f8b4032017-02-22 11:22:05 -0800434 } while (!stop_event_.Wait(kPollIntervalMs));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000435}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000436
stefan4fbd1452015-09-28 03:57:14 -0700437Call::Config RampUpDownUpTester::GetReceiverCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -0700438 Call::Config config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -0700439 config.bitrate_config.min_bitrate_bps = 10000;
440 return config;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000441}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000442
stefan4fbd1452015-09-28 03:57:14 -0700443std::string RampUpDownUpTester::GetModifierString() const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000444 std::string str("_");
stefanff483612015-12-21 03:14:00 -0800445 if (num_video_streams_ > 0) {
446 std::ostringstream s;
447 s << num_video_streams_;
448 str += s.str();
449 str += "stream";
450 str += (num_video_streams_ > 1 ? "s" : "");
451 str += "_";
452 }
453 if (num_audio_streams_ > 0) {
454 std::ostringstream s;
455 s << num_audio_streams_;
456 str += s.str();
457 str += "stream";
458 str += (num_audio_streams_ > 1 ? "s" : "");
459 str += "_";
460 }
stefan4fbd1452015-09-28 03:57:14 -0700461 str += (rtx_ ? "" : "no");
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000462 str += "rtx";
463 return str;
464}
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000465
stefandb752f92016-12-05 08:23:40 -0800466int RampUpDownUpTester::GetExpectedHighBitrate() const {
stefan38d8b3c2017-01-09 04:19:24 -0800467 int expected_bitrate_bps = 0;
468 if (num_audio_streams_ > 0)
469 expected_bitrate_bps += kExpectedHighAudioBitrateBps;
470 if (num_video_streams_ > 0)
471 expected_bitrate_bps += kExpectedHighVideoBitrateBps;
472 return expected_bitrate_bps;
473}
474
philipel5ef2bc12017-02-21 07:28:31 -0800475size_t RampUpDownUpTester::GetFecBytes() const {
476 size_t flex_fec_bytes = 0;
477 if (num_flexfec_streams_ > 0) {
478 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
479 for (const auto& kv : stats.substreams)
480 flex_fec_bytes += kv.second.rtp_stats.fec.TotalBytes();
481 }
482 return flex_fec_bytes;
483}
484
485bool RampUpDownUpTester::ExpectingFec() const {
486 return num_flexfec_streams_ > 0 && forward_transport_config_.loss_percent > 0;
487}
488
stefan4fbd1452015-09-28 03:57:14 -0700489void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000490 int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000491 switch (test_state_) {
philipel5ef2bc12017-02-21 07:28:31 -0800492 case kFirstRampup:
stefan4fbd1452015-09-28 03:57:14 -0700493 EXPECT_FALSE(suspended);
stefandb752f92016-12-05 08:23:40 -0800494 if (bitrate_bps >= GetExpectedHighBitrate()) {
stefan45b5fe52017-03-09 06:27:02 -0800495 if (report_perf_stats_) {
496 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
497 "first_rampup", now - state_start_ms_, "ms",
498 false);
499 }
philipel5ef2bc12017-02-21 07:28:31 -0800500 // Apply loss during the transition between states if FEC is enabled.
501 forward_transport_config_.loss_percent = loss_rates_[test_state_];
502 test_state_ = kTransitionToNextState;
503 next_state_ = kLowRate;
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000504 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000505 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000506 case kLowRate: {
mflodman86cc6ff2016-07-26 04:44:06 -0700507 // Audio streams are never suspended.
508 bool check_suspend_state = num_video_streams_ > 0;
509 if (bitrate_bps < kExpectedLowBitrateBps &&
510 suspended == check_suspend_state) {
stefan45b5fe52017-03-09 06:27:02 -0800511 if (report_perf_stats_) {
512 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
513 "rampdown", now - state_start_ms_, "ms",
514 false);
515 }
philipel5ef2bc12017-02-21 07:28:31 -0800516 // Apply loss during the transition between states if FEC is enabled.
517 forward_transport_config_.loss_percent = loss_rates_[test_state_];
518 test_state_ = kTransitionToNextState;
519 next_state_ = kSecondRampup;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000520 }
521 break;
522 }
philipel5ef2bc12017-02-21 07:28:31 -0800523 case kSecondRampup:
stefandb752f92016-12-05 08:23:40 -0800524 if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) {
stefan45b5fe52017-03-09 06:27:02 -0800525 if (report_perf_stats_) {
526 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
527 "second_rampup", now - state_start_ms_,
528 "ms", false);
529 ReportResult("ramp-up-down-up-average-network-latency",
530 send_transport_->GetAverageDelayMs(), "milliseconds");
531 }
philipel5ef2bc12017-02-21 07:28:31 -0800532 // Apply loss during the transition between states if FEC is enabled.
533 forward_transport_config_.loss_percent = loss_rates_[test_state_];
534 test_state_ = kTransitionToNextState;
535 next_state_ = kTestEnd;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000536 }
537 break;
philipel5ef2bc12017-02-21 07:28:31 -0800538 case kTestEnd:
539 observation_complete_.Set();
540 break;
541 case kTransitionToNextState:
542 if (!ExpectingFec() || GetFecBytes() > 0) {
543 test_state_ = next_state_;
544 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
545 // No loss while ramping up and down as it may affect the BWE
546 // negatively, making the test flaky.
547 forward_transport_config_.loss_percent = 0;
548 state_start_ms_ = now;
549 interval_start_ms_ = now;
550 sent_bytes_ = 0;
551 send_transport_->SetConfig(forward_transport_config_);
552 }
553 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000554 }
555}
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000556
stefan4fbd1452015-09-28 03:57:14 -0700557class RampUpTest : public test::CallTest {
Erik Språng6b8d3552015-09-24 15:06:57 +0200558 public:
stefan4fbd1452015-09-28 03:57:14 -0700559 RampUpTest() {}
Erik Språng6b8d3552015-09-24 15:06:57 +0200560
stefan4fbd1452015-09-28 03:57:14 -0700561 virtual ~RampUpTest() {
stefanff483612015-12-21 03:14:00 -0800562 EXPECT_EQ(nullptr, video_send_stream_);
563 EXPECT_TRUE(video_receive_streams_.empty());
Erik Språng6b8d3552015-09-24 15:06:57 +0200564 }
Erik Språng6b8d3552015-09-24 15:06:57 +0200565};
566
Stefan Holmerff2a6352016-01-14 10:00:21 +0100567static const uint32_t kStartBitrateBps = 60000;
568
stefan38d8b3c2017-01-09 04:19:24 -0800569TEST_F(RampUpTest, UpDownUpAbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800570 std::vector<int> loss_rates = {0, 0, 0, 0};
571 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
stefan45b5fe52017-03-09 06:27:02 -0800572 RtpExtension::kAbsSendTimeUri, true, true, loss_rates,
573 true);
stefane74eef12016-01-08 06:47:13 -0800574 RunBaseTest(&test);
Shao Changbine62202f2015-04-21 20:24:50 +0800575}
Stefan Holmerff2a6352016-01-14 10:00:21 +0100576
stefan38d8b3c2017-01-09 04:19:24 -0800577TEST_F(RampUpTest, UpDownUpTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800578 std::vector<int> loss_rates = {0, 0, 0, 0};
579 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700580 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800581 false, loss_rates, true);
philipel5ef2bc12017-02-21 07:28:31 -0800582 RunBaseTest(&test);
583}
584
stefan45b5fe52017-03-09 06:27:02 -0800585// TODO(holmer): Tests which don't report perf stats should be moved to a
586// different executable since they per definition are not perf tests.
philipel5ef2bc12017-02-21 07:28:31 -0800587TEST_F(RampUpTest, UpDownUpTransportSequenceNumberPacketLoss) {
588 std::vector<int> loss_rates = {20, 0, 0, 0};
589 RampUpDownUpTester test(1, 0, 1, kStartBitrateBps,
590 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800591 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100592 RunBaseTest(&test);
593}
594
stefan38d8b3c2017-01-09 04:19:24 -0800595TEST_F(RampUpTest, UpDownUpAudioVideoTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800596 std::vector<int> loss_rates = {0, 0, 0, 0};
597 RampUpDownUpTester test(3, 1, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700598 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700599 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100600 RunBaseTest(&test);
601}
602
stefan38d8b3c2017-01-09 04:19:24 -0800603TEST_F(RampUpTest, UpDownUpAudioTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800604 std::vector<int> loss_rates = {0, 0, 0, 0};
605 RampUpDownUpTester test(0, 1, 0, kStartBitrateBps,
mflodman86cc6ff2016-07-26 04:44:06 -0700606 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700607 false, loss_rates, false);
mflodman86cc6ff2016-07-26 04:44:06 -0700608 RunBaseTest(&test);
609}
610
stefan38d8b3c2017-01-09 04:19:24 -0800611TEST_F(RampUpTest, TOffsetSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800612 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTimestampOffsetUri, true,
613 true, true);
stefan38d8b3c2017-01-09 04:19:24 -0800614 RunBaseTest(&test);
615}
616
617TEST_F(RampUpTest, AbsSendTime) {
philipel5ef2bc12017-02-21 07:28:31 -0800618 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, false, false,
stefanff2ebf52017-03-13 01:27:03 -0700619 false);
stefane74eef12016-01-08 06:47:13 -0800620 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000621}
622
stefan38d8b3c2017-01-09 04:19:24 -0800623TEST_F(RampUpTest, AbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800624 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, true, true,
stefan5a2c5062017-01-27 06:43:18 -0800625 true);
stefane74eef12016-01-08 06:47:13 -0800626 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000627}
628
stefan38d8b3c2017-01-09 04:19:24 -0800629TEST_F(RampUpTest, TransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800630 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700631 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800632 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200633}
634
635TEST_F(RampUpTest, TransportSequenceNumberSimulcast) {
philipel5ef2bc12017-02-21 07:28:31 -0800636 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700637 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800638 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200639}
640
stefan38d8b3c2017-01-09 04:19:24 -0800641TEST_F(RampUpTest, TransportSequenceNumberSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800642 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefan5a2c5062017-01-27 06:43:18 -0800643 true, true, true);
644 RunBaseTest(&test);
645}
646
647TEST_F(RampUpTest, AudioTransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800648 RampUpTester test(0, 1, 0, 300000, 10000,
stefan5a2c5062017-01-27 06:43:18 -0800649 RtpExtension::kTransportSequenceNumberUri, false, false,
650 false);
stefane74eef12016-01-08 06:47:13 -0800651 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200652}
pbos@webrtc.org744fbc72013-09-10 09:26:25 +0000653} // namespace webrtc