A new PeerConnection level perf test.
This test creates a one way audio and video call, allows for bandwidth
estimation to ramp up and then runs the call for 10 seconds. The
average bandwidth estimate over this time is recorded as a perf metric.
This is done at the PeerConnection level with the intention to catch
regressions related to ICE configurations. Stats are taken from
PeerConnection for BWE, and the network simulation is done with a
VirtualSocketServer.
Bug: webrtc:7668
Change-Id: Ib8a449da80fc74be1e505ac34c0c6b7479cb58db
Reviewed-on: https://webrtc-review.googlesource.com/78361
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Taylor Brandstetter <deadbeef@webrtc.org>
Commit-Queue: Seth Hampson <shampson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23758}diff --git a/BUILD.gn b/BUILD.gn
index 7566f2a..8cedb5b 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -546,13 +546,13 @@
rtc_test("webrtc_perf_tests") {
testonly = true
configs += [ ":rtc_unittests_config" ]
-
deps = [
"audio:audio_perf_tests",
"call:call_perf_tests",
"modules/audio_coding:audio_coding_perf_tests",
"modules/audio_processing:audio_processing_perf_tests",
"modules/remote_bitrate_estimator:remote_bitrate_estimator_perf_tests",
+ "pc:peerconnection_perf_tests",
"test:test_main",
"video:video_full_stack_tests",
]
diff --git a/pc/BUILD.gn b/pc/BUILD.gn
index a5703b7..9d2a05a 100644
--- a/pc/BUILD.gn
+++ b/pc/BUILD.gn
@@ -344,6 +344,39 @@
}
}
+ rtc_source_set("peerconnection_perf_tests") {
+ testonly = true
+ sources = [
+ "peerconnection_rampup_tests.cc",
+ "peerconnectionwrapper.cc",
+ "peerconnectionwrapper.h",
+ ]
+ deps = [
+ ":pc_test_utils",
+ "../api:fakemetricsobserver",
+ "../api:libjingle_peerconnection_api",
+ "../api:libjingle_peerconnection_test_api",
+ "../api:rtc_stats_api",
+ "../api/audio_codecs:builtin_audio_decoder_factory",
+ "../api/audio_codecs:builtin_audio_encoder_factory",
+ "../api/video_codecs:builtin_video_decoder_factory",
+ "../api/video_codecs:builtin_video_encoder_factory",
+ "../media:rtc_media_tests_utils",
+ "../p2p:p2p_test_utils",
+ "../p2p:rtc_p2p",
+ "../pc:peerconnection",
+ "../rtc_base:rtc_base",
+ "../rtc_base:rtc_base_approved",
+ "../rtc_base:rtc_base_tests_utils",
+ "../test:perf_test",
+ "../test:test_support",
+ ]
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+ }
+
rtc_source_set("pc_test_utils") {
testonly = true
sources = [
@@ -358,6 +391,7 @@
"test/fakesctptransport.h",
"test/fakevideotrackrenderer.h",
"test/fakevideotracksource.h",
+ "test/framegeneratorcapturervideotracksource.h",
"test/mock_datachannel.h",
"test/mock_peerconnection.h",
"test/mock_rtpreceiverinternal.h",
@@ -395,6 +429,7 @@
"../rtc_base:rtc_base_tests_utils",
"../rtc_base:rtc_task_queue",
"../test:test_support",
+ "../test:video_test_common",
]
if (!build_with_chromium && is_clang) {
diff --git a/pc/peerconnection_rampup_tests.cc b/pc/peerconnection_rampup_tests.cc
new file mode 100644
index 0000000..74cbb0f
--- /dev/null
+++ b/pc/peerconnection_rampup_tests.cc
@@ -0,0 +1,318 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/fakeconstraints.h"
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "p2p/base/testturnserver.h"
+#include "p2p/client/basicportallocator.h"
+#include "pc/peerconnection.h"
+#include "pc/peerconnectionwrapper.h"
+#include "pc/test/fakeaudiocapturemodule.h"
+#include "pc/test/fakeperiodicvideotracksource.h"
+#include "pc/test/fakevideotrackrenderer.h"
+#include "pc/test/framegeneratorcapturervideotracksource.h"
+#include "rtc_base/fakenetwork.h"
+#include "rtc_base/firewallsocketserver.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/platform_thread.h"
+#include "rtc_base/socketaddress.h"
+#include "rtc_base/virtualsocketserver.h"
+#include "test/gtest.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+
+namespace {
+static const int kDefaultTestTimeMs = 15000;
+static const int kRampUpTimeMs = 5000;
+static const int kPollIntervalTimeMs = 50;
+static const int kDefaultTimeoutMs = 10000;
+static const rtc::SocketAddress kDefaultLocalAddress("1.1.1.1", 0);
+// The video's configured max bitrate in webrtcvideoengine.cc is 1.7 Mbps.
+// Setting the network bandwidth to 1 Mbps allows the video's bitrate to push
+// the network's limitations.
+static const int kNetworkBandwidth = 1000000;
+} // namespace
+
+using RTCConfiguration = PeerConnectionInterface::RTCConfiguration;
+
+// This is an end to end test to verify that BWE is functioning when setting
+// up a one to one call at the PeerConnection level. The intention of the test
+// is to catch potential regressions for different ICE path configurations. The
+// test uses a VirtualSocketServer for it's underlying simulated network and
+// fake audio and video sources. The test is based upon rampup_tests.cc, but
+// instead is at the PeerConnection level and uses a different fake network
+// (rampup_tests.cc uses SimulatedNetwork). In the future, this test could
+// potentially test different network conditions and test video quality as well
+// (video_quality_test.cc does this, but at the call level).
+//
+// The perf test results are printed using the perf test support. If the
+// isolated_script_test_perf_output flag is specified in test_main.cc, then
+// the results are written to a JSON formatted file for the Chrome perf
+// dashboard. Since this test is a webrtc_perf_test, it will be run in the perf
+// console every webrtc commit.
+class PeerConnectionWrapperForRampUpTest : public PeerConnectionWrapper {
+ public:
+ using PeerConnectionWrapper::PeerConnectionWrapper;
+
+ PeerConnectionWrapperForRampUpTest(
+ rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory,
+ rtc::scoped_refptr<PeerConnectionInterface> pc,
+ std::unique_ptr<MockPeerConnectionObserver> observer,
+ rtc::FakeNetworkManager* fake_network_manager)
+ : PeerConnectionWrapper::PeerConnectionWrapper(pc_factory,
+ pc,
+ std::move(observer)),
+ fake_network_manager_(std::move(fake_network_manager)) {}
+
+ bool AddIceCandidates(std::vector<const IceCandidateInterface*> candidates) {
+ bool success = true;
+ for (const auto candidate : candidates) {
+ if (!pc()->AddIceCandidate(candidate)) {
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ rtc::scoped_refptr<VideoTrackInterface> CreateLocalVideoTrack(
+ FrameGeneratorCapturerVideoTrackSource::Config config,
+ Clock* clock) {
+ video_track_sources_.emplace_back(
+ new rtc::RefCountedObject<FrameGeneratorCapturerVideoTrackSource>(
+ config, clock));
+ video_track_sources_.back()->Start();
+ return rtc::scoped_refptr<VideoTrackInterface>(
+ pc_factory()->CreateVideoTrack(rtc::CreateRandomUuid(),
+ video_track_sources_.back()));
+ }
+
+ rtc::scoped_refptr<AudioTrackInterface> CreateLocalAudioTrack(
+ const cricket::AudioOptions options) {
+ rtc::scoped_refptr<AudioSourceInterface> source =
+ pc_factory()->CreateAudioSource(options);
+ return pc_factory()->CreateAudioTrack(rtc::CreateRandomUuid(), source);
+ }
+
+ private:
+ // This is owned by the Test, not the Wrapper. It needs to outlive the
+ // Wrapper, because the port allocator expects its lifetime to be longer than
+ // the PeerConnection's lifetime.
+ rtc::FakeNetworkManager* fake_network_manager_;
+ std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+ video_track_sources_;
+};
+
+// TODO(shampson): Paramaterize the test to run for both Plan B & Unified Plan.
+class PeerConnectionRampUpTest : public ::testing::Test {
+ public:
+ PeerConnectionRampUpTest()
+ : clock_(Clock::GetRealTimeClock()),
+ virtual_socket_server_(new rtc::VirtualSocketServer()),
+ firewall_socket_server_(
+ new rtc::FirewallSocketServer(virtual_socket_server_.get())),
+ network_thread_(new rtc::Thread(firewall_socket_server_.get())),
+ worker_thread_(rtc::Thread::Create()) {
+ network_thread_->SetName("PCNetworkThread", this);
+ worker_thread_->SetName("PCWorkerThread", this);
+ RTC_CHECK(network_thread_->Start());
+ RTC_CHECK(worker_thread_->Start());
+
+ virtual_socket_server_->set_bandwidth(kNetworkBandwidth / 8);
+ pc_factory_ = CreatePeerConnectionFactory(
+ network_thread_.get(), worker_thread_.get(), rtc::Thread::Current(),
+ rtc::scoped_refptr<AudioDeviceModule>(FakeAudioCaptureModule::Create()),
+ CreateBuiltinAudioEncoderFactory(), CreateBuiltinAudioDecoderFactory(),
+ CreateBuiltinVideoEncoderFactory(), CreateBuiltinVideoDecoderFactory(),
+ nullptr /* audio_mixer */, nullptr /* audio_processing */);
+ }
+
+ virtual ~PeerConnectionRampUpTest() {
+ network_thread()->Invoke<void>(RTC_FROM_HERE,
+ [this] { turn_servers_.clear(); });
+ }
+
+ bool CreatePeerConnectionWrappers(const RTCConfiguration& caller_config,
+ const RTCConfiguration& callee_config) {
+ caller_ = CreatePeerConnectionWrapper(caller_config);
+ callee_ = CreatePeerConnectionWrapper(callee_config);
+ return caller_ && callee_;
+ }
+
+ std::unique_ptr<PeerConnectionWrapperForRampUpTest>
+ CreatePeerConnectionWrapper(const RTCConfiguration& config) {
+ auto* fake_network_manager = new rtc::FakeNetworkManager();
+ fake_network_manager->AddInterface(kDefaultLocalAddress);
+ fake_network_managers_.emplace_back(fake_network_manager);
+ auto port_allocator =
+ rtc::MakeUnique<cricket::BasicPortAllocator>(fake_network_manager);
+
+ port_allocator->set_step_delay(cricket::kDefaultStepDelay);
+ auto observer = rtc::MakeUnique<MockPeerConnectionObserver>();
+ auto pc = pc_factory_->CreatePeerConnection(
+ config, std::move(port_allocator), nullptr, observer.get());
+ if (!pc) {
+ return nullptr;
+ }
+
+ return rtc::MakeUnique<PeerConnectionWrapperForRampUpTest>(
+ pc_factory_, pc, std::move(observer), fake_network_manager);
+ }
+
+ void SetupOneWayCall() {
+ ASSERT_TRUE(caller_);
+ ASSERT_TRUE(callee_);
+ FrameGeneratorCapturerVideoTrackSource::Config config;
+ caller_->AddTrack(caller_->CreateLocalVideoTrack(config, clock_));
+ // Disable highpass filter so that we can get all the test audio frames.
+ cricket::AudioOptions options;
+ options.highpass_filter = false;
+ caller_->AddTrack(caller_->CreateLocalAudioTrack(options));
+
+ // Do the SDP negotiation, and also exchange ice candidates.
+ ASSERT_TRUE(caller_->ExchangeOfferAnswerWith(callee_.get()));
+ ASSERT_TRUE_WAIT(
+ caller_->signaling_state() == PeerConnectionInterface::kStable,
+ kDefaultTimeoutMs);
+ ASSERT_TRUE_WAIT(caller_->IsIceGatheringDone(), kDefaultTimeoutMs);
+ ASSERT_TRUE_WAIT(callee_->IsIceGatheringDone(), kDefaultTimeoutMs);
+
+ // Connect an ICE candidate pairs.
+ ASSERT_TRUE(
+ callee_->AddIceCandidates(caller_->observer()->GetAllCandidates()));
+ ASSERT_TRUE(
+ caller_->AddIceCandidates(callee_->observer()->GetAllCandidates()));
+ // This means that ICE and DTLS are connected.
+ ASSERT_TRUE_WAIT(callee_->IsIceConnected(), kDefaultTimeoutMs);
+ ASSERT_TRUE_WAIT(caller_->IsIceConnected(), kDefaultTimeoutMs);
+ }
+
+ void CreateTurnServer(cricket::ProtocolType type) {
+ rtc::Thread* thread = network_thread();
+ std::unique_ptr<cricket::TestTurnServer> turn_server =
+ network_thread_->Invoke<std::unique_ptr<cricket::TestTurnServer>>(
+ RTC_FROM_HERE, [thread, type] {
+ static const rtc::SocketAddress turn_server_internal_address{
+ "88.88.88.0", 3478};
+ static const rtc::SocketAddress turn_server_external_address{
+ "88.88.88.1", 0};
+ return rtc::MakeUnique<cricket::TestTurnServer>(
+ thread, turn_server_internal_address,
+ turn_server_external_address, type);
+ });
+ turn_servers_.push_back(std::move(turn_server));
+ }
+
+ // First runs the call for kRampUpTimeMs to ramp up the bandwidth estimate.
+ // Then runs the test for the remaining test time, grabbing the bandwidth
+ // estimation stat, every kPollIntervalTimeMs. When finished, averages the
+ // bandwidth estimations and prints the bandwidth estimation result as a perf
+ // metric.
+ void RunTest(const std::string& test_string) {
+ rtc::Thread::Current()->ProcessMessages(kRampUpTimeMs);
+ int number_of_polls =
+ (kDefaultTestTimeMs - kRampUpTimeMs) / kPollIntervalTimeMs;
+ int total_bwe = 0;
+ for (int i = 0; i < number_of_polls; ++i) {
+ rtc::Thread::Current()->ProcessMessages(kPollIntervalTimeMs);
+ total_bwe += static_cast<int>(GetCallerAvailableBitrateEstimate());
+ }
+ double average_bandwidth_estimate = total_bwe / number_of_polls;
+ std::string value_description =
+ "bwe_after_" + std::to_string(kDefaultTestTimeMs / 1000) + "_seconds";
+ test::PrintResult("peerconnection_ramp_up_", test_string, value_description,
+ average_bandwidth_estimate, "bwe", false);
+ }
+
+ rtc::Thread* network_thread() { return network_thread_.get(); }
+
+ PeerConnectionWrapperForRampUpTest* caller() { return caller_.get(); }
+
+ PeerConnectionWrapperForRampUpTest* callee() { return callee_.get(); }
+
+ private:
+ // Gets the caller's outgoing available bitrate from the stats. Returns 0 if
+ // something went wrong. It takes the outgoing bitrate from the current
+ // selected ICE candidate pair's stats.
+ double GetCallerAvailableBitrateEstimate() {
+ auto stats = caller_->GetStats();
+ auto transport_stats = stats->GetStatsOfType<RTCTransportStats>();
+ if (transport_stats.size() == 0u ||
+ !transport_stats[0]->selected_candidate_pair_id.is_defined()) {
+ return 0;
+ }
+ std::string selected_ice_id =
+ transport_stats[0]->selected_candidate_pair_id.ValueToString();
+ // Use the selected ICE candidate pair ID to get the appropriate ICE stats.
+ const RTCIceCandidatePairStats ice_candidate_pair_stats =
+ stats->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>();
+ if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) {
+ return *ice_candidate_pair_stats.available_outgoing_bitrate;
+ }
+ // We couldn't get the |available_outgoing_bitrate| for the active candidate
+ // pair.
+ return 0;
+ }
+
+ Clock* const clock_;
+ // The turn servers should be accessed & deleted on the network thread to
+ // avoid a race with the socket read/write which occurs on the network thread.
+ std::vector<std::unique_ptr<cricket::TestTurnServer>> turn_servers_;
+ // |virtual_socket_server_| is used by |network_thread_| so it must be
+ // destroyed later.
+ // TODO(bugs.webrtc.org/7668): We would like to update the virtual network we
+ // use for this test. VirtualSocketServer isn't ideal because:
+ // 1) It uses the same queue & network capacity for both directions.
+ // 2) VirtualSocketServer implements how the network bandwidth affects the
+ // send delay differently than the SimulatedNetwork, used by the
+ // FakeNetworkPipe. It would be ideal if all of levels of virtual
+ // networks used in testing were consistent.
+ // We would also like to update this test to record the time to ramp up,
+ // down, and back up (similar to in rampup_tests.cc). This is problematic with
+ // the VirtualSocketServer. The first ramp down time is very noisy and the
+ // second ramp up time can take up to 300 seconds, most likely due to a built
+ // up queue.
+ std::unique_ptr<rtc::VirtualSocketServer> virtual_socket_server_;
+ std::unique_ptr<rtc::FirewallSocketServer> firewall_socket_server_;
+ std::unique_ptr<rtc::Thread> network_thread_;
+ std::unique_ptr<rtc::Thread> worker_thread_;
+ // The |pc_factory| uses |network_thread_| & |worker_thread_|, so it must be
+ // destroyed first.
+ std::vector<std::unique_ptr<rtc::FakeNetworkManager>> fake_network_managers_;
+ rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory_;
+ std::unique_ptr<PeerConnectionWrapperForRampUpTest> caller_;
+ std::unique_ptr<PeerConnectionWrapperForRampUpTest> callee_;
+};
+
+TEST_F(PeerConnectionRampUpTest, TurnOverTCP) {
+ CreateTurnServer(cricket::ProtocolType::PROTO_TCP);
+ PeerConnectionInterface::IceServer ice_server;
+ ice_server.urls.push_back("turn:88.88.88.0:3478?transport=tcp");
+ ice_server.username = "test";
+ ice_server.password = "test";
+ PeerConnectionInterface::RTCConfiguration client_1_config;
+ client_1_config.servers.push_back(ice_server);
+ client_1_config.type = PeerConnectionInterface::kRelay;
+ PeerConnectionInterface::RTCConfiguration client_2_config;
+ client_2_config.servers.push_back(ice_server);
+ client_2_config.type = PeerConnectionInterface::kRelay;
+ ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config));
+
+ SetupOneWayCall();
+ RunTest("turn_over_tcp");
+}
+
+// TODO(bugs.webrtc.org/7668): Test other ICE configurations.
+
+} // namespace webrtc
diff --git a/pc/test/framegeneratorcapturervideotracksource.h b/pc/test/framegeneratorcapturervideotracksource.h
new file mode 100644
index 0000000..b406e3b
--- /dev/null
+++ b/pc/test/framegeneratorcapturervideotracksource.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_
+#define PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_
+
+#include <memory>
+
+#include "pc/videotracksource.h"
+#include "test/frame_generator_capturer.h"
+
+namespace webrtc {
+
+// Implements a VideoTrackSourceInterface to be used for creating VideoTracks.
+// The video source is generated using a FrameGeneratorCapturer, specifically
+// a SquareGenerator that generates frames with randomly sized and colored
+// squares.
+class FrameGeneratorCapturerVideoTrackSource : public VideoTrackSource {
+ public:
+ static const int kDefaultFramesPerSecond = 30;
+ static const int kDefaultWidth = 640;
+ static const int kDefaultHeight = 480;
+ static const int kNumSquaresGenerated = 50;
+
+ struct Config {
+ int frames_per_second = kDefaultFramesPerSecond;
+ int width = kDefaultWidth;
+ int height = kDefaultHeight;
+ int num_squares_generated = 50;
+ };
+
+ explicit FrameGeneratorCapturerVideoTrackSource(Clock* clock)
+ : FrameGeneratorCapturerVideoTrackSource(Config(), clock) {}
+
+ FrameGeneratorCapturerVideoTrackSource(Config config, Clock* clock)
+ : VideoTrackSource(false /* remote */) {
+ video_capturer_.reset(test::FrameGeneratorCapturer::Create(
+ config.width, config.height, rtc::nullopt, config.num_squares_generated,
+ config.frames_per_second, clock));
+ }
+
+ ~FrameGeneratorCapturerVideoTrackSource() = default;
+
+ void Start() {
+ video_capturer_->Start();
+ SetState(kLive);
+ }
+
+ void Stop() {
+ video_capturer_->Stop();
+ SetState(kMuted);
+ }
+
+ protected:
+ rtc::VideoSourceInterface<VideoFrame>* source() override {
+ return video_capturer_.get();
+ }
+
+ private:
+ std::unique_ptr<test::FrameGeneratorCapturer> video_capturer_;
+};
+
+} // namespace webrtc
+
+#endif // PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_
diff --git a/pc/test/mockpeerconnectionobservers.h b/pc/test/mockpeerconnectionobservers.h
index e864706..147b674 100644
--- a/pc/test/mockpeerconnectionobservers.h
+++ b/pc/test/mockpeerconnectionobservers.h
@@ -93,8 +93,13 @@
void OnIceConnectionChange(
PeerConnectionInterface::IceConnectionState new_state) override {
RTC_DCHECK(pc_->ice_connection_state() == new_state);
+ // When ICE is finished, the caller will get to a kIceConnectionCompleted
+ // state, because it has the ICE controlling role, while the callee
+ // will get to a kIceConnectionConnected state. This means that both ICE
+ // and DTLS are connected.
ice_connected_ =
- (new_state == PeerConnectionInterface::kIceConnectionConnected);
+ (new_state == PeerConnectionInterface::kIceConnectionConnected) ||
+ (new_state == PeerConnectionInterface::kIceConnectionCompleted);
callback_triggered_ = true;
}
void OnIceGatheringChange(
@@ -188,6 +193,14 @@
}
}
+ std::vector<const IceCandidateInterface*> GetAllCandidates() {
+ std::vector<const IceCandidateInterface*> candidates;
+ for (const auto& candidate : candidates_) {
+ candidates.push_back(candidate.get());
+ }
+ return candidates;
+ }
+
std::vector<IceCandidateInterface*> GetCandidatesByMline(int mline_index) {
std::vector<IceCandidateInterface*> candidates;
for (const auto& candidate : candidates_) {