Add baseline generation/verification to BWE test framework.
Updating resource file separately, once LGTM. Generates ~628k of files for current tests, highly compressable, once/if we need that.
BUG=
R=mflodman@webrtc.org, stefan@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/4639004
git-svn-id: http://webrtc.googlecode.com/svn/trunk@5204 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1
new file mode 100644
index 0000000..0e1c108
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1
@@ -0,0 +1 @@
+27bca982f40243ce6a491c05792f271166776eb9
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1
new file mode 100644
index 0000000..0e1c108
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1
@@ -0,0 +1 @@
+27bca982f40243ce6a491c05792f271166776eb9
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1
new file mode 100644
index 0000000..781691f
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1
@@ -0,0 +1 @@
+2437de4b814694bfae0c7d47f851e356d8302796
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1
new file mode 100644
index 0000000..781691f
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1
@@ -0,0 +1 @@
+2437de4b814694bfae0c7d47f851e356d8302796
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1
new file mode 100644
index 0000000..9a18507
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1
@@ -0,0 +1 @@
+c2d507d2d3edadab06942736a81c6ca3221e23c7
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1
new file mode 100644
index 0000000..9a18507
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1
@@ -0,0 +1 @@
+c2d507d2d3edadab06942736a81c6ca3221e23c7
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1
new file mode 100644
index 0000000..46ee7df
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1
@@ -0,0 +1 @@
+feeeed364dc66dea552b26271094ff11707a1e65
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1
new file mode 100644
index 0000000..46ee7df
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1
@@ -0,0 +1 @@
+feeeed364dc66dea552b26271094ff11707a1e65
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1
new file mode 100644
index 0000000..21cf98c
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1
@@ -0,0 +1 @@
+333ad9b7310be8d9c6a36c2b1fa63999a1098b88
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1
new file mode 100644
index 0000000..21cf98c
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1
@@ -0,0 +1 @@
+333ad9b7310be8d9c6a36c2b1fa63999a1098b88
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1
new file mode 100644
index 0000000..2fff908
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1
@@ -0,0 +1 @@
+95a356db2eb39fd1bbf1ece1cfbf673b1f893487
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1
new file mode 100644
index 0000000..2fff908
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1
@@ -0,0 +1 @@
+95a356db2eb39fd1bbf1ece1cfbf673b1f893487
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1
new file mode 100644
index 0000000..8e70aa2
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1
@@ -0,0 +1 @@
+d7814fe1106ee4d030bb5c4316765615f4b747dd
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1
new file mode 100644
index 0000000..8e70aa2
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1
@@ -0,0 +1 @@
+d7814fe1106ee4d030bb5c4316765615f4b747dd
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1
new file mode 100644
index 0000000..473aaa6
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1
@@ -0,0 +1 @@
+dba0be49e995fec988bb92d8d62b9e39b3f01bbe
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1
new file mode 100644
index 0000000..473aaa6
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1
@@ -0,0 +1 @@
+dba0be49e995fec988bb92d8d62b9e39b3f01bbe
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1
new file mode 100644
index 0000000..5d617bb
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1
@@ -0,0 +1 @@
+091869a91d200add958904c0b5a42d47b1f5f9ac
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1
new file mode 100644
index 0000000..5d617bb
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1
@@ -0,0 +1 @@
+091869a91d200add958904c0b5a42d47b1f5f9ac
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1
new file mode 100644
index 0000000..fcc16b6
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1
@@ -0,0 +1 @@
+988a157505bb3b938c813282d90cb445a5cea19b
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1
new file mode 100644
index 0000000..fcc16b6
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1
@@ -0,0 +1 @@
+988a157505bb3b938c813282d90cb445a5cea19b
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1
new file mode 100644
index 0000000..8395bcf
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1
@@ -0,0 +1 @@
+e367a4e6b3ba1b4cd6ac2e71b55cdaed3b63a739
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1
new file mode 100644
index 0000000..8395bcf
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1
@@ -0,0 +1 @@
+e367a4e6b3ba1b4cd6ac2e71b55cdaed3b63a739
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1
new file mode 100644
index 0000000..df83598
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1
@@ -0,0 +1 @@
+26e5e14a114a869a0777458520e6c27b5791863d
\ No newline at end of file
diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1
new file mode 100644
index 0000000..df83598
--- /dev/null
+++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1
@@ -0,0 +1 @@
+26e5e14a114a869a0777458520e6c27b5791863d
\ No newline at end of file
diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp
index 579ae14..53e7b31 100644
--- a/webrtc/modules/modules.gyp
+++ b/webrtc/modules/modules.gyp
@@ -179,6 +179,10 @@
'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h',
'remote_bitrate_estimator/remote_bitrate_estimators_test.cc',
'remote_bitrate_estimator/rtp_to_ntp_unittest.cc',
+ 'remote_bitrate_estimator/test/bwe_test_baselinefile.cc',
+ 'remote_bitrate_estimator/test/bwe_test_baselinefile.h',
+ 'remote_bitrate_estimator/test/bwe_test_fileutils.cc',
+ 'remote_bitrate_estimator/test/bwe_test_fileutils.h',
'remote_bitrate_estimator/test/bwe_test_framework.cc',
'remote_bitrate_estimator/test/bwe_test_framework.h',
'remote_bitrate_estimator/test/bwe_test_framework_unittest.cc',
diff --git a/webrtc/modules/modules_unittests.isolate b/webrtc/modules/modules_unittests.isolate
index ae72e75..32fc662 100644
--- a/webrtc/modules/modules_unittests.isolate
+++ b/webrtc/modules/modules_unittests.isolate
@@ -46,6 +46,30 @@
'../../resources/near16_stereo.pcm',
'../../resources/near32_stereo.pcm',
'../../resources/near8_stereo.pcm',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin',
+ '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin',
'../../resources/video_coding/frame-ethernet-ii.pcap',
'../../resources/video_coding/frame-loopback.pcap',
'../../resources/video_coding/pltype103.rtp',
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
index 3046a5b..e04328c 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
@@ -10,6 +10,7 @@
#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h"
#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h"
#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
#include "webrtc/system_wrappers/interface/clock.h"
@@ -35,15 +36,19 @@
class BweTest::TestedEstimator : public RemoteBitrateObserver {
public:
- explicit TestedEstimator(const BweTestConfig::EstimatorConfig& config)
+ TestedEstimator(const string& test_name,
+ const BweTestConfig::EstimatorConfig& config)
: debug_name_(config.debug_name),
clock_(0),
stats_(),
relative_estimator_stats_(),
- latest_estimate_kbps_(-1.0),
+ latest_estimate_bps_(-1),
estimator_(config.estimator_factory->Create(this, &clock_)),
- relative_estimator_(NULL) {
+ relative_estimator_(NULL),
+ baseline_(BaseLineFileInterface::Create(test_name + "_" + debug_name_,
+ config.update_baseline)) {
assert(estimator_.get());
+ assert(baseline_.get());
// Default RTT in RemoteRateControl is 200 ms ; 50 ms is more realistic.
estimator_->OnRttUpdate(50);
}
@@ -55,7 +60,7 @@
void EatPacket(const Packet& packet) {
BWE_TEST_LOGGING_CONTEXT(debug_name_);
- latest_estimate_kbps_ = -1.0;
+ latest_estimate_bps_ = -1;
// We're treating the send time (from previous filter) as the arrival
// time once packet reaches the estimator.
@@ -79,17 +84,22 @@
bool CheckEstimate(PacketSender::Feedback* feedback) {
assert(feedback);
BWE_TEST_LOGGING_CONTEXT(debug_name_);
- double estimated_kbps = 0.0;
- if (LatestEstimate(&estimated_kbps)) {
+ uint32_t estimated_bps = 0;
+ if (LatestEstimate(&estimated_bps)) {
+ feedback->estimated_bps = estimated_bps;
+ baseline_->Estimate(clock_.TimeInMilliseconds(), estimated_bps);
+
+ double estimated_kbps = static_cast<double>(estimated_bps) / 1000.0;
stats_.Push(estimated_kbps);
BWE_TEST_LOGGING_PLOT("Estimate", clock_.TimeInMilliseconds(),
- estimated_kbps / 100);
- double relative_estimate_kbps = 0.0;
+ estimated_kbps / 1000.0);
+ uint32_t relative_estimate_bps = 0;
if (relative_estimator_ &&
- relative_estimator_->LatestEstimate(&relative_estimate_kbps)) {
+ relative_estimator_->LatestEstimate(&relative_estimate_bps)) {
+ double relative_estimate_kbps =
+ static_cast<double>(relative_estimate_bps) / 1000.0;
relative_estimator_stats_.Push(estimated_kbps - relative_estimate_kbps);
}
- feedback->estimated_kbps = estimated_kbps;
return true;
}
return false;
@@ -105,32 +115,36 @@
}
}
+ void VerifyOrWriteBaseline() {
+ EXPECT_TRUE(baseline_->VerifyOrWrite());
+ }
+
virtual void OnReceiveBitrateChanged(const vector<unsigned int>& ssrcs,
unsigned int bitrate) {
}
private:
- bool LatestEstimate(double* estimate_kbps) {
- if (latest_estimate_kbps_ < 0.0) {
+ bool LatestEstimate(uint32_t* estimate_bps) {
+ if (latest_estimate_bps_ < 0) {
vector<unsigned int> ssrcs;
unsigned int bps = 0;
if (!estimator_->LatestEstimate(&ssrcs, &bps)) {
return false;
}
- latest_estimate_kbps_ = bps / 1000.0;
+ latest_estimate_bps_ = bps;
}
- *estimate_kbps = latest_estimate_kbps_;
+ *estimate_bps = latest_estimate_bps_;
return true;
}
string debug_name_;
- bool log_estimates_;
SimulatedClock clock_;
Stats<double> stats_;
Stats<double> relative_estimator_stats_;
- double latest_estimate_kbps_;
+ int64_t latest_estimate_bps_;
scoped_ptr<RemoteBitrateEstimator> estimator_;
TestedEstimator* relative_estimator_;
+ scoped_ptr<BaseLineFileInterface> baseline_;
DISALLOW_IMPLICIT_CONSTRUCTORS(TestedEstimator);
};
@@ -150,8 +164,11 @@
}
void BweTest::SetUp() {
- BWE_TEST_LOGGING_GLOBAL_CONTEXT(::testing::UnitTest::GetInstance()->
- current_test_info()->test_case_name());
+ const ::testing::TestInfo* const test_info =
+ ::testing::UnitTest::GetInstance()->current_test_info();
+ string test_name =
+ string(test_info->test_case_name()) + "_" + string(test_info->name());
+ BWE_TEST_LOGGING_GLOBAL_CONTEXT(test_name);
const BweTestConfig& config = GetParam();
@@ -175,7 +192,7 @@
for (vector<BweTestConfig::EstimatorConfig>:: const_iterator it =
config.estimator_configs.begin(); it != config.estimator_configs.end();
++it) {
- estimators_.push_back(new TestedEstimator(*it));
+ estimators_.push_back(new TestedEstimator(test_name, *it));
}
if (estimators_.size() > 1) {
// Set all estimators as relative to the first one.
@@ -189,7 +206,13 @@
void BweTest::TearDown() {
BWE_TEST_LOGGING_GLOBAL_ENABLE(true);
- LogStats();
+
+ for (vector<TestedEstimator*>::iterator eit = estimators_.begin();
+ eit != estimators_.end(); ++eit) {
+ (*eit)->VerifyOrWriteBaseline();
+ (*eit)->LogStats();
+ }
+
BWE_TEST_LOGGING_GLOBAL_CONTEXT("");
}
@@ -251,13 +274,6 @@
}
}
}
-
-void BweTest::LogStats() {
- for (vector<TestedEstimator*>::iterator eit = estimators_.begin();
- eit != estimators_.end(); ++eit) {
- (*eit)->LogStats();
- }
-}
} // namespace bwe
} // namespace testing
} // namespace webrtc
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
index 0945ea6..286720b 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
@@ -23,14 +23,27 @@
struct BweTestConfig {
struct EstimatorConfig {
- EstimatorConfig() : debug_name(), estimator_factory(NULL) {}
+ EstimatorConfig()
+ : debug_name(),
+ estimator_factory(NULL),
+ update_baseline(false) {
+ }
EstimatorConfig(std::string debug_name,
const RemoteBitrateEstimatorFactory* estimator_factory)
: debug_name(debug_name),
- estimator_factory(estimator_factory) {
+ estimator_factory(estimator_factory),
+ update_baseline(false) {
+ }
+ EstimatorConfig(std::string debug_name,
+ const RemoteBitrateEstimatorFactory* estimator_factory,
+ bool update_baseline)
+ : debug_name(debug_name),
+ estimator_factory(estimator_factory),
+ update_baseline(update_baseline) {
}
std::string debug_name;
const RemoteBitrateEstimatorFactory* estimator_factory;
+ bool update_baseline;
};
std::vector<const PacketSenderFactory*> sender_factories;
@@ -51,7 +64,6 @@
protected:
void VerboseLogging(bool enable);
void RunFor(int64_t time_ms);
- void LogStats();
private:
class TestedEstimator;
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc
new file mode 100644
index 0000000..30723be
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <vector>
+
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h"
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h"
+#include "webrtc/system_wrappers/interface/constructor_magic.h"
+#include "webrtc/system_wrappers/interface/scoped_ptr.h"
+#include "webrtc/test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace testing {
+namespace bwe {
+
+// The format of BWE test baseline files is extremely simple:
+// 1. All read/written entities are 32-bit unsigned integers in network byte
+// order (Big Endian).
+// 2. Files beging with a 2 word header containing a magic marker and file
+// format version indicator. The Magic marker reads "BWE!" in a hex dump.
+// 3. Each estimate is logged as a pair of words: time in milliseconds and
+// estimated bit rate, in bits per second.
+const uint32_t kMagicMarker = 0x42574521;
+const uint32_t kFileVersion1 = 0x00000001;
+const char kResourceSubDir[] = "remote_bitrate_estimator";
+
+class BaseLineFileVerify : public BaseLineFileInterface {
+ public:
+ // If |allow_missing_file| is set, VerifyOrWrite() will return true even if
+ // the baseline file is missing. This is the default when verifying files, but
+ // not when updating (i.e. we always write it out if missing).
+ BaseLineFileVerify(const std::string& filepath, bool allow_missing_file)
+ : reader_(),
+ fail_to_read_response_(false) {
+ scoped_ptr<ResourceFileReader> reader;
+ reader.reset(ResourceFileReader::Create(filepath, "bin"));
+ if (!reader.get()) {
+ printf("WARNING: Missing baseline file for BWE test: %s.bin\n",
+ filepath.c_str());
+ fail_to_read_response_ = allow_missing_file;
+ } else {
+ uint32_t magic_marker = 0;
+ uint32_t file_version = 0;
+ if (reader->Read(&magic_marker) && magic_marker == kMagicMarker &&
+ reader->Read(&file_version) && file_version == kFileVersion1) {
+ reader_.swap(reader);
+ } else {
+ printf("WARNING: Bad baseline file header for BWE test: %s.bin\n",
+ filepath.c_str());
+ }
+ }
+ }
+ virtual ~BaseLineFileVerify() {}
+
+ virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) {
+ if (reader_.get()) {
+ uint32_t read_ms = 0;
+ uint32_t read_bps = 0;
+ if (reader_->Read(&read_ms) && read_ms == time_ms &&
+ reader_->Read(&read_bps) && read_bps == estimate_bps) {
+ } else {
+ printf("ERROR: Baseline differs starting at: %d ms (%d vs %d)!\n",
+ static_cast<uint32_t>(time_ms), estimate_bps, read_bps);
+ reader_.reset(NULL);
+ }
+ }
+ }
+
+ virtual bool VerifyOrWrite() {
+ if (reader_.get()) {
+ if (reader_->IsAtEnd()) {
+ return true;
+ } else {
+ printf("ERROR: Baseline file contains more data!\n");
+ return false;
+ }
+ }
+ return fail_to_read_response_;
+ }
+
+ private:
+ scoped_ptr<ResourceFileReader> reader_;
+ bool fail_to_read_response_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BaseLineFileVerify);
+};
+
+class BaseLineFileUpdate : public BaseLineFileInterface {
+ public:
+ BaseLineFileUpdate(const std::string& filepath,
+ BaseLineFileInterface* verifier)
+ : verifier_(verifier),
+ output_content_(),
+ filepath_(filepath) {
+ output_content_.push_back(kMagicMarker);
+ output_content_.push_back(kFileVersion1);
+ }
+ virtual ~BaseLineFileUpdate() {}
+
+ virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) {
+ verifier_->Estimate(time_ms, estimate_bps);
+ output_content_.push_back(static_cast<uint32_t>(time_ms));
+ output_content_.push_back(estimate_bps);
+ }
+
+ virtual bool VerifyOrWrite() {
+ if (!verifier_->VerifyOrWrite()) {
+ std::string dir_path = webrtc::test::OutputPath() + kResourceSubDir;
+ if (!webrtc::test::CreateDirectory(dir_path)) {
+ printf("WARNING: Cannot create output dir: %s\n", dir_path.c_str());
+ return false;
+ }
+ scoped_ptr<OutputFileWriter> writer;
+ writer.reset(OutputFileWriter::Create(filepath_, "bin"));
+ if (!writer.get()) {
+ printf("WARNING: Cannot create output file: %s.bin\n",
+ filepath_.c_str());
+ return false;
+ }
+ printf("NOTE: Writing baseline file for BWE test: %s.bin\n",
+ filepath_.c_str());
+ for (std::vector<uint32_t>::iterator it = output_content_.begin();
+ it != output_content_.end(); ++it) {
+ writer->Write(*it);
+ }
+ return true;
+ }
+ printf("NOTE: No change, not writing: %s\n", filepath_.c_str());
+ return true;
+ }
+
+ private:
+ scoped_ptr<BaseLineFileInterface> verifier_;
+ std::vector<uint32_t> output_content_;
+ std::string filepath_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BaseLineFileUpdate);
+};
+
+BaseLineFileInterface* BaseLineFileInterface::Create(
+ const std::string& filename, bool write_output_file) {
+ std::string filepath = filename;
+ std::replace(filepath.begin(), filepath.end(), '/', '_');
+ filepath = std::string(kResourceSubDir) + "/" + filepath;
+
+ scoped_ptr<BaseLineFileInterface> result;
+ result.reset(new BaseLineFileVerify(filepath, !write_output_file));
+ if (write_output_file) {
+ // Takes ownership of the |verifier| instance.
+ result.reset(new BaseLineFileUpdate(filepath, result.release()));
+ }
+ return result.release();
+}
+} // namespace bwe
+} // namespace testing
+} // namespace webrtc
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h
new file mode 100644
index 0000000..64dfa85
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_
+#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_
+
+#include <string>
+#include "webrtc/modules/interface/module_common_types.h"
+
+namespace webrtc {
+namespace testing {
+namespace bwe {
+
+class BaseLineFileInterface {
+ public:
+ virtual ~BaseLineFileInterface() {}
+
+ // Compare, or log, one estimate against the baseline file.
+ virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) = 0;
+
+ // Verify whether there are any differences between the logged estimates and
+ // those read from the baseline file. If updating the baseline file, write out
+ // new file if there were differences. Return true if logged estimates are
+ // identical, or if output file was updated successfully.
+ virtual bool VerifyOrWrite() = 0;
+
+ // Create an instance for either verifying estimates against a baseline file
+ // with name |filename|, living in the resources/ directory or, if the flag
+ // |write_updated_file| is set, write logged estimates to a file with the same
+ // name, living in the out/ directory.
+ static BaseLineFileInterface* Create(const std::string& filename,
+ bool write_updated_file);
+};
+} // namespace bwe
+} // namespace testing
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc
new file mode 100644
index 0000000..4e5d407
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h"
+
+#ifdef WIN32
+#include <Winsock2.h>
+#else
+#include <arpa/inet.h>
+#endif
+#include <cassert>
+
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h"
+#include "webrtc/system_wrappers/interface/scoped_ptr.h"
+#include "webrtc/test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace testing {
+namespace bwe {
+
+ResourceFileReader::~ResourceFileReader() {
+ if (file_ != NULL) {
+ fclose(file_);
+ file_ = NULL;
+ }
+}
+
+bool ResourceFileReader::IsAtEnd() {
+ int32_t current_pos = ftell(file_);
+ fseek(file_, 0, SEEK_END);
+ int32_t end_pos = ftell(file_);
+ fseek(file_, current_pos, SEEK_SET);
+ return current_pos == end_pos;
+}
+
+bool ResourceFileReader::Read(uint32_t* out) {
+ assert(out);
+ uint32_t tmp = 0;
+ if (fread(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) {
+ printf("Error reading!\n");
+ return false;
+ }
+ *out = ntohl(tmp);
+ return true;
+}
+
+ResourceFileReader* ResourceFileReader::Create(const std::string& filename,
+ const std::string& extension) {
+ std::string filepath = webrtc::test::ResourcePath(filename, extension);
+ FILE* file = fopen(filepath.c_str(), "rb");
+ if (file == NULL) {
+ BWE_TEST_LOGGING_CONTEXT("ResourceFileReader");
+ BWE_TEST_LOGGING_LOG1("Create", "Can't read file: %s", filepath.c_str());
+ return 0;
+ } else {
+ return new ResourceFileReader(file);
+ }
+}
+
+OutputFileWriter::~OutputFileWriter() {
+ if (file_ != NULL) {
+ fclose(file_);
+ file_ = NULL;
+ }
+}
+
+bool OutputFileWriter::Write(uint32_t value) {
+ uint32_t tmp = htonl(value);
+ if (fwrite(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) {
+ return false;
+ }
+ return true;
+}
+
+OutputFileWriter* OutputFileWriter::Create(const std::string& filename,
+ const std::string& extension) {
+ std::string filepath = webrtc::test::OutputPath() + filename + "." +
+ extension;
+ FILE* file = fopen(filepath.c_str(), "wb");
+ if (file == NULL) {
+ BWE_TEST_LOGGING_CONTEXT("OutputFileWriter");
+ BWE_TEST_LOGGING_LOG1("Create", "Can't write file: %s", filepath.c_str());
+ return NULL;
+ } else {
+ return new OutputFileWriter(file);
+ }
+}
+} // namespace bwe
+} // namespace testing
+} // namespace webrtc
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h
new file mode 100644
index 0000000..397a169
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_
+#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_
+
+#include <cstdio>
+#include <string>
+
+#include "webrtc/modules/interface/module_common_types.h"
+#include "webrtc/system_wrappers/interface/constructor_magic.h"
+
+namespace webrtc {
+namespace testing {
+namespace bwe {
+
+class ResourceFileReader {
+ public:
+ ~ResourceFileReader();
+
+ bool IsAtEnd();
+ bool Read(uint32_t* out);
+
+ static ResourceFileReader* Create(const std::string& filename,
+ const std::string& extension);
+
+ private:
+ explicit ResourceFileReader(FILE* file) : file_(file) {}
+ FILE* file_;
+ DISALLOW_IMPLICIT_CONSTRUCTORS(ResourceFileReader);
+};
+
+class OutputFileWriter {
+ public:
+ ~OutputFileWriter();
+
+ bool Write(uint32_t value);
+
+ static OutputFileWriter* Create(const std::string& filename,
+ const std::string& extension);
+
+ private:
+ explicit OutputFileWriter(FILE* file) : file_(file) {}
+ FILE* file_;
+ DISALLOW_IMPLICIT_CONSTRUCTORS(OutputFileWriter);
+};
+} // namespace bwe
+} // namespace testing
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h
index c19cb2b..71fa0ea 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h
@@ -288,7 +288,7 @@
class PacketSender : public PacketProcessor {
public:
struct Feedback {
- double estimated_kbps;
+ uint32_t estimated_bps;
};
explicit PacketSender(PacketProcessorListener* listener);