Calculate local/remote clock delta and capture ntp timestamp in receiver's timebase.
BUG=3111
TEST=new performance tests
R=niklas.enbom@webrtc.org, stefan@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/11689004
git-svn-id: http://webrtc.googlecode.com/svn/trunk@5976 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h b/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h
index 7928abf..08a4d46 100644
--- a/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h
+++ b/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h
@@ -29,6 +29,15 @@
typedef std::list<RtcpMeasurement> RtcpList;
+// Updates |rtcp_list| with timestamps from the latest RTCP SR.
+// |new_rtcp_sr| will be set to true if these are the timestamps which have
+// never be added to |rtcp_list|.
+bool UpdateRtcpList(uint32_t ntp_secs,
+ uint32_t ntp_frac,
+ uint32_t rtp_timestamp,
+ RtcpList* rtcp_list,
+ bool* new_rtcp_sr);
+
// Converts an RTP timestamp to the NTP domain in milliseconds using two
// (RTP timestamp, NTP timestamp) pairs.
bool RtpToNtpMs(int64_t rtp_timestamp, const RtcpList& rtcp,
diff --git a/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc b/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc
index 109edae..775cd0d 100644
--- a/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc
+++ b/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc
@@ -57,6 +57,40 @@
return true;
}
+bool UpdateRtcpList(uint32_t ntp_secs,
+ uint32_t ntp_frac,
+ uint32_t rtp_timestamp,
+ RtcpList* rtcp_list,
+ bool* new_rtcp_sr) {
+ *new_rtcp_sr = false;
+ if (ntp_secs == 0 && ntp_frac == 0) {
+ return false;
+ }
+
+ RtcpMeasurement measurement;
+ measurement.ntp_secs = ntp_secs;
+ measurement.ntp_frac = ntp_frac;
+ measurement.rtp_timestamp = rtp_timestamp;
+
+ for (RtcpList::iterator it = rtcp_list->begin();
+ it != rtcp_list->end(); ++it) {
+ if (measurement.ntp_secs == (*it).ntp_secs &&
+ measurement.ntp_frac == (*it).ntp_frac) {
+ // This RTCP has already been added to the list.
+ return true;
+ }
+ }
+
+ // We need two RTCP SR reports to map between RTP and NTP. More than two will
+ // not improve the mapping.
+ if (rtcp_list->size() == 2) {
+ rtcp_list->pop_back();
+ }
+ rtcp_list->push_front(measurement);
+ *new_rtcp_sr = true;
+ return true;
+}
+
// Converts |rtp_timestamp| to the NTP time base using the NTP and RTP timestamp
// pairs in |rtcp|. The converted timestamp is returned in
// |rtp_timestamp_in_ms|. This function compensates for wrap arounds in RTP
diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
index dc65336..70a3443 100644
--- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
+++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
@@ -733,7 +733,12 @@
uint16_t* avg_rtt,
uint16_t* min_rtt,
uint16_t* max_rtt) const {
- return rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt);
+ int32_t ret = rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt);
+ if (rtt && *rtt == 0) {
+ // Try to get RTT from RtcpRttStats class.
+ *rtt = static_cast<uint16_t>(rtt_ms());
+ }
+ return ret;
}
// Reset RoundTripTime statistics.
diff --git a/webrtc/test/fake_decoder.cc b/webrtc/test/fake_decoder.cc
index a9e6f50..1624ea0 100644
--- a/webrtc/test/fake_decoder.cc
+++ b/webrtc/test/fake_decoder.cc
@@ -36,6 +36,7 @@
const CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) {
frame_.set_timestamp(input._timeStamp);
+ frame_.set_ntp_time_ms(input.ntp_time_ms_);
frame_.set_render_time_ms(render_time_ms);
callback_->Decoded(frame_);
diff --git a/webrtc/test/frame_generator_capturer.cc b/webrtc/test/frame_generator_capturer.cc
index 6570c6f..d1c17c7 100644
--- a/webrtc/test/frame_generator_capturer.cc
+++ b/webrtc/test/frame_generator_capturer.cc
@@ -67,7 +67,8 @@
tick_(EventWrapper::Create()),
lock_(CriticalSectionWrapper::CreateCriticalSection()),
frame_generator_(frame_generator),
- target_fps_(target_fps) {
+ target_fps_(target_fps),
+ first_frame_capture_time_(-1) {
assert(input != NULL);
assert(frame_generator != NULL);
assert(target_fps > 0);
@@ -113,6 +114,9 @@
if (sending_) {
I420VideoFrame* frame = frame_generator_->NextFrame();
frame->set_render_time_ms(clock_->CurrentNtpInMilliseconds());
+ if (first_frame_capture_time_ == -1) {
+ first_frame_capture_time_ = frame->render_time_ms();
+ }
input_->SwapFrame(frame);
}
}
diff --git a/webrtc/test/frame_generator_capturer.h b/webrtc/test/frame_generator_capturer.h
index 5be6bb2..84b3c49 100644
--- a/webrtc/test/frame_generator_capturer.h
+++ b/webrtc/test/frame_generator_capturer.h
@@ -43,6 +43,8 @@
virtual void Start() OVERRIDE;
virtual void Stop() OVERRIDE;
+ int64_t first_frame_capture_time() const { return first_frame_capture_time_; }
+
private:
FrameGeneratorCapturer(Clock* clock,
VideoSendStreamInput* input,
@@ -61,6 +63,8 @@
scoped_ptr<FrameGenerator> frame_generator_;
int target_fps_;
+
+ int64_t first_frame_capture_time_;
};
} // test
} // webrtc
diff --git a/webrtc/video/call_perf_tests.cc b/webrtc/video/call_perf_tests.cc
index a5effe1..85d7e73 100644
--- a/webrtc/video/call_perf_tests.cc
+++ b/webrtc/video/call_perf_tests.cc
@@ -80,6 +80,11 @@
void TestMinTransmitBitrate(bool pad_to_min_bitrate);
+ void TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config,
+ int threshold_ms,
+ int start_time_ms,
+ int run_time_ms);
+
VideoSendStream* send_stream_;
test::FakeEncoder fake_encoder_;
};
@@ -343,6 +348,194 @@
VoiceEngine::Delete(voice_engine);
}
+class CaptureNtpTimeObserver : public test::RtpRtcpObserver,
+ public VideoRenderer {
+ public:
+ CaptureNtpTimeObserver(Clock* clock,
+ const FakeNetworkPipe::Config& config,
+ int threshold_ms,
+ int start_time_ms,
+ int run_time_ms)
+ : RtpRtcpObserver(kLongTimeoutMs, config),
+ clock_(clock),
+ threshold_ms_(threshold_ms),
+ start_time_ms_(start_time_ms),
+ run_time_ms_(run_time_ms),
+ creation_time_ms_(clock_->TimeInMilliseconds()),
+ capturer_(NULL),
+ rtp_start_timestamp_set_(false),
+ rtp_start_timestamp_(0) {}
+
+ virtual void RenderFrame(const I420VideoFrame& video_frame,
+ int time_to_render_ms) OVERRIDE {
+ if (video_frame.ntp_time_ms() <= 0) {
+ // Haven't got enough RTCP SR in order to calculate the capture ntp time.
+ return;
+ }
+
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ int64_t time_since_creation = now_ms - creation_time_ms_;
+ if (time_since_creation < start_time_ms_) {
+ // Wait for |start_time_ms_| before start measuring.
+ return;
+ }
+
+ if (time_since_creation > run_time_ms_) {
+ observation_complete_->Set();
+ }
+
+ FrameCaptureTimeList::iterator iter =
+ capture_time_list_.find(video_frame.timestamp());
+ EXPECT_TRUE(iter != capture_time_list_.end());
+
+ // The real capture time has been wrapped to uint32_t before converted
+ // to rtp timestamp in the sender side. So here we convert the estimated
+ // capture time to a uint32_t 90k timestamp also for comparing.
+ uint32_t estimated_capture_timestamp =
+ 90 * static_cast<uint32_t>(video_frame.ntp_time_ms());
+ uint32_t real_capture_timestamp = iter->second;
+ int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp;
+ time_offset_ms = time_offset_ms / 90;
+ std::stringstream ss;
+ ss << time_offset_ms;
+
+ webrtc::test::PrintResult("capture_ntp_time",
+ "",
+ "real - estimated",
+ ss.str(),
+ "ms",
+ true);
+ EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_);
+ }
+
+ virtual Action OnSendRtp(const uint8_t* packet, size_t length) {
+ RTPHeader header;
+ EXPECT_TRUE(parser_->Parse(packet, static_cast<int>(length), &header));
+
+ if (!rtp_start_timestamp_set_) {
+ // Calculate the rtp timestamp offset in order to calculate the real
+ // capture time.
+ uint32_t first_capture_timestamp =
+ 90 * static_cast<uint32_t>(capturer_->first_frame_capture_time());
+ rtp_start_timestamp_ = header.timestamp - first_capture_timestamp;
+ rtp_start_timestamp_set_ = true;
+ }
+
+ uint32_t capture_timestamp = header.timestamp - rtp_start_timestamp_;
+ capture_time_list_.insert(capture_time_list_.end(),
+ std::make_pair(header.timestamp,
+ capture_timestamp));
+ return SEND_PACKET;
+ }
+
+ void SetCapturer(test::FrameGeneratorCapturer* capturer) {
+ capturer_ = capturer;
+ }
+
+ private:
+ Clock* clock_;
+ int threshold_ms_;
+ int start_time_ms_;
+ int run_time_ms_;
+ int64_t creation_time_ms_;
+ test::FrameGeneratorCapturer* capturer_;
+ bool rtp_start_timestamp_set_;
+ uint32_t rtp_start_timestamp_;
+ typedef std::map<uint32_t, uint32_t> FrameCaptureTimeList;
+ FrameCaptureTimeList capture_time_list_;
+};
+
+void CallPerfTest::TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config,
+ int threshold_ms,
+ int start_time_ms,
+ int run_time_ms) {
+ CaptureNtpTimeObserver observer(Clock::GetRealTimeClock(),
+ net_config,
+ threshold_ms,
+ start_time_ms,
+ run_time_ms);
+
+ // Sender/receiver call.
+ Call::Config receiver_config(observer.ReceiveTransport());
+ scoped_ptr<Call> receiver_call(Call::Create(receiver_config));
+ scoped_ptr<Call> sender_call(
+ Call::Create(Call::Config(observer.SendTransport())));
+ observer.SetReceivers(receiver_call->Receiver(), sender_call->Receiver());
+
+ // Configure send stream.
+ VideoSendStream::Config send_config = GetSendTestConfig(sender_call.get());
+ VideoSendStream* send_stream =
+ sender_call->CreateVideoSendStream(send_config);
+ scoped_ptr<test::FrameGeneratorCapturer> capturer(
+ test::FrameGeneratorCapturer::Create(
+ send_stream->Input(),
+ send_config.encoder_settings.streams[0].width,
+ send_config.encoder_settings.streams[0].height,
+ 30,
+ Clock::GetRealTimeClock()));
+ observer.SetCapturer(capturer.get());
+
+ // Configure receive stream.
+ VideoReceiveStream::Config receive_config =
+ receiver_call->GetDefaultReceiveConfig();
+ assert(receive_config.codecs.empty());
+ VideoCodec codec =
+ test::CreateDecoderVideoCodec(send_config.encoder_settings);
+ receive_config.codecs.push_back(codec);
+ assert(receive_config.external_decoders.empty());
+ ExternalVideoDecoder decoder;
+ test::FakeDecoder fake_decoder;
+ decoder.decoder = &fake_decoder;
+ decoder.payload_type = send_config.encoder_settings.payload_type;
+ receive_config.external_decoders.push_back(decoder);
+ receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0];
+ receive_config.rtp.local_ssrc = kReceiverLocalSsrc;
+ receive_config.renderer = &observer;
+ // Enable the receiver side rtt calculation.
+ receive_config.rtp.rtcp_xr.receiver_reference_time_report = true;
+ VideoReceiveStream* receive_stream =
+ receiver_call->CreateVideoReceiveStream(receive_config);
+
+ // Start the test
+ receive_stream->Start();
+ send_stream->Start();
+ capturer->Start();
+
+ EXPECT_EQ(kEventSignaled, observer.Wait())
+ << "Timed out while waiting for estimated capture ntp time to be "
+ << "within bounds.";
+
+ capturer->Stop();
+ send_stream->Stop();
+ receive_stream->Stop();
+ observer.StopSending();
+
+ sender_call->DestroyVideoSendStream(send_stream);
+ receiver_call->DestroyVideoReceiveStream(receive_stream);
+}
+
+TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkDelay) {
+ FakeNetworkPipe::Config net_config;
+ net_config.queue_delay_ms = 100;
+ // TODO(wu): lower the threshold as the calculation/estimatation becomes more
+ // accurate.
+ const int kThresholdMs = 30;
+ const int kStartTimeMs = 10000;
+ const int kRunTimeMs = 20000;
+ TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
+}
+
+TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkJitter) {
+ FakeNetworkPipe::Config net_config;
+ net_config.delay_standard_deviation_ms = 10;
+ // TODO(wu): lower the threshold as the calculation/estimatation becomes more
+ // accurate.
+ const int kThresholdMs = 30;
+ const int kStartTimeMs = 10000;
+ const int kRunTimeMs = 20000;
+ TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
+}
+
TEST_F(CallPerfTest, RegisterCpuOveruseObserver) {
// Verifies that either a normal or overuse callback is triggered.
class OveruseCallbackObserver : public test::RtpRtcpObserver,
diff --git a/webrtc/video_engine/vie_receiver.cc b/webrtc/video_engine/vie_receiver.cc
index 75f53ab..76228b3 100644
--- a/webrtc/video_engine/vie_receiver.cc
+++ b/webrtc/video_engine/vie_receiver.cc
@@ -21,7 +21,9 @@
#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
#include "webrtc/modules/utility/interface/rtp_dump.h"
#include "webrtc/modules/video_coding/main/interface/video_coding.h"
+#include "webrtc/modules/video_coding/main/source/timestamp_extrapolator.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
+#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/tick_util.h"
#include "webrtc/system_wrappers/interface/trace.h"
@@ -45,6 +47,8 @@
rtp_rtcp_(NULL),
vcm_(module_vcm),
remote_bitrate_estimator_(remote_bitrate_estimator),
+ clock_(Clock::GetRealTimeClock()),
+ ts_extrapolator_(new VCMTimestampExtrapolator(clock_)),
rtp_dump_(NULL),
receiving_(false),
restored_packet_in_use_(false),
@@ -171,14 +175,37 @@
int32_t ViEReceiver::OnReceivedPayloadData(
const uint8_t* payload_data, const uint16_t payload_size,
const WebRtcRTPHeader* rtp_header) {
- // TODO(wu): Calculate ntp_time_ms
- if (vcm_->IncomingPacket(payload_data, payload_size, *rtp_header) != 0) {
+ WebRtcRTPHeader rtp_header_with_ntp = *rtp_header;
+ CalculateCaptureNtpTime(&rtp_header_with_ntp);
+ if (vcm_->IncomingPacket(payload_data,
+ payload_size,
+ rtp_header_with_ntp) != 0) {
// Check this...
return -1;
}
return 0;
}
+void ViEReceiver::CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header) {
+ if (rtcp_list_.size() < 2) {
+ // We need two RTCP SR reports to calculate NTP.
+ return;
+ }
+
+ int64_t sender_capture_ntp_ms = 0;
+ if (!synchronization::RtpToNtpMs(rtp_header->header.timestamp,
+ rtcp_list_,
+ &sender_capture_ntp_ms)) {
+ return;
+ }
+ uint32_t timestamp = sender_capture_ntp_ms * 90;
+ int64_t receiver_capture_ms =
+ ts_extrapolator_->ExtrapolateLocalTime(timestamp);
+ int64_t ntp_offset =
+ clock_->CurrentNtpInMilliseconds() - clock_->TimeInMilliseconds();
+ rtp_header->ntp_time_ms = receiver_capture_ms + ntp_offset;
+}
+
bool ViEReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
int rtp_packet_length) {
RTPHeader header;
@@ -329,7 +356,56 @@
}
}
assert(rtp_rtcp_); // Should be set by owner at construction time.
- return rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
+ int ret = rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (!GetRtcpTimestamp()) {
+ LOG(LS_WARNING) << "Failed to retrieve timestamp information from RTCP SR.";
+ }
+
+ return 0;
+}
+
+bool ViEReceiver::GetRtcpTimestamp() {
+ uint16_t rtt = 0;
+ rtp_rtcp_->RTT(rtp_receiver_->SSRC(), &rtt, NULL, NULL, NULL);
+ if (rtt == 0) {
+ // Waiting for valid rtt.
+ return true;
+ }
+
+ // Update RTCP list
+ uint32_t ntp_secs = 0;
+ uint32_t ntp_frac = 0;
+ uint32_t rtp_timestamp = 0;
+ if (0 != rtp_rtcp_->RemoteNTP(&ntp_secs,
+ &ntp_frac,
+ NULL,
+ NULL,
+ &rtp_timestamp)) {
+ return false;
+ }
+
+ bool new_rtcp_sr = false;
+ if (!synchronization::UpdateRtcpList(
+ ntp_secs, ntp_frac, rtp_timestamp, &rtcp_list_, &new_rtcp_sr)) {
+ return false;
+ }
+
+ if (!new_rtcp_sr) {
+ // No new RTCP SR since last time this function was called.
+ return true;
+ }
+
+ // Update extrapolator with the new arrival time.
+ // The extrapolator assumes the TimeInMilliseconds time.
+ int64_t receiver_arrival_time = clock_->TimeInMilliseconds();
+ int64_t sender_send_time_ms = Clock::NtpToMs(ntp_secs, ntp_frac);
+ int64_t sender_arrival_time_90k = (sender_send_time_ms + rtt / 2) * 90;
+ ts_extrapolator_->Update(receiver_arrival_time, sender_arrival_time_90k);
+ return true;
}
void ViEReceiver::StartReceive() {
diff --git a/webrtc/video_engine/vie_receiver.h b/webrtc/video_engine/vie_receiver.h
index b987c99..512a7b6 100644
--- a/webrtc/video_engine/vie_receiver.h
+++ b/webrtc/video_engine/vie_receiver.h
@@ -14,6 +14,7 @@
#include <list>
#include "webrtc/engine_configurations.h"
+#include "webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h"
#include "webrtc/modules/rtp_rtcp/interface/receive_statistics.h"
#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
@@ -21,6 +22,9 @@
#include "webrtc/video_engine/include/vie_network.h"
#include "webrtc/video_engine/vie_defines.h"
+// TODO(wu): Move rtp_to_ntp.h and timestamp_extrapolator.h to somewhere that
+// can be shared between audio and video.
+
namespace webrtc {
class CriticalSectionWrapper;
@@ -32,6 +36,7 @@
class RTPPayloadRegistry;
class RtpReceiver;
class RtpRtcp;
+class VCMTimestampExtrapolator;
class VideoCodingModule;
struct ReceiveBandwidthEstimatorStats;
@@ -105,6 +110,9 @@
bool IsPacketInOrder(const RTPHeader& header) const;
bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const;
+ bool GetRtcpTimestamp();
+ void CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header);
+
scoped_ptr<CriticalSectionWrapper> receive_cs_;
const int32_t channel_id_;
scoped_ptr<RtpHeaderParser> rtp_header_parser_;
@@ -117,6 +125,10 @@
VideoCodingModule* vcm_;
RemoteBitrateEstimator* remote_bitrate_estimator_;
+ Clock* clock_;
+ scoped_ptr<VCMTimestampExtrapolator> ts_extrapolator_;
+ synchronization::RtcpList rtcp_list_;
+
RtpDump* rtp_dump_;
bool receiving_;
uint8_t restored_packet_[kViEMaxMtu];
diff --git a/webrtc/video_engine/vie_sync_module.cc b/webrtc/video_engine/vie_sync_module.cc
index 89da022..b7c74a7 100644
--- a/webrtc/video_engine/vie_sync_module.cc
+++ b/webrtc/video_engine/vie_sync_module.cc
@@ -30,31 +30,24 @@
return -1;
if (!receiver.LastReceivedTimeMs(&stream->latest_receive_time_ms))
return -1;
- synchronization::RtcpMeasurement measurement;
- if (0 != rtp_rtcp.RemoteNTP(&measurement.ntp_secs,
- &measurement.ntp_frac,
+
+ uint32_t ntp_secs = 0;
+ uint32_t ntp_frac = 0;
+ uint32_t rtp_timestamp = 0;
+ if (0 != rtp_rtcp.RemoteNTP(&ntp_secs,
+ &ntp_frac,
NULL,
NULL,
- &measurement.rtp_timestamp)) {
+ &rtp_timestamp)) {
return -1;
}
- if (measurement.ntp_secs == 0 && measurement.ntp_frac == 0) {
+
+ bool new_rtcp_sr = false;
+ if (!synchronization::UpdateRtcpList(
+ ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) {
return -1;
}
- for (synchronization::RtcpList::iterator it = stream->rtcp.begin();
- it != stream->rtcp.end(); ++it) {
- if (measurement.ntp_secs == (*it).ntp_secs &&
- measurement.ntp_frac == (*it).ntp_frac) {
- // This RTCP has already been added to the list.
- return 0;
- }
- }
- // We need two RTCP SR reports to map between RTP and NTP. More than two will
- // not improve the mapping.
- if (stream->rtcp.size() == 2) {
- stream->rtcp.pop_back();
- }
- stream->rtcp.push_front(measurement);
+
return 0;
}
