Add processing time to VideoFrame
Bug: chromium:1011581
Change-Id: Icd675cb98b8b5052933b9a8eebe718be94c2fef2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166162
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30281}
diff --git a/api/video/video_frame.h b/api/video/video_frame.h
index f312e7a..08c939d 100644
--- a/api/video/video_frame.h
+++ b/api/video/video_frame.h
@@ -72,6 +72,12 @@
int scaled_height) const;
};
+ struct RTC_EXPORT ProcessingTime {
+ TimeDelta Elapsed() const { return finish - start; }
+ Timestamp start;
+ Timestamp finish;
+ };
+
// Preferred way of building VideoFrame objects.
class RTC_EXPORT Builder {
public:
@@ -223,6 +229,13 @@
packet_infos_ = std::move(value);
}
+ const absl::optional<ProcessingTime> processing_time() const {
+ return processing_time_;
+ }
+ void set_processing_time(const ProcessingTime& processing_time) {
+ processing_time_ = processing_time;
+ }
+
private:
VideoFrame(uint16_t id,
const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
@@ -252,6 +265,11 @@
// MediaStreamTrack, in order to implement getContributingSources(). See:
// https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
RtpPacketInfos packet_infos_;
+ // Processing timestamps of the frame. For received video frames these are the
+ // timestamps when the frame is sent to the decoder and the decoded image
+ // returned from the decoder.
+ // Currently, not set for locally captured video frames.
+ absl::optional<ProcessingTime> processing_time_;
};
} // namespace webrtc
diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc
index 2cd3204..100686d 100644
--- a/modules/video_coding/generic_decoder.cc
+++ b/modules/video_coding/generic_decoder.cc
@@ -99,11 +99,13 @@
decodedImage.set_packet_infos(frameInfo->packet_infos);
decodedImage.set_rotation(frameInfo->rotation);
- const int64_t now_ms = _clock->TimeInMilliseconds();
+ const Timestamp now = _clock->CurrentTime();
+ RTC_DCHECK(frameInfo->decodeStart);
if (!decode_time_ms) {
- decode_time_ms = now_ms - frameInfo->decodeStartTimeMs;
+ decode_time_ms = (now - *frameInfo->decodeStart).ms();
}
- _timing->StopDecodeTimer(*decode_time_ms, now_ms);
+ _timing->StopDecodeTimer(*decode_time_ms, now.ms());
+ decodedImage.set_processing_time({*frameInfo->decodeStart, now});
// Report timing information.
TimingFrameInfo timing_frame_info;
@@ -147,8 +149,8 @@
}
timing_frame_info.flags = frameInfo->timing.flags;
- timing_frame_info.decode_start_ms = frameInfo->decodeStartTimeMs;
- timing_frame_info.decode_finish_ms = now_ms;
+ timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
+ timing_frame_info.decode_finish_ms = now.ms();
timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
timing_frame_info.rtp_timestamp = decodedImage.timestamp();
timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
@@ -210,10 +212,10 @@
return decoder_->InitDecode(settings, numberOfCores);
}
-int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) {
+int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
frame.Timestamp());
- _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
+ _frameInfos[_nextFrameInfoIdx].decodeStart = now;
_frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
_frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
_frameInfos[_nextFrameInfoIdx].timing = frame.video_timing();
diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h
index a9d9698..4b4d83e 100644
--- a/modules/video_coding/generic_decoder.h
+++ b/modules/video_coding/generic_decoder.h
@@ -30,14 +30,14 @@
struct VCMFrameInformation {
int64_t renderTimeMs;
- int64_t decodeStartTimeMs;
+ absl::optional<Timestamp> decodeStart;
void* userData;
VideoRotation rotation;
VideoContentType content_type;
EncodedImage::Timing timing;
int64_t ntp_time_ms;
RtpPacketInfos packet_infos;
- // ColorSpace is not storred here, as it might be modified by decoders.
+ // ColorSpace is not stored here, as it might be modified by decoders.
};
class VCMDecodedFrameCallback : public DecodedImageCallback {
@@ -92,7 +92,7 @@
*
* inputVideoBuffer reference to encoded video frame
*/
- int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs);
+ int32_t Decode(const VCMEncodedFrame& inputFrame, Timestamp now);
/**
* Set decode callback. Deregistering while decoding is illegal.
diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc
index 66167eb..3e07a2a 100644
--- a/modules/video_coding/generic_decoder_unittest.cc
+++ b/modules/video_coding/generic_decoder_unittest.cc
@@ -93,7 +93,7 @@
RtpPacketInfos packet_infos = CreatePacketInfos(3);
VCMEncodedFrame encoded_frame;
encoded_frame.SetPacketInfos(packet_infos);
- generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds());
+ generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
ASSERT_TRUE(decoded_frame.has_value());
EXPECT_EQ(decoded_frame->packet_infos().size(), 3U);
@@ -107,7 +107,7 @@
// Ensure the original frame is destroyed before the decoding is completed.
VCMEncodedFrame encoded_frame;
encoded_frame.SetPacketInfos(packet_infos);
- generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds());
+ generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
}
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(200);
diff --git a/modules/video_coding/video_receiver.cc b/modules/video_coding/video_receiver.cc
index e52abf2..44d5526 100644
--- a/modules/video_coding/video_receiver.cc
+++ b/modules/video_coding/video_receiver.cc
@@ -259,7 +259,7 @@
if (decoder == nullptr) {
return VCM_NO_CODEC_REGISTERED;
}
- return decoder->Decode(frame, clock_->TimeInMilliseconds());
+ return decoder->Decode(frame, clock_->CurrentTime());
}
// Register possible receive codecs, can be called multiple times
diff --git a/modules/video_coding/video_receiver2.cc b/modules/video_coding/video_receiver2.cc
index d1e57d1..8eaefbb 100644
--- a/modules/video_coding/video_receiver2.cc
+++ b/modules/video_coding/video_receiver2.cc
@@ -91,7 +91,7 @@
if (decoder == nullptr) {
return VCM_NO_CODEC_REGISTERED;
}
- return decoder->Decode(*frame, clock_->TimeInMilliseconds());
+ return decoder->Decode(*frame, clock_->CurrentTime());
}
// Register possible receive codecs, can be called multiple times
