blob: e921a934577097eaaf81feabd72163e7aead7a68 [file] [log] [blame]
Erik Språng6a7baa72019-02-26 17:31:001/*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3611#include "video/frame_encode_metadata_writer.h"
Erik Språng6a7baa72019-02-26 17:31:0012
13#include <algorithm>
Zhaoliang Ma528e4892021-11-12 05:05:0714#include <memory>
Niels Möller4d504c72019-06-18 13:56:5615#include <utility>
Erik Språng6a7baa72019-02-26 17:31:0016
Mirta Dvornicic28f0eb22019-05-28 14:30:1617#include "common_video/h264/sps_vui_rewriter.h"
Erik Språng6a7baa72019-02-26 17:31:0018#include "modules/include/module_common_types_public.h"
19#include "modules/video_coding/include/video_coding_defines.h"
Zhaoliang Ma528e4892021-11-12 05:05:0720#include "modules/video_coding/svc/create_scalability_structure.h"
Erik Språng6a7baa72019-02-26 17:31:0021#include "rtc_base/logging.h"
22#include "rtc_base/time_utils.h"
23
24namespace webrtc {
25namespace {
26const int kMessagesThrottlingThreshold = 2;
27const int kThrottleRatio = 100000;
Niels Möller4d504c72019-06-18 13:56:5628
29class EncodedImageBufferWrapper : public EncodedImageBufferInterface {
30 public:
31 explicit EncodedImageBufferWrapper(rtc::Buffer&& buffer)
32 : buffer_(std::move(buffer)) {}
33
34 const uint8_t* data() const override { return buffer_.data(); }
35 uint8_t* data() override { return buffer_.data(); }
36 size_t size() const override { return buffer_.size(); }
37
Niels Möller4d504c72019-06-18 13:56:5638 private:
39 rtc::Buffer buffer_;
40};
41
Erik Språng6a7baa72019-02-26 17:31:0042} // namespace
43
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3644FrameEncodeMetadataWriter::TimingFramesLayerInfo::TimingFramesLayerInfo() =
45 default;
46FrameEncodeMetadataWriter::TimingFramesLayerInfo::~TimingFramesLayerInfo() =
47 default;
Erik Språng6a7baa72019-02-26 17:31:0048
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3649FrameEncodeMetadataWriter::FrameEncodeMetadataWriter(
50 EncodedImageCallback* frame_drop_callback)
Erik Språng6a7baa72019-02-26 17:31:0051 : frame_drop_callback_(frame_drop_callback),
Erik Språng6a7baa72019-02-26 17:31:0052 framerate_fps_(0),
53 last_timing_frame_time_ms_(-1),
Erik Språng6a7baa72019-02-26 17:31:0054 reordered_frames_logged_messages_(0),
55 stalled_encoder_logged_messages_(0) {
56 codec_settings_.timing_frame_thresholds = {-1, 0};
57}
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3658FrameEncodeMetadataWriter::~FrameEncodeMetadataWriter() {}
Erik Språng6a7baa72019-02-26 17:31:0059
Niels Möller13d16362021-11-16 10:39:2760void FrameEncodeMetadataWriter::OnEncoderInit(const VideoCodec& codec) {
Markus Handella3765182020-07-08 11:13:3261 MutexLock lock(&lock_);
Erik Språng6a7baa72019-02-26 17:31:0062 codec_settings_ = codec;
Zhaoliang Ma528e4892021-11-12 05:05:0763 size_t num_spatial_layers = codec_settings_.numberOfSimulcastStreams;
64 if (codec_settings_.codecType == kVideoCodecVP9) {
65 num_spatial_layers = std::max(
66 num_spatial_layers,
67 static_cast<size_t>(codec_settings_.VP9()->numberOfSpatialLayers));
68 } else if (codec_settings_.codecType == kVideoCodecAV1 &&
Niels Möller79d566b2022-04-29 09:03:1369 codec_settings_.GetScalabilityMode().has_value()) {
Zhaoliang Ma528e4892021-11-12 05:05:0770 std::unique_ptr<ScalableVideoController> structure =
Niels Möller79d566b2022-04-29 09:03:1371 CreateScalabilityStructure(*codec_settings_.GetScalabilityMode());
Zhaoliang Maea8dfd52022-02-11 04:54:0672 if (structure) {
73 num_spatial_layers = structure->StreamConfig().num_spatial_layers;
74 } else {
75 // |structure| maybe nullptr if the scalability mode is invalid.
76 RTC_LOG(LS_WARNING) << "Cannot create ScalabilityStructure, since the "
77 "scalability mode is invalid";
78 }
Zhaoliang Ma528e4892021-11-12 05:05:0779 }
80 num_spatial_layers_ = std::max(num_spatial_layers, size_t{1});
Erik Språng6a7baa72019-02-26 17:31:0081}
82
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3683void FrameEncodeMetadataWriter::OnSetRates(
Erik Språng6a7baa72019-02-26 17:31:0084 const VideoBitrateAllocation& bitrate_allocation,
85 uint32_t framerate_fps) {
Markus Handella3765182020-07-08 11:13:3286 MutexLock lock(&lock_);
Erik Språng6a7baa72019-02-26 17:31:0087 framerate_fps_ = framerate_fps;
Zhaoliang Ma528e4892021-11-12 05:05:0788 if (timing_frames_info_.size() < num_spatial_layers_) {
89 timing_frames_info_.resize(num_spatial_layers_);
Erik Språng6a7baa72019-02-26 17:31:0090 }
Zhaoliang Ma528e4892021-11-12 05:05:0791 for (size_t i = 0; i < num_spatial_layers_; ++i) {
Erik Språng6a7baa72019-02-26 17:31:0092 timing_frames_info_[i].target_bitrate_bytes_per_sec =
93 bitrate_allocation.GetSpatialLayerSum(i) / 8;
94 }
95}
96
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:3697void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
Markus Handella3765182020-07-08 11:13:3298 MutexLock lock(&lock_);
Erik Språng6a7baa72019-02-26 17:31:0099
Zhaoliang Ma528e4892021-11-12 05:05:07100 timing_frames_info_.resize(num_spatial_layers_);
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36101 FrameMetadata metadata;
102 metadata.rtp_timestamp = frame.timestamp();
103 metadata.encode_start_time_ms = rtc::TimeMillis();
104 metadata.ntp_time_ms = frame.ntp_time_ms();
105 metadata.timestamp_us = frame.timestamp_us();
106 metadata.rotation = frame.rotation();
107 metadata.color_space = frame.color_space();
Chen Xingf00bf422019-06-20 08:05:55108 metadata.packet_infos = frame.packet_infos();
Zhaoliang Ma528e4892021-11-12 05:05:07109 for (size_t si = 0; si < num_spatial_layers_; ++si) {
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36110 RTC_DCHECK(timing_frames_info_[si].frames.empty() ||
111 rtc::TimeDiff(
112 frame.render_time_ms(),
113 timing_frames_info_[si].frames.back().timestamp_us / 1000) >=
114 0);
Erik Språng6a7baa72019-02-26 17:31:00115 // If stream is disabled due to low bandwidth OnEncodeStarted still will be
116 // called and have to be ignored.
117 if (timing_frames_info_[si].target_bitrate_bytes_per_sec == 0)
Ilya Nikolaevskiy9b540cb2019-12-17 09:16:53118 continue;
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36119 if (timing_frames_info_[si].frames.size() == kMaxEncodeStartTimeListSize) {
Erik Språng6a7baa72019-02-26 17:31:00120 ++stalled_encoder_logged_messages_;
121 if (stalled_encoder_logged_messages_ <= kMessagesThrottlingThreshold ||
122 stalled_encoder_logged_messages_ % kThrottleRatio == 0) {
123 RTC_LOG(LS_WARNING) << "Too many frames in the encode_start_list."
124 " Did encoder stall?";
125 if (stalled_encoder_logged_messages_ == kMessagesThrottlingThreshold) {
126 RTC_LOG(LS_WARNING)
127 << "Too many log messages. Further stalled encoder"
128 "warnings will be throttled.";
129 }
130 }
131 frame_drop_callback_->OnDroppedFrame(
132 EncodedImageCallback::DropReason::kDroppedByEncoder);
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36133 timing_frames_info_[si].frames.pop_front();
Erik Språng6a7baa72019-02-26 17:31:00134 }
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36135 timing_frames_info_[si].frames.emplace_back(metadata);
Erik Språng6a7baa72019-02-26 17:31:00136 }
137}
138
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36139void FrameEncodeMetadataWriter::FillTimingInfo(size_t simulcast_svc_idx,
140 EncodedImage* encoded_image) {
Markus Handella3765182020-07-08 11:13:32141 MutexLock lock(&lock_);
Erik Språng6a7baa72019-02-26 17:31:00142 absl::optional<size_t> outlier_frame_size;
143 absl::optional<int64_t> encode_start_ms;
144 uint8_t timing_flags = VideoSendTiming::kNotTriggered;
145
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36146 int64_t encode_done_ms = rtc::TimeMillis();
147
Niels Möller13d16362021-11-16 10:39:27148 encode_start_ms =
149 ExtractEncodeStartTimeAndFillMetadata(simulcast_svc_idx, encoded_image);
Erik Språng6a7baa72019-02-26 17:31:00150
151 if (timing_frames_info_.size() > simulcast_svc_idx) {
152 size_t target_bitrate =
153 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
154 if (framerate_fps_ > 0 && target_bitrate > 0) {
155 // framerate and target bitrate were reported by encoder.
156 size_t average_frame_size = target_bitrate / framerate_fps_;
157 outlier_frame_size.emplace(
158 average_frame_size *
159 codec_settings_.timing_frame_thresholds.outlier_ratio_percent / 100);
160 }
161 }
162
163 // Outliers trigger timing frames, but do not affect scheduled timing
164 // frames.
165 if (outlier_frame_size && encoded_image->size() >= *outlier_frame_size) {
166 timing_flags |= VideoSendTiming::kTriggeredBySize;
167 }
168
169 // Check if it's time to send a timing frame.
170 int64_t timing_frame_delay_ms =
171 encoded_image->capture_time_ms_ - last_timing_frame_time_ms_;
172 // Trigger threshold if it's a first frame, too long passed since the last
173 // timing frame, or we already sent timing frame on a different simulcast
174 // stream with the same capture time.
175 if (last_timing_frame_time_ms_ == -1 ||
176 timing_frame_delay_ms >=
177 codec_settings_.timing_frame_thresholds.delay_ms ||
178 timing_frame_delay_ms == 0) {
179 timing_flags |= VideoSendTiming::kTriggeredByTimer;
180 last_timing_frame_time_ms_ = encoded_image->capture_time_ms_;
181 }
182
Erik Språng6a7baa72019-02-26 17:31:00183 // If encode start is not available that means that encoder uses internal
184 // source. In that case capture timestamp may be from a different clock with a
185 // drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
186 // because to being sent in the network capture time required to be less than
187 // all the other timestamps.
188 if (encode_start_ms) {
189 encoded_image->SetEncodeTime(*encode_start_ms, encode_done_ms);
190 encoded_image->timing_.flags = timing_flags;
191 } else {
192 encoded_image->timing_.flags = VideoSendTiming::kInvalid;
193 }
194}
195
Danil Chapovalov6d008a82020-07-22 17:49:36196void FrameEncodeMetadataWriter::UpdateBitstream(
Mirta Dvornicic28f0eb22019-05-28 14:30:16197 const CodecSpecificInfo* codec_specific_info,
Mirta Dvornicic28f0eb22019-05-28 14:30:16198 EncodedImage* encoded_image) {
199 if (!codec_specific_info ||
Danil Chapovalov6d008a82020-07-22 17:49:36200 codec_specific_info->codecType != kVideoCodecH264 ||
Mirta Dvornicic28f0eb22019-05-28 14:30:16201 encoded_image->_frameType != VideoFrameType::kVideoFrameKey) {
Danil Chapovalov6d008a82020-07-22 17:49:36202 return;
Mirta Dvornicic28f0eb22019-05-28 14:30:16203 }
204
Mirta Dvornicic28f0eb22019-05-28 14:30:16205 // Make sure that the data is not copied if owned by EncodedImage.
206 const EncodedImage& buffer = *encoded_image;
Danil Chapovalov6d008a82020-07-22 17:49:36207 rtc::Buffer modified_buffer =
Hua, Chunbo9345dcf2020-09-22 02:34:56208 SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(
Danil Chapovalov6d008a82020-07-22 17:49:36209 buffer, encoded_image->ColorSpace());
Mirta Dvornicic28f0eb22019-05-28 14:30:16210
Niels Möller4d504c72019-06-18 13:56:56211 encoded_image->SetEncodedData(
Tomas Gunnarssonc1d58912021-04-22 17:21:43212 rtc::make_ref_counted<EncodedImageBufferWrapper>(
Niels Möller4d504c72019-06-18 13:56:56213 std::move(modified_buffer)));
Mirta Dvornicic28f0eb22019-05-28 14:30:16214}
215
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36216void FrameEncodeMetadataWriter::Reset() {
Markus Handella3765182020-07-08 11:13:32217 MutexLock lock(&lock_);
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:14218 for (auto& info : timing_frames_info_) {
219 info.frames.clear();
220 }
Erik Språng6a7baa72019-02-26 17:31:00221 last_timing_frame_time_ms_ = -1;
222 reordered_frames_logged_messages_ = 0;
223 stalled_encoder_logged_messages_ = 0;
224}
225
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36226absl::optional<int64_t>
227FrameEncodeMetadataWriter::ExtractEncodeStartTimeAndFillMetadata(
Erik Språng6a7baa72019-02-26 17:31:00228 size_t simulcast_svc_idx,
229 EncodedImage* encoded_image) {
230 absl::optional<int64_t> result;
231 size_t num_simulcast_svc_streams = timing_frames_info_.size();
232 if (simulcast_svc_idx < num_simulcast_svc_streams) {
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36233 auto metadata_list = &timing_frames_info_[simulcast_svc_idx].frames;
Erik Språng6a7baa72019-02-26 17:31:00234 // Skip frames for which there was OnEncodeStarted but no OnEncodedImage
235 // call. These are dropped by encoder internally.
236 // Because some hardware encoders don't preserve capture timestamp we
237 // use RTP timestamps here.
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36238 while (!metadata_list->empty() &&
Danil Chapovalov9c584832023-09-18 13:48:49239 IsNewerTimestamp(encoded_image->RtpTimestamp(),
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36240 metadata_list->front().rtp_timestamp)) {
Erik Språng6a7baa72019-02-26 17:31:00241 frame_drop_callback_->OnDroppedFrame(
242 EncodedImageCallback::DropReason::kDroppedByEncoder);
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36243 metadata_list->pop_front();
Erik Språng6a7baa72019-02-26 17:31:00244 }
Ilya Nikolaevskiyba96e2f2019-06-04 13:15:14245
246 encoded_image->content_type_ =
247 (codec_settings_.mode == VideoCodecMode::kScreensharing)
248 ? VideoContentType::SCREENSHARE
249 : VideoContentType::UNSPECIFIED;
250
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36251 if (!metadata_list->empty() &&
Danil Chapovalov9c584832023-09-18 13:48:49252 metadata_list->front().rtp_timestamp == encoded_image->RtpTimestamp()) {
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36253 result.emplace(metadata_list->front().encode_start_time_ms);
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36254 encoded_image->capture_time_ms_ =
255 metadata_list->front().timestamp_us / 1000;
256 encoded_image->ntp_time_ms_ = metadata_list->front().ntp_time_ms;
257 encoded_image->rotation_ = metadata_list->front().rotation;
258 encoded_image->SetColorSpace(metadata_list->front().color_space);
Chen Xingf00bf422019-06-20 08:05:55259 encoded_image->SetPacketInfos(metadata_list->front().packet_infos);
Ilya Nikolaevskiy2ebf5232019-05-13 14:13:36260 metadata_list->pop_front();
Erik Språng6a7baa72019-02-26 17:31:00261 } else {
262 ++reordered_frames_logged_messages_;
263 if (reordered_frames_logged_messages_ <= kMessagesThrottlingThreshold ||
264 reordered_frames_logged_messages_ % kThrottleRatio == 0) {
265 RTC_LOG(LS_WARNING) << "Frame with no encode started time recordings. "
266 "Encoder may be reordering frames "
267 "or not preserving RTP timestamps.";
268 if (reordered_frames_logged_messages_ == kMessagesThrottlingThreshold) {
269 RTC_LOG(LS_WARNING) << "Too many log messages. Further frames "
270 "reordering warnings will be throttled.";
271 }
272 }
273 }
274 }
275 return result;
276}
277
Erik Språng6a7baa72019-02-26 17:31:00278} // namespace webrtc