Revert "Add spatial index to EncodedImage."
This reverts commit da0898dfae3b0a013ca8ad3828e9adfdc749748d.
Reason for revert: Broke downstream tests.
Original change's description:
> Add spatial index to EncodedImage.
>
> Replaces the VP8 simulcast index and VP9 spatial index formely part of
> CodecSpecificInfo.
>
> Bug: webrtc:9378
> Change-Id: I80eafd63fbdee0a25864338196a690628b4bd3d2
> Reviewed-on: https://webrtc-review.googlesource.com/83161
> Commit-Queue: Niels Moller <nisse@webrtc.org>
> Reviewed-by: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Sebastian Jansson <srte@webrtc.org>
> Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
> Reviewed-by: Philip Eliasson <philipel@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#24485}
TBR=brandtr@webrtc.org,magjed@webrtc.org,nisse@webrtc.org,sprang@webrtc.org,philipel@webrtc.org,srte@webrtc.org
Change-Id: Idb4fb9d72e5574d7353c631cb404a1311f3fd148
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: webrtc:9378
Reviewed-on: https://webrtc-review.googlesource.com/96664
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24486}diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc
index 0dbe183..608688b 100644
--- a/call/rtp_payload_params.cc
+++ b/call/rtp_payload_params.cc
@@ -22,7 +22,6 @@
namespace {
void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
- absl::optional<int> spatial_index,
RTPVideoHeader* rtp) {
rtp->codec = info.codecType;
switch (info.codecType) {
@@ -32,7 +31,7 @@
rtp->vp8().temporalIdx = info.codecSpecific.VP8.temporalIdx;
rtp->vp8().layerSync = info.codecSpecific.VP8.layerSync;
rtp->vp8().keyIdx = info.codecSpecific.VP8.keyIdx;
- rtp->simulcastIdx = spatial_index.value_or(0);
+ rtp->simulcastIdx = info.codecSpecific.VP8.simulcastIdx;
return;
}
case kVideoCodecVP9: {
@@ -45,16 +44,13 @@
vp9_header.non_ref_for_inter_layer_pred =
info.codecSpecific.VP9.non_ref_for_inter_layer_pred;
vp9_header.temporal_idx = info.codecSpecific.VP9.temporal_idx;
+ vp9_header.spatial_idx = info.codecSpecific.VP9.spatial_idx;
vp9_header.temporal_up_switch = info.codecSpecific.VP9.temporal_up_switch;
vp9_header.inter_layer_predicted =
info.codecSpecific.VP9.inter_layer_predicted;
vp9_header.gof_idx = info.codecSpecific.VP9.gof_idx;
vp9_header.num_spatial_layers = info.codecSpecific.VP9.num_spatial_layers;
- if (vp9_header.num_spatial_layers > 1) {
- vp9_header.spatial_idx = spatial_index.value_or(kNoSpatialIdx);
- } else {
- vp9_header.spatial_idx = kNoSpatialIdx;
- }
+
if (info.codecSpecific.VP9.ss_data_available) {
vp9_header.spatial_layer_resolution_present =
info.codecSpecific.VP9.spatial_layer_resolution_present;
@@ -79,13 +75,13 @@
auto& h264_header = rtp->video_type_header.emplace<RTPVideoHeaderH264>();
h264_header.packetization_mode =
info.codecSpecific.H264.packetization_mode;
- rtp->simulcastIdx = spatial_index.value_or(0);
+ rtp->simulcastIdx = info.codecSpecific.H264.simulcast_idx;
return;
}
case kVideoCodecMultiplex:
case kVideoCodecGeneric:
rtp->codec = kVideoCodecGeneric;
- rtp->simulcastIdx = spatial_index.value_or(0);
+ rtp->simulcastIdx = info.codecSpecific.generic.simulcast_idx;
return;
default:
return;
@@ -135,8 +131,7 @@
int64_t shared_frame_id) {
RTPVideoHeader rtp_video_header;
if (codec_specific_info) {
- PopulateRtpWithCodecSpecifics(*codec_specific_info, image.SpatialIndex(),
- &rtp_video_header);
+ PopulateRtpWithCodecSpecifics(*codec_specific_info, &rtp_video_header);
}
rtp_video_header.rotation = image.rotation_;
rtp_video_header.content_type = image.content_type_;
diff --git a/call/rtp_payload_params_unittest.cc b/call/rtp_payload_params_unittest.cc
index b0c0027..4d0b017 100644
--- a/call/rtp_payload_params_unittest.cc
+++ b/call/rtp_payload_params_unittest.cc
@@ -38,11 +38,11 @@
EncodedImage encoded_image;
encoded_image.rotation_ = kVideoRotation_90;
encoded_image.content_type_ = VideoContentType::SCREENSHARE;
- encoded_image.SetSpatialIndex(1);
CodecSpecificInfo codec_info;
memset(&codec_info, 0, sizeof(CodecSpecificInfo));
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
codec_info.codecSpecific.VP8.temporalIdx = 0;
codec_info.codecSpecific.VP8.keyIdx = kNoKeyIdx;
codec_info.codecSpecific.VP8.layerSync = false;
@@ -52,6 +52,7 @@
params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare);
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
codec_info.codecSpecific.VP8.temporalIdx = 1;
codec_info.codecSpecific.VP8.layerSync = true;
@@ -78,12 +79,13 @@
EncodedImage encoded_image;
encoded_image.rotation_ = kVideoRotation_90;
encoded_image.content_type_ = VideoContentType::SCREENSHARE;
- encoded_image.SetSpatialIndex(0);
+
CodecSpecificInfo codec_info;
memset(&codec_info, 0, sizeof(CodecSpecificInfo));
codec_info.codecType = kVideoCodecVP9;
codec_info.codecSpecific.VP9.num_spatial_layers = 3;
codec_info.codecSpecific.VP9.first_frame_in_picture = true;
+ codec_info.codecSpecific.VP9.spatial_idx = 0;
codec_info.codecSpecific.VP9.temporal_idx = 2;
codec_info.codecSpecific.VP9.end_of_picture = false;
@@ -98,7 +100,7 @@
EXPECT_EQ(kPictureId + 1, vp9_header.picture_id);
EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx);
EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx);
- EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex());
+ EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx);
EXPECT_EQ(vp9_header.num_spatial_layers,
codec_info.codecSpecific.VP9.num_spatial_layers);
EXPECT_EQ(vp9_header.end_of_picture,
@@ -106,9 +108,9 @@
// Next spatial layer.
codec_info.codecSpecific.VP9.first_frame_in_picture = false;
+ codec_info.codecSpecific.VP9.spatial_idx += 1;
codec_info.codecSpecific.VP9.end_of_picture = true;
- encoded_image.SetSpatialIndex(1);
header = params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare);
EXPECT_EQ(kVideoRotation_90, header.rotation);
@@ -117,7 +119,7 @@
EXPECT_EQ(kPictureId + 1, vp9_header.picture_id);
EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx);
EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx);
- EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex());
+ EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx);
EXPECT_EQ(vp9_header.num_spatial_layers,
codec_info.codecSpecific.VP9.num_spatial_layers);
EXPECT_EQ(vp9_header.end_of_picture,
@@ -152,6 +154,7 @@
CodecSpecificInfo codec_info;
memset(&codec_info, 0, sizeof(CodecSpecificInfo));
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
RtpPayloadParams params(kSsrc1, &state);
RTPVideoHeader header =
diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc
index a2df830..1fbd102 100644
--- a/call/rtp_video_sender.cc
+++ b/call/rtp_video_sender.cc
@@ -95,6 +95,21 @@
return modules;
}
+absl::optional<size_t> GetSimulcastIdx(const CodecSpecificInfo* info) {
+ if (!info)
+ return absl::nullopt;
+ switch (info->codecType) {
+ case kVideoCodecVP8:
+ return absl::optional<size_t>(info->codecSpecific.VP8.simulcastIdx);
+ case kVideoCodecH264:
+ return absl::optional<size_t>(info->codecSpecific.H264.simulcast_idx);
+ case kVideoCodecMultiplex:
+ case kVideoCodecGeneric:
+ return absl::optional<size_t>(info->codecSpecific.generic.simulcast_idx);
+ default:
+ return absl::nullopt;
+ }
+}
bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
const VideoCodecType codecType = PayloadStringToCodecType(payload_name);
if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) {
@@ -305,14 +320,7 @@
return Result(Result::ERROR_SEND_FAILED);
shared_frame_id_++;
- size_t stream_index = 0;
- if (codec_specific_info &&
- (codec_specific_info->codecType == kVideoCodecVP8 ||
- codec_specific_info->codecType == kVideoCodecH264 ||
- codec_specific_info->codecType == kVideoCodecGeneric)) {
- // Map spatial index to simulcast.
- stream_index = encoded_image.SpatialIndex().value_or(0);
- }
+ size_t stream_index = GetSimulcastIdx(codec_specific_info).value_or(0);
RTC_DCHECK_LT(stream_index, rtp_modules_.size());
RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader(
encoded_image, codec_specific_info, shared_frame_id_);
diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc
index bd8a6a6..9e3290d 100644
--- a/call/rtp_video_sender_unittest.cc
+++ b/call/rtp_video_sender_unittest.cc
@@ -169,41 +169,44 @@
TEST(RtpVideoSenderTest, SendSimulcastSetActive) {
uint8_t payload = 'a';
- EncodedImage encoded_image_1;
- encoded_image_1.SetTimestamp(1);
- encoded_image_1.capture_time_ms_ = 2;
- encoded_image_1._frameType = kVideoFrameKey;
- encoded_image_1._buffer = &payload;
- encoded_image_1._length = 1;
+ EncodedImage encoded_image;
+ encoded_image.SetTimestamp(1);
+ encoded_image.capture_time_ms_ = 2;
+ encoded_image._frameType = kVideoFrameKey;
+ encoded_image._buffer = &payload;
+ encoded_image._length = 1;
RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {});
- CodecSpecificInfo codec_info;
- memset(&codec_info, 0, sizeof(CodecSpecificInfo));
- codec_info.codecType = kVideoCodecVP8;
+ CodecSpecificInfo codec_info_1;
+ memset(&codec_info_1, 0, sizeof(CodecSpecificInfo));
+ codec_info_1.codecType = kVideoCodecVP8;
+ codec_info_1.codecSpecific.VP8.simulcastIdx = 0;
test.router()->SetActive(true);
EXPECT_EQ(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
.error);
- EncodedImage encoded_image_2(encoded_image_1);
- encoded_image_2.SetSpatialIndex(1);
+ CodecSpecificInfo codec_info_2;
+ memset(&codec_info_2, 0, sizeof(CodecSpecificInfo));
+ codec_info_2.codecType = kVideoCodecVP8;
+ codec_info_2.codecSpecific.VP8.simulcastIdx = 1;
EXPECT_EQ(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
.error);
// Inactive.
test.router()->SetActive(false);
EXPECT_NE(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
.error);
EXPECT_NE(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
.error);
}
@@ -213,19 +216,22 @@
// be sent if both modules are inactive.
TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) {
uint8_t payload = 'a';
- EncodedImage encoded_image_1;
- encoded_image_1.SetTimestamp(1);
- encoded_image_1.capture_time_ms_ = 2;
- encoded_image_1._frameType = kVideoFrameKey;
- encoded_image_1._buffer = &payload;
- encoded_image_1._length = 1;
- EncodedImage encoded_image_2(encoded_image_1);
- encoded_image_2.SetSpatialIndex(1);
+ EncodedImage encoded_image;
+ encoded_image.SetTimestamp(1);
+ encoded_image.capture_time_ms_ = 2;
+ encoded_image._frameType = kVideoFrameKey;
+ encoded_image._buffer = &payload;
+ encoded_image._length = 1;
RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {});
- CodecSpecificInfo codec_info;
- memset(&codec_info, 0, sizeof(CodecSpecificInfo));
- codec_info.codecType = kVideoCodecVP8;
+ CodecSpecificInfo codec_info_1;
+ memset(&codec_info_1, 0, sizeof(CodecSpecificInfo));
+ codec_info_1.codecType = kVideoCodecVP8;
+ codec_info_1.codecSpecific.VP8.simulcastIdx = 0;
+ CodecSpecificInfo codec_info_2;
+ memset(&codec_info_2, 0, sizeof(CodecSpecificInfo));
+ codec_info_2.codecType = kVideoCodecVP8;
+ codec_info_2.codecSpecific.VP8.simulcastIdx = 1;
// Only setting one stream to active will still set the payload router to
// active and allow sending data on the active stream.
@@ -233,7 +239,7 @@
test.router()->SetActiveModules(active_modules);
EXPECT_EQ(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
.error);
// Setting both streams to inactive will turn the payload router to
@@ -244,11 +250,11 @@
// because the payload router is inactive.
EXPECT_NE(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
.error);
EXPECT_NE(EncodedImageCallback::Result::OK,
test.router()
- ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
+ ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
.error);
}
diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index 5827fc5..4ab83a0 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -449,14 +449,16 @@
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo,
const RTPFragmentationHeader* fragmentation) {
- EncodedImage stream_image(encodedImage);
CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
stream_codec_specific.codec_name = implementation_name_.c_str();
-
- stream_image.SetSpatialIndex(stream_idx);
+ if (stream_codec_specific.codecType == webrtc::kVideoCodecVP8) {
+ stream_codec_specific.codecSpecific.VP8.simulcastIdx = stream_idx;
+ } else if (stream_codec_specific.codecType == webrtc::kVideoCodecH264) {
+ stream_codec_specific.codecSpecific.H264.simulcast_idx = stream_idx;
+ }
return encoded_complete_callback_->OnEncodedImage(
- stream_image, &stream_codec_specific, fragmentation);
+ encodedImage, &stream_codec_specific, fragmentation);
}
void SimulcastEncoderAdapter::PopulateStreamCodec(
diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc
index 3999840..410b85e 100644
--- a/media/engine/simulcast_encoder_adapter_unittest.cc
+++ b/media/engine/simulcast_encoder_adapter_unittest.cc
@@ -339,10 +339,11 @@
const RTPFragmentationHeader* fragmentation) override {
last_encoded_image_width_ = encoded_image._encodedWidth;
last_encoded_image_height_ = encoded_image._encodedHeight;
- last_encoded_image_simulcast_index_ =
- encoded_image.SpatialIndex().value_or(-1);
-
- return Result(Result::OK, encoded_image._timeStamp);
+ if (codec_specific_info) {
+ last_encoded_image_simulcast_index_ =
+ codec_specific_info->codecSpecific.VP8.simulcastIdx;
+ }
+ return Result(Result::OK, encoded_image.Timestamp());
}
bool GetLastEncodedImageInfo(int* out_width,
diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
index 4effcdb..4fc9b4f 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
@@ -506,7 +506,6 @@
: VideoContentType::UNSPECIFIED;
encoded_images_[i].timing_.flags = VideoSendTiming::kInvalid;
encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
- encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
// Split encoded image up into fragments. This also updates
// |encoded_image_|.
@@ -527,6 +526,8 @@
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode =
packetization_mode_;
+ codec_specific.codecSpecific.H264.simulcast_idx =
+ configurations_[i].simulcast_idx;
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
&codec_specific, &frag_header);
}
diff --git a/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc b/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc
index c3a7506..4733b3a 100644
--- a/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc
+++ b/modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc
@@ -285,6 +285,7 @@
CodecSpecificInfo codec_info = *codecSpecificInfo;
codec_info.codecType = kVideoCodecMultiplex;
+ codec_info.codecSpecific.generic.simulcast_idx = 0;
encoded_complete_callback_->OnEncodedImage(combined_image_, &codec_info,
fragmentation);
}
diff --git a/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc b/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc
index 56de138..303914d 100644
--- a/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc
+++ b/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc
@@ -232,7 +232,7 @@
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
- EXPECT_FALSE(encoded_frame.SpatialIndex());
+ EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx);
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
@@ -252,7 +252,7 @@
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
- EXPECT_FALSE(encoded_frame.SpatialIndex());
+ EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx);
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc
index 75605ae..ccf7053 100644
--- a/modules/video_coding/codecs/test/videoprocessor.cc
+++ b/modules/video_coding/codecs/test/videoprocessor.cc
@@ -56,17 +56,22 @@
return max_size;
}
-size_t GetTemporalLayerIndex(const CodecSpecificInfo& codec_specific) {
- size_t temporal_idx = 0;
+void GetLayerIndices(const CodecSpecificInfo& codec_specific,
+ size_t* spatial_idx,
+ size_t* temporal_idx) {
if (codec_specific.codecType == kVideoCodecVP8) {
- temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
+ *spatial_idx = codec_specific.codecSpecific.VP8.simulcastIdx;
+ *temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
} else if (codec_specific.codecType == kVideoCodecVP9) {
- temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
+ *spatial_idx = codec_specific.codecSpecific.VP9.spatial_idx;
+ *temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
}
- if (temporal_idx == kNoTemporalIdx) {
- temporal_idx = 0;
+ if (*spatial_idx == kNoSpatialIdx) {
+ *spatial_idx = 0;
}
- return temporal_idx;
+ if (*temporal_idx == kNoTemporalIdx) {
+ *temporal_idx = 0;
+ }
}
int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
@@ -342,8 +347,9 @@
}
// Layer metadata.
- size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
- size_t temporal_idx = GetTemporalLayerIndex(codec_specific);
+ size_t spatial_idx = 0;
+ size_t temporal_idx = 0;
+ GetLayerIndices(codec_specific, &spatial_idx, &temporal_idx);
FrameStatistics* frame_stat =
stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx);
diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
index ef46fd2..3826e14 100644
--- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
+++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
@@ -817,6 +817,7 @@
codec_specific->codecType = kVideoCodecVP8;
codec_specific->codec_name = ImplementationName();
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
+ vp8Info->simulcastIdx = stream_idx;
vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
temporal_layers_[stream_idx]->PopulateCodecSpecific(
@@ -875,7 +876,6 @@
encoded_images_[encoder_idx]._frameType = kVideoFrameKey;
is_keyframe = true;
}
- encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt,
stream_idx, input_image.timestamp());
break;
diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
index dc55d17..aaa10dd 100644
--- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
@@ -70,7 +70,7 @@
VerifyQpParser(*encoded_frame);
EXPECT_STREQ("libvpx", codec_specific_info->codec_name);
EXPECT_EQ(kVideoCodecVP8, codec_specific_info->codecType);
- EXPECT_EQ(0, encoded_frame->SpatialIndex());
+ EXPECT_EQ(0u, codec_specific_info->codecSpecific.VP8.simulcastIdx);
}
void EncodeAndExpectFrameWith(const VideoFrame& input_frame,
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index 0f5855f..f6594c1 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -356,7 +356,7 @@
encoder_->Encode(*NextInputFrame(), nullptr, nullptr));
ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific));
- EXPECT_FALSE(frames[0].SpatialIndex());
+ EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, kNoSpatialIdx);
EXPECT_TRUE(codec_specific[0].codecSpecific.VP9.end_of_picture);
}
@@ -395,7 +395,7 @@
// Key frame.
EXPECT_FALSE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted);
- EXPECT_EQ(frames[0].SpatialIndex(), 0);
+ EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, 0);
EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred,
inter_layer_pred == InterLayerPredMode::kOff);
EXPECT_TRUE(
@@ -408,7 +408,7 @@
// Delta frame.
EXPECT_TRUE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted);
- EXPECT_EQ(frames[0].SpatialIndex(), 0);
+ EXPECT_EQ(codec_specific[0].codecSpecific.VP9.spatial_idx, 0);
EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred,
inter_layer_pred == InterLayerPredMode::kOff ||
inter_layer_pred == InterLayerPredMode::kOnKeyPic);
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc
index e6ed0a8..b4c13f9 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -754,7 +754,6 @@
}
void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
- absl::optional<int>* spatial_idx,
const vpx_codec_cx_pkt& pkt,
uint32_t timestamp,
bool first_frame_in_picture) {
@@ -781,9 +780,9 @@
}
if (num_active_spatial_layers_ == 1) {
RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
- *spatial_idx = absl::nullopt;
+ vp9_info->spatial_idx = kNoSpatialIdx;
} else {
- *spatial_idx = layer_id.spatial_layer_id;
+ vp9_info->spatial_idx = layer_id.spatial_layer_id;
}
if (layer_id.spatial_layer_id != 0) {
vp9_info->ss_data_available = false;
@@ -1022,10 +1021,8 @@
RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size);
memset(&codec_specific_, 0, sizeof(codec_specific_));
- absl::optional<int> spatial_index;
- PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt,
- input_image_->timestamp(), first_frame_in_picture);
- encoded_image_.SetSpatialIndex(spatial_index);
+ PopulateCodecSpecific(&codec_specific_, *pkt, input_image_->timestamp(),
+ first_frame_in_picture);
if (is_flexible_mode_) {
UpdateReferenceBuffers(*pkt, pics_since_key_);
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h
index 4deade3..d448306 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.h
+++ b/modules/video_coding/codecs/vp9/vp9_impl.h
@@ -61,7 +61,6 @@
int InitAndSetControlSettings(const VideoCodec* inst);
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
- absl::optional<int>* spatial_idx,
const vpx_codec_cx_pkt& pkt,
uint32_t timestamp,
bool first_frame_in_picture);
diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc
index 37914e4..c53a737 100644
--- a/modules/video_coding/encoded_frame.cc
+++ b/modules/video_coding/encoded_frame.cc
@@ -82,6 +82,7 @@
if (_codecSpecificInfo.codecType != kVideoCodecVP9) {
// This is the first packet for this frame.
_codecSpecificInfo.codecSpecific.VP9.temporal_idx = 0;
+ _codecSpecificInfo.codecSpecific.VP9.spatial_idx = 0;
_codecSpecificInfo.codecSpecific.VP9.gof_idx = 0;
_codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted = false;
_codecSpecificInfo.codecType = kVideoCodecVP9;
@@ -105,6 +106,8 @@
vp9_header.temporal_up_switch;
}
if (vp9_header.spatial_idx != kNoSpatialIdx) {
+ _codecSpecificInfo.codecSpecific.VP9.spatial_idx =
+ vp9_header.spatial_idx;
_codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted =
vp9_header.inter_layer_predicted;
}
diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc
index b1d7c28..a8999fc 100644
--- a/modules/video_coding/generic_encoder.cc
+++ b/modules/video_coding/generic_encoder.cc
@@ -391,10 +391,21 @@
const RTPFragmentationHeader* fragmentation_header) {
TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
"timestamp", encoded_image.Timestamp());
- const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
+ size_t simulcast_svc_idx = 0;
+ if (codec_specific->codecType == kVideoCodecVP9) {
+ if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1)
+ simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx;
+ } else if (codec_specific->codecType == kVideoCodecVP8) {
+ simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx;
+ } else if (codec_specific->codecType == kVideoCodecGeneric) {
+ simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx;
+ } else if (codec_specific->codecType == kVideoCodecH264) {
+ // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here.
+ }
+
EncodedImage image_copy(encoded_image);
- FillTimingInfo(spatial_idx, &image_copy);
+ FillTimingInfo(simulcast_svc_idx, &image_copy);
// Piggyback ALR experiment group id and simulcast id into the content type.
uint8_t experiment_id =
@@ -410,7 +421,7 @@
// id in content type to +1 of that is actual simulcast index. This is because
// value 0 on the wire is reserved for 'no simulcast stream specified'.
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
- &image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
+ &image_copy.content_type_, static_cast<uint8_t>(simulcast_svc_idx + 1)));
Result result = post_encode_callback_->OnEncodedImage(
image_copy, codec_specific, fragmentation_header);
diff --git a/modules/video_coding/generic_encoder_unittest.cc b/modules/video_coding/generic_encoder_unittest.cc
index 2be6856..c889769 100644
--- a/modules/video_coding/generic_encoder_unittest.cc
+++ b/modules/video_coding/generic_encoder_unittest.cc
@@ -95,8 +95,8 @@
image._length = FrameSize(min_frame_size, max_frame_size, s, i);
image.capture_time_ms_ = current_timestamp;
image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90));
- image.SetSpatialIndex(s);
codec_specific.codecType = kVideoCodecGeneric;
+ codec_specific.codecSpecific.generic.simulcast_idx = s;
callback.OnEncodeStarted(static_cast<uint32_t>(current_timestamp * 90),
current_timestamp, s);
if (dropped) {
@@ -189,6 +189,7 @@
image.capture_time_ms_ = timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
+ codec_specific.codecSpecific.generic.simulcast_idx = 0;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink, nullptr);
VideoCodec::TimingFrameTriggerThresholds thresholds;
@@ -220,6 +221,7 @@
image.capture_time_ms_ = timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
+ codec_specific.codecSpecific.generic.simulcast_idx = 0;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink, nullptr);
callback.SetInternalSource(true);
@@ -255,6 +257,7 @@
const int64_t kTimestampMs3 = 47721860;
const int64_t kTimestampMs4 = 47721870;
codec_specific.codecType = kVideoCodecGeneric;
+ codec_specific.codecSpecific.generic.simulcast_idx = 0;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink, nullptr);
// Any non-zero bitrate needed to be set before the first frame.
@@ -290,6 +293,7 @@
CodecSpecificInfo codec_specific;
const int64_t kTimestampMs = 123456;
codec_specific.codecType = kVideoCodecGeneric;
+ codec_specific.codecSpecific.generic.simulcast_idx = 0;
FakeEncodedImageCallback sink;
VCMEncodedFrameCallback callback(&sink, nullptr);
// Any non-zero bitrate needed to be set before the first frame.
diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h
index 94d4271..9108625 100644
--- a/modules/video_coding/include/video_codec_interface.h
+++ b/modules/video_coding/include/video_codec_interface.h
@@ -28,8 +28,6 @@
// with a copy-constructor. See below.
struct CodecSpecificInfoVP8 {
bool nonReference;
- // TODO(bugs.webrtc.org/9378): Delete simulcastIdx, replaced by spatial index
- // member in EncodedImage. Unused, but assigned in downstream code.
uint8_t simulcastIdx;
uint8_t temporalIdx;
bool layerSync;
@@ -45,8 +43,6 @@
bool non_ref_for_inter_layer_pred;
uint8_t temporal_idx;
- // TODO(bugs.webrtc.org/9378): Delete spatial_idx, replaced by spatial index
- // member in EncodedImage. Unused, but assigned in downstream code.
uint8_t spatial_idx;
bool temporal_up_switch;
bool inter_layer_predicted; // Frame is dependent on directly lower spatial
@@ -67,14 +63,13 @@
bool end_of_picture;
};
-// TODO(bugs.webrtc.org/9378): Delete this struct. Unused, except that
-// simulcast_idx is assigned in downstream code.
struct CodecSpecificInfoGeneric {
uint8_t simulcast_idx;
};
struct CodecSpecificInfoH264 {
H264PacketizationMode packetization_mode;
+ uint8_t simulcast_idx;
};
union CodecSpecificInfoUnion {
diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
index 4af526c..03de176 100644
--- a/modules/video_coding/utility/simulcast_test_fixture_impl.cc
+++ b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
@@ -76,9 +76,15 @@
virtual Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) {
+ uint16_t simulcast_idx = 0;
bool is_vp8 = (codec_specific_info->codecType == kVideoCodecVP8);
+ if (is_vp8) {
+ simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx;
+ } else {
+ simulcast_idx = codec_specific_info->codecSpecific.H264.simulcast_idx;
+ }
// Only store the base layer.
- if (encoded_image.SpatialIndex().value_or(0) == 0) {
+ if (simulcast_idx) {
if (encoded_image._frameType == kVideoFrameKey) {
delete[] encoded_key_frame_._buffer;
encoded_key_frame_._buffer = new uint8_t[encoded_image._size];
@@ -98,9 +104,9 @@
}
}
if (is_vp8) {
- layer_sync_[encoded_image.SpatialIndex().value_or(0)] =
+ layer_sync_[codec_specific_info->codecSpecific.VP8.simulcastIdx] =
codec_specific_info->codecSpecific.VP8.layerSync;
- temporal_layer_[encoded_image.SpatialIndex().value_or(0)] =
+ temporal_layer_[codec_specific_info->codecSpecific.VP8.simulcastIdx] =
codec_specific_info->codecSpecific.VP8.temporalIdx;
}
return Result(Result::OK, encoded_image.Timestamp());
diff --git a/sdk/android/src/jni/androidmediaencoder.cc b/sdk/android/src/jni/androidmediaencoder.cc
index 435bc71..35608c8 100644
--- a/sdk/android/src/jni/androidmediaencoder.cc
+++ b/sdk/android/src/jni/androidmediaencoder.cc
@@ -1008,6 +1008,7 @@
info.codecType = codec_type;
if (codec_type == kVideoCodecVP8) {
info.codecSpecific.VP8.nonReference = false;
+ info.codecSpecific.VP8.simulcastIdx = 0;
info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx;
info.codecSpecific.VP8.layerSync = false;
info.codecSpecific.VP8.keyIdx = kNoKeyIdx;
@@ -1019,6 +1020,7 @@
info.codecSpecific.VP9.flexible_mode = false;
info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx;
+ info.codecSpecific.VP9.spatial_idx = kNoSpatialIdx;
info.codecSpecific.VP9.temporal_up_switch = true;
info.codecSpecific.VP9.inter_layer_predicted = false;
info.codecSpecific.VP9.gof_idx =
diff --git a/sdk/android/src/jni/videoencoderwrapper.cc b/sdk/android/src/jni/videoencoderwrapper.cc
index c50e9f6..ff8be05 100644
--- a/sdk/android/src/jni/videoencoderwrapper.cc
+++ b/sdk/android/src/jni/videoencoderwrapper.cc
@@ -390,6 +390,7 @@
switch (codec_settings_.codecType) {
case kVideoCodecVP8:
info.codecSpecific.VP8.nonReference = false;
+ info.codecSpecific.VP8.simulcastIdx = 0;
info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx;
info.codecSpecific.VP8.layerSync = false;
info.codecSpecific.VP8.keyIdx = kNoKeyIdx;
@@ -402,6 +403,7 @@
info.codecSpecific.VP9.flexible_mode = false;
info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx;
+ info.codecSpecific.VP9.spatial_idx = kNoSpatialIdx;
info.codecSpecific.VP9.temporal_up_switch = true;
info.codecSpecific.VP9.inter_layer_predicted = false;
info.codecSpecific.VP9.gof_idx =
diff --git a/test/fake_encoder.cc b/test/fake_encoder.cc
index 2075399..b8d04e6 100644
--- a/test/fake_encoder.cc
+++ b/test/fake_encoder.cc
@@ -102,6 +102,7 @@
CodecSpecificInfo specifics;
memset(&specifics, 0, sizeof(specifics));
specifics.codecType = kVideoCodecGeneric;
+ specifics.codecSpecific.generic.simulcast_idx = i;
std::unique_ptr<uint8_t[]> encoded_buffer(
new uint8_t[frame_info.layers[i].size]);
memcpy(encoded_buffer.get(), encoded_buffer_, frame_info.layers[i].size);
@@ -117,7 +118,6 @@
encoded.content_type_ = (mode == VideoCodecMode::kScreensharing)
? VideoContentType::SCREENSHARE
: VideoContentType::UNSPECIFIED;
- encoded.SetSpatialIndex(i);
specifics.codec_name = ImplementationName();
if (callback->OnEncodedImage(encoded, &specifics, nullptr).error !=
EncodedImageCallback::Result::OK) {
diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc
index 04dff00..9d05297 100644
--- a/test/fake_vp8_encoder.cc
+++ b/test/fake_vp8_encoder.cc
@@ -88,6 +88,7 @@
codec_specific->codecType = kVideoCodecVP8;
codec_specific->codec_name = ImplementationName();
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
+ vp8Info->simulcastIdx = stream_idx;
vp8Info->keyIdx = kNoKeyIdx;
vp8Info->nonReference = false;
temporal_layers_[stream_idx]->PopulateCodecSpecific(
@@ -99,7 +100,7 @@
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragments) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
- uint8_t stream_idx = encoded_image.SpatialIndex().value_or(0);
+ uint8_t stream_idx = codec_specific_info->codecSpecific.generic.simulcast_idx;
CodecSpecificInfo overrided_specific_info;
TemporalLayers::FrameConfig tl_config =
temporal_layers_[stream_idx]->UpdateLayerConfig(encoded_image._timeStamp);
diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc
index 0057933..2ab025a 100644
--- a/video/send_statistics_proxy.cc
+++ b/video/send_statistics_proxy.cc
@@ -81,9 +81,9 @@
kVideoMax);
}
-bool IsForcedFallbackPossible(const CodecSpecificInfo* codec_info,
- int simulcast_index) {
- return codec_info->codecType == kVideoCodecVP8 && simulcast_index == 0 &&
+bool IsForcedFallbackPossible(const CodecSpecificInfo* codec_info) {
+ return codec_info->codecType == kVideoCodecVP8 &&
+ codec_info->codecSpecific.VP8.simulcastIdx == 0 &&
(codec_info->codecSpecific.VP8.temporalIdx == 0 ||
codec_info->codecSpecific.VP8.temporalIdx == kNoTemporalIdx);
}
@@ -219,7 +219,7 @@
sent_height_counter_.Add(it->second.max_height);
// Check number of encoded streams per timestamp.
- if (num_streams_ > static_cast<size_t>(it->second.max_simulcast_idx)) {
+ if (num_streams_ > it->second.max_simulcast_idx) {
*is_limited_in_resolution = false;
if (num_streams_ > 1) {
int disabled_streams =
@@ -241,7 +241,7 @@
bool SendStatisticsProxy::UmaSamplesContainer::InsertEncodedFrame(
const EncodedImage& encoded_frame,
- int simulcast_idx,
+ size_t simulcast_idx,
bool* is_limited_in_resolution) {
int64_t now_ms = clock_->TimeInMilliseconds();
RemoveOld(now_ms, is_limited_in_resolution);
@@ -806,15 +806,14 @@
void SendStatisticsProxy::UpdateEncoderFallbackStats(
const CodecSpecificInfo* codec_info,
- int pixels,
- int simulcast_index) {
- UpdateFallbackDisabledStats(codec_info, pixels, simulcast_index);
+ int pixels) {
+ UpdateFallbackDisabledStats(codec_info, pixels);
if (!fallback_max_pixels_ || !uma_container_->fallback_info_.is_possible) {
return;
}
- if (!IsForcedFallbackPossible(codec_info, simulcast_index)) {
+ if (!IsForcedFallbackPossible(codec_info)) {
uma_container_->fallback_info_.is_possible = false;
return;
}
@@ -856,15 +855,14 @@
void SendStatisticsProxy::UpdateFallbackDisabledStats(
const CodecSpecificInfo* codec_info,
- int pixels,
- int simulcast_index) {
+ int pixels) {
if (!fallback_max_pixels_disabled_ ||
!uma_container_->fallback_info_disabled_.is_possible ||
stats_.has_entered_low_resolution) {
return;
}
- if (!IsForcedFallbackPossible(codec_info, simulcast_index) ||
+ if (!IsForcedFallbackPossible(codec_info) ||
strcmp(codec_info->codec_name, kVp8SwCodecName) == 0) {
uma_container_->fallback_info_disabled_.is_possible = false;
return;
@@ -884,27 +882,26 @@
void SendStatisticsProxy::OnSendEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_info) {
- // Simulcast is used for VP8, H264 and Generic.
- int simulcast_idx =
- (codec_info && (codec_info->codecType == kVideoCodecVP8 ||
- codec_info->codecType == kVideoCodecH264 ||
- codec_info->codecType == kVideoCodecGeneric))
- ? encoded_image.SpatialIndex().value_or(0)
- : 0;
+ size_t simulcast_idx = 0;
rtc::CritScope lock(&crit_);
++stats_.frames_encoded;
if (codec_info) {
+ if (codec_info->codecType == kVideoCodecVP8) {
+ simulcast_idx = codec_info->codecSpecific.VP8.simulcastIdx;
+ } else if (codec_info->codecType == kVideoCodecH264) {
+ simulcast_idx = codec_info->codecSpecific.H264.simulcast_idx;
+ } else if (codec_info->codecType == kVideoCodecGeneric) {
+ simulcast_idx = codec_info->codecSpecific.generic.simulcast_idx;
+ }
if (codec_info->codec_name) {
- UpdateEncoderFallbackStats(
- codec_info,
- encoded_image._encodedWidth * encoded_image._encodedHeight,
- simulcast_idx);
+ UpdateEncoderFallbackStats(codec_info, encoded_image._encodedWidth *
+ encoded_image._encodedHeight);
stats_.encoder_implementation_name = codec_info->codec_name;
}
}
- if (static_cast<size_t>(simulcast_idx) >= rtp_config_.ssrcs.size()) {
+ if (simulcast_idx >= rtp_config_.ssrcs.size()) {
RTC_LOG(LS_ERROR) << "Encoded image outside simulcast range ("
<< simulcast_idx << " >= " << rtp_config_.ssrcs.size()
<< ").";
@@ -931,13 +928,20 @@
if (codec_info) {
if (codec_info->codecType == kVideoCodecVP8) {
- int spatial_idx = (rtp_config_.ssrcs.size() == 1) ? -1 : simulcast_idx;
+ int spatial_idx = (rtp_config_.ssrcs.size() == 1)
+ ? -1
+ : static_cast<int>(simulcast_idx);
uma_container_->qp_counters_[spatial_idx].vp8.Add(encoded_image.qp_);
} else if (codec_info->codecType == kVideoCodecVP9) {
- int spatial_idx = encoded_image.SpatialIndex().value_or(-1);
+ int spatial_idx =
+ (codec_info->codecSpecific.VP9.num_spatial_layers == 1)
+ ? -1
+ : codec_info->codecSpecific.VP9.spatial_idx;
uma_container_->qp_counters_[spatial_idx].vp9.Add(encoded_image.qp_);
} else if (codec_info->codecType == kVideoCodecH264) {
- int spatial_idx = (rtp_config_.ssrcs.size() == 1) ? -1 : simulcast_idx;
+ int spatial_idx = (rtp_config_.ssrcs.size() == 1)
+ ? -1
+ : static_cast<int>(simulcast_idx);
uma_container_->qp_counters_[spatial_idx].h264.Add(encoded_image.qp_);
}
}
diff --git a/video/send_statistics_proxy.h b/video/send_statistics_proxy.h
index b5868d0..875348f 100644
--- a/video/send_statistics_proxy.h
+++ b/video/send_statistics_proxy.h
@@ -188,7 +188,10 @@
}
};
struct Frame {
- Frame(int64_t send_ms, uint32_t width, uint32_t height, int simulcast_idx)
+ Frame(int64_t send_ms,
+ uint32_t width,
+ uint32_t height,
+ size_t simulcast_idx)
: send_ms(send_ms),
max_width(width),
max_height(height),
@@ -197,7 +200,7 @@
send_ms; // Time when first frame with this timestamp is sent.
uint32_t max_width; // Max width with this timestamp.
uint32_t max_height; // Max height with this timestamp.
- int max_simulcast_idx; // Max simulcast index with this timestamp.
+ size_t max_simulcast_idx; // Max simulcast index with this timestamp.
};
typedef std::map<uint32_t, Frame, TimestampOlderThan> EncodedFrameMap;
@@ -215,12 +218,10 @@
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
void UpdateEncoderFallbackStats(const CodecSpecificInfo* codec_info,
- int pixels,
- int simulcast_index)
+ int pixels)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
void UpdateFallbackDisabledStats(const CodecSpecificInfo* codec_info,
- int pixels,
- int simulcast_index)
+ int pixels)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
Clock* const clock_;
@@ -256,7 +257,7 @@
void InitializeBitrateCounters(const VideoSendStream::Stats& stats);
bool InsertEncodedFrame(const EncodedImage& encoded_frame,
- int simulcast_idx,
+ size_t simulcast_idx,
bool* is_limited_in_resolution);
void RemoveOld(int64_t now_ms, bool* is_limited_in_resolution);
diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc
index 4dbfc03..d4a7876 100644
--- a/video/send_statistics_proxy_unittest.cc
+++ b/video/send_statistics_proxy_unittest.cc
@@ -36,6 +36,7 @@
const CodecSpecificInfo kDefaultCodecInfo = []() {
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
return codec_info;
}();
} // namespace
@@ -1312,10 +1313,10 @@
codec_info.codecType = kVideoCodecVP8;
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
- encoded_image.SetSpatialIndex(0);
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
encoded_image.qp_ = kQpIdx0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
- encoded_image.SetSpatialIndex(1);
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
encoded_image.qp_ = kQpIdx1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
}
@@ -1337,7 +1338,7 @@
codec_info.codecType = kVideoCodecVP8;
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
- encoded_image.SetSpatialIndex(0);
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
encoded_image.qp_ = kQpIdx0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
}
@@ -1354,10 +1355,10 @@
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
encoded_image.qp_ = kQpIdx0;
- encoded_image.SetSpatialIndex(0);
+ codec_info.codecSpecific.VP9.spatial_idx = 0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
encoded_image.qp_ = kQpIdx1;
- encoded_image.SetSpatialIndex(1);
+ codec_info.codecSpecific.VP9.spatial_idx = 1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
}
statistics_proxy_.reset();
@@ -1380,6 +1381,7 @@
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
encoded_image.qp_ = kQpIdx0;
+ codec_info.codecSpecific.VP9.spatial_idx = 0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
}
statistics_proxy_.reset();
@@ -1393,10 +1395,10 @@
codec_info.codecType = kVideoCodecH264;
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
- encoded_image.SetSpatialIndex(0);
+ codec_info.codecSpecific.H264.simulcast_idx = 0;
encoded_image.qp_ = kQpIdx0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
- encoded_image.SetSpatialIndex(1);
+ codec_info.codecSpecific.H264.simulcast_idx = 1;
encoded_image.qp_ = kQpIdx1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
}
@@ -1538,7 +1540,6 @@
VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts,
quality_counts);
EncodedImage encoded_image;
- encoded_image.SetSpatialIndex(0);
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i)
statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo);
@@ -1559,7 +1560,6 @@
VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts,
quality_counts);
EncodedImage encoded_image;
- encoded_image.SetSpatialIndex(0);
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i)
statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo);
@@ -1584,7 +1584,6 @@
VideoStreamEncoderObserver::AdaptationReason::kNone, cpu_counts,
quality_counts);
EncodedImage encoded_image;
- encoded_image.SetSpatialIndex(0);
for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i)
statistics_proxy_->OnSendEncodedImage(encoded_image, &kDefaultCodecInfo);
// Histograms are updated when the statistics_proxy_ is deleted.
@@ -1718,13 +1717,13 @@
EncodedImage encoded_image;
encoded_image._encodedWidth = kEncodedWidth;
encoded_image._encodedHeight = kEncodedHeight;
- encoded_image.SetSpatialIndex(0);
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
- encoded_image.SetSpatialIndex(1);
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
VideoSendStream::Stats stats = statistics_proxy_->GetStats();
@@ -1747,6 +1746,7 @@
// Report stats for second SSRC to make sure it's not outdated along with the
// first SSRC.
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
// Forward 1 ms, reach timeout, substream 0 should have no resolution
@@ -1765,13 +1765,13 @@
EncodedImage encoded_image;
encoded_image._encodedWidth = kEncodedWidth;
encoded_image._encodedHeight = kEncodedHeight;
- encoded_image.SetSpatialIndex(0);
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
+ codec_info.codecSpecific.VP8.simulcastIdx = 0;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
- encoded_image.SetSpatialIndex(1);
+ codec_info.codecSpecific.VP8.simulcastIdx = 1;
statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
statistics_proxy_->OnInactiveSsrc(config_.rtp.ssrcs[1]);
@@ -2182,11 +2182,11 @@
explicit ForcedFallbackTest(const std::string& field_trials)
: SendStatisticsProxyTest(field_trials) {
codec_info_.codecType = kVideoCodecVP8;
+ codec_info_.codecSpecific.VP8.simulcastIdx = 0;
codec_info_.codecSpecific.VP8.temporalIdx = 0;
codec_info_.codec_name = "fake_codec";
encoded_image_._encodedWidth = kWidth;
encoded_image_._encodedHeight = kHeight;
- encoded_image_.SetSpatialIndex(0);
}
~ForcedFallbackTest() override {}
@@ -2260,7 +2260,7 @@
}
TEST_F(ForcedFallbackEnabled, StatsNotUpdatedForSimulcast) {
- encoded_image_.SetSpatialIndex(1);
+ codec_info_.codecSpecific.VP8.simulcastIdx = 1;
InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
statistics_proxy_.reset();
EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc
index 493f50f..bd61881 100644
--- a/video/video_receive_stream.cc
+++ b/video/video_receive_stream.cc
@@ -318,6 +318,10 @@
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) {
stats_proxy_.OnPreDecode(encoded_image, codec_specific_info);
+ size_t simulcast_idx = 0;
+ if (codec_specific_info->codecType == kVideoCodecVP8) {
+ simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx;
+ }
{
rtc::CritScope lock(&ivf_writer_lock_);
if (ivf_writer_.get()) {
diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc
index 822b951..cf4bf87 100644
--- a/video/video_send_stream_impl.cc
+++ b/video/video_send_stream_impl.cc
@@ -522,13 +522,11 @@
// Encoded is called on whatever thread the real encoder implementation run
// on. In the case of hardware encoders, there might be several encoders
// running in parallel on different threads.
- const size_t simulcast_idx =
- (codec_specific_info->codecType != kVideoCodecVP9)
- ? encoded_image.SpatialIndex().value_or(0)
- : 0;
+ size_t simulcast_idx = 0;
+ if (codec_specific_info->codecType == kVideoCodecVP8) {
+ simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx;
+ }
if (config_->post_encode_callback) {
- // TODO(nisse): Delete webrtc::EncodedFrame class, pass EncodedImage
- // instead.
config_->post_encode_callback->EncodedFrameCallback(EncodedFrame(
encoded_image._buffer, encoded_image._length, encoded_image._frameType,
simulcast_idx, encoded_image.Timestamp()));
@@ -544,10 +542,15 @@
EncodedImageCallback::Result result = rtp_video_sender_->OnEncodedImage(
encoded_image, codec_specific_info, fragmentation);
+ RTC_DCHECK(codec_specific_info);
+
+ int layer = codec_specific_info->codecType == kVideoCodecVP8
+ ? codec_specific_info->codecSpecific.VP8.simulcastIdx
+ : 0;
{
rtc::CritScope lock(&ivf_writers_crit_);
- if (file_writers_[simulcast_idx].get()) {
- bool ok = file_writers_[simulcast_idx]->WriteFrame(
+ if (file_writers_[layer].get()) {
+ bool ok = file_writers_[layer]->WriteFrame(
encoded_image, codec_specific_info->codecType);
RTC_DCHECK(ok);
}
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index 788b60b..cb11ebc 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -3041,10 +3041,10 @@
encoded.capture_time_ms_ = input_image.render_time_ms();
for (size_t i = 0; i < kNumStreams; ++i) {
+ specifics.codecSpecific.generic.simulcast_idx = static_cast<uint8_t>(i);
encoded._frameType = (*frame_types)[i];
encoded._encodedWidth = kEncodedResolution[i].width;
encoded._encodedHeight = kEncodedResolution[i].height;
- encoded.SetSpatialIndex(i);
EncodedImageCallback* callback;
{
rtc::CritScope cs(&crit_sect_);
