Fix GetSingleActiveLayerPixels When simulcast is configured (numberOfSimulcastStreams > 1), use resolution from codec.simulcastStream[]. Bug: b/517029078, b/507191231, webrtc:510393737 Change-Id: I2b4bab0a1f1e2d7dcfad6caa5164f8be7ed93bcf Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/476640 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Erik Språng <sprang@webrtc.org> Cr-Commit-Position: refs/heads/main@{#47852}
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 5abc96f..d2365be 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn
@@ -120,6 +120,7 @@ "../../api:scoped_refptr", "../../api/adaptation:resource_adaptation_api", "../../api/video:video_adaptation", + "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", "../../test:create_test_field_trials",
diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index 595f167..9bd5920 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc
@@ -30,7 +30,6 @@ #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" #include "call/adaptation/video_stream_input_state_provider.h" -#include "modules/video_coding/svc/scalability_mode_util.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" @@ -729,29 +728,21 @@ const VideoCodec& codec) { int num_active = 0; std::optional<uint32_t> pixels; - if (codec.codecType == VideoCodecType::kVideoCodecAV1 && - codec.GetScalabilityMode().has_value()) { - for (int i = 0; - i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode())); - ++i) { - if (codec.spatialLayers[i].active) { + if (codec.numberOfSimulcastStreams > 1 || + !(codec.codecType == VideoCodecType::kVideoCodecAV1 || + codec.codecType == VideoCodecType::kVideoCodecVP9)) { + // Simulcast or non-SVC codec. + for (const auto& stream : codec.simulcastStream) { + if (stream.active) { ++num_active; - pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; - } - } - } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) { - for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { - if (codec.spatialLayers[i].active) { - ++num_active; - pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; + pixels = stream.width * stream.height; } } } else { - for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { - if (codec.simulcastStream[i].active) { + for (const auto& stream : codec.spatialLayers) { + if (stream.active) { ++num_active; - pixels = - codec.simulcastStream[i].width * codec.simulcastStream[i].height; + pixels = stream.width * stream.height; } } }
diff --git a/call/adaptation/video_stream_adapter_unittest.cc b/call/adaptation/video_stream_adapter_unittest.cc index a491644..0c62716 100644 --- a/call/adaptation/video_stream_adapter_unittest.cc +++ b/call/adaptation/video_stream_adapter_unittest.cc
@@ -19,6 +19,8 @@ #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "call/adaptation/adaptation_constraint.h" #include "call/adaptation/test/fake_frame_rate_provider.h" @@ -911,6 +913,191 @@ adapter_.RemoveAdaptationConstraint(&adaptation_constraint); } +using VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest = + ::testing::TestWithParam<VideoCodecType>; + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + SimulcastNoActiveStreams) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = false; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = false; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + SimulcastOneActiveStream) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = false; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = true; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), 640 * 360u); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + SimulcastMultipleActiveStreams) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = true; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = true; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + SinglecastOneActiveSpatialLayer) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 1; + codec.spatialLayers[0].active = false; + codec.spatialLayers[0].width = 320; + codec.spatialLayers[0].height = 180; + codec.spatialLayers[1].active = true; + codec.spatialLayers[1].width = 640; + codec.spatialLayers[1].height = 360; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), 640 * 360u); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + SinglecastMultipleActiveSpatialLayers) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 1; + codec.spatialLayers[0].active = true; + codec.spatialLayers[0].width = 320; + codec.spatialLayers[0].height = 180; + codec.spatialLayers[1].active = true; + codec.spatialLayers[1].width = 640; + codec.spatialLayers[1].height = 360; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +INSTANTIATE_TEST_SUITE_P(VideoStreamAdapterGetSingleActiveLayerPixelsTest, + VideoStreamAdapterGetSingleActiveLayerPixelsSvcTest, + ::testing::Values(VideoCodecType::kVideoCodecVP9, + VideoCodecType::kVideoCodecAV1)); + +using VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest = + ::testing::TestWithParam<VideoCodecType>; + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + SimulcastNoActiveStreams) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = false; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = false; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + SimulcastOneActiveStream) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = false; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = true; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), 640 * 360u); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + SimulcastMultipleActiveStreams) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 3; + codec.simulcastStream[0].active = true; + codec.simulcastStream[0].width = 320; + codec.simulcastStream[0].height = 180; + codec.simulcastStream[1].active = true; + codec.simulcastStream[1].width = 640; + codec.simulcastStream[1].height = 360; + codec.simulcastStream[2].active = false; + codec.simulcastStream[2].width = 1280; + codec.simulcastStream[2].height = 720; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + SinglecastOneActiveStream) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 1; + codec.simulcastStream[0].active = true; + codec.simulcastStream[0].width = 640; + codec.simulcastStream[0].height = 360; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), 640 * 360u); +} + +TEST_P(VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + SinglecastSpatialLayersAreIgnored) { + VideoCodec codec; + codec.codecType = GetParam(); + codec.numberOfSimulcastStreams = 1; + codec.spatialLayers[0].active = true; + codec.spatialLayers[0].width = 640; + codec.spatialLayers[0].height = 360; + + EXPECT_EQ(VideoStreamAdapter::GetSingleActiveLayerPixels(codec), + std::nullopt); +} + +INSTANTIATE_TEST_SUITE_P(VideoStreamAdapterGetSingleActiveLayerPixelsTest, + VideoStreamAdapterGetSingleActiveLayerPixelsNonSvcTest, + ::testing::Values(VideoCodecType::kVideoCodecVP8, + VideoCodecType::kVideoCodecH264, + VideoCodecType::kVideoCodecH265)); + // Death tests. // Disabled on Android because death tests misbehave on Android, see // base/test/gtest_util.h.