Add support for scalability modes L2T3 and S2T3
Bug: webrtc:11607
Change-Id: I1d0bd171564d2852f2f6ee2bbee26c7a1c0e1c3f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/267103
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Niels Moller <nisse@webrtc.org>
Reviewed-by: Florent Castelli <orphis@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37389}
diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h
index 262e7ce..a33b9e4 100644
--- a/api/video_codecs/scalability_mode.h
+++ b/api/video_codecs/scalability_mode.h
@@ -44,6 +44,7 @@
kL3T3h,
kL3T3_KEY,
kS2T1,
+ kS2T3,
kS3T3,
};
diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
index b252fb7..35675f5 100644
--- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
+++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
@@ -466,7 +466,7 @@
}
for (int tid = 0; tid < num_temporal_layers_; ++tid) {
config_->layer_target_bitrate[sid * num_temporal_layers_ + tid] =
- current_bitrate_allocation_.GetBitrate(sid, tid) / 1000;
+ current_bitrate_allocation_.GetTemporalLayerSum(sid, tid) / 1000;
}
config_->ss_target_bitrate[sid] =
current_bitrate_allocation_.GetSpatialLayerSum(sid) / 1000;
diff --git a/modules/video_coding/svc/BUILD.gn b/modules/video_coding/svc/BUILD.gn
index d82f316..b8ce91d 100644
--- a/modules/video_coding/svc/BUILD.gn
+++ b/modules/video_coding/svc/BUILD.gn
@@ -112,6 +112,7 @@
"../../../api/video:video_frame_type",
"../../../api/video_codecs:scalability_mode",
"../../../common_video/generic_frame_descriptor",
+ "../../../rtc_base:stringutils",
"../../../test:test_support",
]
absl_deps = [
diff --git a/modules/video_coding/svc/create_scalability_structure.cc b/modules/video_coding/svc/create_scalability_structure.cc
index dfbfb14..d5a85e0 100644
--- a/modules/video_coding/svc/create_scalability_structure.cc
+++ b/modules/video_coding/svc/create_scalability_structure.cc
@@ -106,6 +106,13 @@
{1, 1},
{2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = {
+ /*num_spatial_layers=*/2,
+ /*num_temporal_layers=*/3,
+ /*uses_reference_scaling=*/false,
+ {1, 1},
+ {2, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = {
/*num_spatial_layers=*/3,
/*num_temporal_layers=*/3,
@@ -127,6 +134,7 @@
kConfigL2T2},
{ScalabilityMode::kL2T2_KEY_SHIFT, Create<ScalabilityStructureL2T2KeyShift>,
kConfigL2T2},
+ {ScalabilityMode::kL2T3, Create<ScalabilityStructureL2T3>, kConfigL2T3},
{ScalabilityMode::kL2T3_KEY, Create<ScalabilityStructureL2T3Key>,
kConfigL2T3},
{ScalabilityMode::kL3T1, Create<ScalabilityStructureL3T1>, kConfigL3T1},
@@ -134,6 +142,7 @@
{ScalabilityMode::kL3T3_KEY, Create<ScalabilityStructureL3T3Key>,
kConfigL3T3},
{ScalabilityMode::kS2T1, Create<ScalabilityStructureS2T1>, kConfigS2T1},
+ {ScalabilityMode::kS2T3, Create<ScalabilityStructureS2T3>, kConfigS2T3},
{ScalabilityMode::kS3T3, Create<ScalabilityStructureS3T3>, kConfigS3T3},
};
diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc
index d0a56af..510c9fd3 100644
--- a/modules/video_coding/svc/scalability_mode_util.cc
+++ b/modules/video_coding/svc/scalability_mode_util.cc
@@ -75,6 +75,8 @@
if (mode_string == "S2T1")
return ScalabilityMode::kS2T1;
+ if (mode_string == "S2T3")
+ return ScalabilityMode::kS2T3;
if (mode_string == "S3T3")
return ScalabilityMode::kS3T3;
@@ -133,6 +135,8 @@
return "L3T3_KEY";
case ScalabilityMode::kS2T1:
return "S2T1";
+ case ScalabilityMode::kS2T3:
+ return "S2T3";
case ScalabilityMode::kS3T3:
return "S3T3";
}
@@ -179,6 +183,7 @@
case ScalabilityMode::kL3T3_KEY:
return InterLayerPredMode::kOnKeyPic;
case ScalabilityMode::kS2T1:
+ case ScalabilityMode::kS2T3:
case ScalabilityMode::kS3T3:
return InterLayerPredMode::kOff;
}
@@ -215,6 +220,7 @@
case ScalabilityMode::kL3T3_KEY:
return 3;
case ScalabilityMode::kS2T1:
+ case ScalabilityMode::kS2T3:
return 2;
case ScalabilityMode::kS3T3:
return 3;
@@ -259,6 +265,7 @@
return 3;
case ScalabilityMode::kS2T1:
return 1;
+ case ScalabilityMode::kS2T3:
case ScalabilityMode::kS3T3:
return 3;
}
diff --git a/modules/video_coding/svc/scalability_structure_full_svc.cc b/modules/video_coding/svc/scalability_structure_full_svc.cc
index 8920592..d73f167 100644
--- a/modules/video_coding/svc/scalability_structure_full_svc.cc
+++ b/modules/video_coding/svc/scalability_structure_full_svc.cc
@@ -353,6 +353,26 @@
return structure;
}
+FrameDependencyStructure ScalabilityStructureL2T3::DependencyStructure() const {
+ FrameDependencyStructure structure;
+ structure.num_decode_targets = 6;
+ structure.num_chains = 2;
+ structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1};
+ auto& t = structure.templates;
+ t.resize(10);
+ t[1].S(0).T(0).Dtis("SSSSSS").ChainDiffs({0, 0});
+ t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 1}).FrameDiffs({1});
+ t[3].S(0).T(2).Dtis("--D--R").ChainDiffs({2, 1}).FrameDiffs({2});
+ t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2, 1});
+ t[2].S(0).T(1).Dtis("-DS-RR").ChainDiffs({4, 3}).FrameDiffs({4});
+ t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4, 1});
+ t[4].S(0).T(2).Dtis("--D--R").ChainDiffs({6, 5}).FrameDiffs({2});
+ t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2, 1});
+ t[0].S(0).T(0).Dtis("SSSRRR").ChainDiffs({8, 7}).FrameDiffs({8});
+ t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 1}).FrameDiffs({8, 1});
+ return structure;
+}
+
FrameDependencyStructure ScalabilityStructureL3T1::DependencyStructure() const {
FrameDependencyStructure structure;
structure.num_decode_targets = 3;
diff --git a/modules/video_coding/svc/scalability_structure_full_svc.h b/modules/video_coding/svc/scalability_structure_full_svc.h
index a3cad0a..03141ff 100644
--- a/modules/video_coding/svc/scalability_structure_full_svc.h
+++ b/modules/video_coding/svc/scalability_structure_full_svc.h
@@ -133,6 +133,23 @@
FrameDependencyStructure DependencyStructure() const override;
};
+// S1T2 4 ,8
+// S1T1 / | 6' |
+// S1T0 2--+-'+--+-...
+// | | | |
+// S0T2 | 3 | ,7
+// S0T1 | / 5'
+// S0T0 1----'-----...
+// Time-> 0 1 2 3
+class ScalabilityStructureL2T3 : public ScalabilityStructureFullSvc {
+ public:
+ explicit ScalabilityStructureL2T3(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureFullSvc(2, 3, resolution_factor) {}
+ ~ScalabilityStructureL2T3() override = default;
+
+ FrameDependencyStructure DependencyStructure() const override;
+};
+
// S2 0-0-0-
// | | |
// S1 0-0-0-...
diff --git a/modules/video_coding/svc/scalability_structure_simulcast.cc b/modules/video_coding/svc/scalability_structure_simulcast.cc
index e5fa4c4..0a06e9e 100644
--- a/modules/video_coding/svc/scalability_structure_simulcast.cc
+++ b/modules/video_coding/svc/scalability_structure_simulcast.cc
@@ -242,6 +242,26 @@
return structure;
}
+FrameDependencyStructure ScalabilityStructureS2T3::DependencyStructure() const {
+ FrameDependencyStructure structure;
+ structure.num_decode_targets = 6;
+ structure.num_chains = 2;
+ structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1};
+ auto& t = structure.templates;
+ t.resize(10);
+ t[1].S(0).T(0).Dtis("SSS---").ChainDiffs({0, 0});
+ t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 0});
+ t[3].S(0).T(2).Dtis("--D---").ChainDiffs({2, 1}).FrameDiffs({2});
+ t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2});
+ t[2].S(0).T(1).Dtis("-DS---").ChainDiffs({4, 3}).FrameDiffs({4});
+ t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4});
+ t[4].S(0).T(2).Dtis("--D---").ChainDiffs({6, 5}).FrameDiffs({2});
+ t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2});
+ t[0].S(0).T(0).Dtis("SSS---").ChainDiffs({8, 7}).FrameDiffs({8});
+ t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 8}).FrameDiffs({8});
+ return structure;
+}
+
FrameDependencyStructure ScalabilityStructureS3T3::DependencyStructure() const {
FrameDependencyStructure structure;
structure.num_decode_targets = 9;
diff --git a/modules/video_coding/svc/scalability_structure_simulcast.h b/modules/video_coding/svc/scalability_structure_simulcast.h
index 7b57df2..53f491c 100644
--- a/modules/video_coding/svc/scalability_structure_simulcast.h
+++ b/modules/video_coding/svc/scalability_structure_simulcast.h
@@ -76,6 +76,26 @@
FrameDependencyStructure DependencyStructure() const override;
};
+// S1T2 3 7
+// | /
+// S1T1 / 5
+// |_/
+// S1T0 1-------9...
+//
+// S0T2 2 6
+// | /
+// S0T1 / 4
+// |_/
+// S0T0 0-------8...
+// Time-> 0 1 2 3 4
+class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast {
+ public:
+ ScalabilityStructureS2T3() : ScalabilityStructureSimulcast(2, 3) {}
+ ~ScalabilityStructureS2T3() override = default;
+
+ FrameDependencyStructure DependencyStructure() const override;
+};
+
class ScalabilityStructureS3T3 : public ScalabilityStructureSimulcast {
public:
ScalabilityStructureS3T3() : ScalabilityStructureSimulcast(3, 3) {}
diff --git a/modules/video_coding/svc/scalability_structure_unittest.cc b/modules/video_coding/svc/scalability_structure_unittest.cc
index ffc085d..86d7cc0 100644
--- a/modules/video_coding/svc/scalability_structure_unittest.cc
+++ b/modules/video_coding/svc/scalability_structure_unittest.cc
@@ -22,6 +22,7 @@
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/scalability_structure_test_helpers.h"
#include "modules/video_coding/svc/scalable_video_controller.h"
+#include "rtc_base/strings/string_builder.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -43,6 +44,41 @@
using ::testing::TestWithParam;
using ::testing::Values;
+std::string FrameDependencyTemplateToString(const FrameDependencyTemplate& t) {
+ rtc::StringBuilder sb;
+ sb << "S" << t.spatial_id << "T" << t.temporal_id;
+ sb << ": dtis = ";
+ for (const auto dtis : t.decode_target_indications) {
+ switch (dtis) {
+ case DecodeTargetIndication::kNotPresent:
+ sb << "-";
+ break;
+ case DecodeTargetIndication::kDiscardable:
+ sb << "D";
+ break;
+ case DecodeTargetIndication::kSwitch:
+ sb << "S";
+ break;
+ case DecodeTargetIndication::kRequired:
+ sb << "R";
+ break;
+ default:
+ sb << "?";
+ break;
+ }
+ }
+ sb << ", frame diffs = { ";
+ for (int d : t.frame_diffs) {
+ sb << d << ", ";
+ }
+ sb << "}, chain diffs = { ";
+ for (int d : t.chain_diffs) {
+ sb << d << ", ";
+ }
+ sb << "}";
+ return sb.Release();
+}
+
struct SvcTestParam {
friend std::ostream& operator<<(std::ostream& os, const SvcTestParam& param) {
return os << param.name;
@@ -176,7 +212,8 @@
.GenerateFrames(GetParam().num_temporal_units);
for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) {
EXPECT_THAT(structure.templates, Contains(frame_infos[frame_id]))
- << " for frame " << frame_id;
+ << " for frame " << frame_id << ", Expected "
+ << FrameDependencyTemplateToString(frame_infos[frame_id]);
}
}
@@ -336,10 +373,12 @@
SvcTestParam{"L3T1", /*num_temporal_units=*/3},
SvcTestParam{"L3T3", /*num_temporal_units=*/8},
SvcTestParam{"S2T1", /*num_temporal_units=*/3},
+ SvcTestParam{"S2T3", /*num_temporal_units=*/8},
SvcTestParam{"S3T3", /*num_temporal_units=*/8},
SvcTestParam{"L2T2", /*num_temporal_units=*/4},
SvcTestParam{"L2T2_KEY", /*num_temporal_units=*/4},
SvcTestParam{"L2T2_KEY_SHIFT", /*num_temporal_units=*/4},
+ SvcTestParam{"L2T3", /*num_temporal_units=*/8},
SvcTestParam{"L2T3_KEY", /*num_temporal_units=*/8},
SvcTestParam{"L3T3_KEY", /*num_temporal_units=*/8}),
[](const testing::TestParamInfo<SvcTestParam>& info) {
diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc
index 30a0ca4..993d18b 100644
--- a/pc/test/svc_e2e_tests.cc
+++ b/pc/test/svc_e2e_tests.cc
@@ -295,6 +295,7 @@
SvcTestParameters{cricket::kVp9CodecName, "L2T2", 2, 2},
SvcTestParameters{cricket::kVp9CodecName, "L2T2_KEY", 2, 2},
SvcTestParameters{cricket::kVp9CodecName, "L2T2_KEY_SHIFT", 2, 2},
+ SvcTestParameters{cricket::kVp9CodecName, "L2T3", 2, 3},
SvcTestParameters{cricket::kVp9CodecName, "L2T3_KEY", 2, 3},
SvcTestParameters{cricket::kVp9CodecName, "L3T1", 3, 1},
SvcTestParameters{cricket::kVp9CodecName, "L3T3", 3, 3}),
