commit d6afbead2d85da78930d19c87b8308e892227eb7
author Sergey Silkin <ssilkin@webrtc.org> Fri Jul 09 11:45:29 2021
committer WebRTC LUCI CQ <webrtc-scoped@luci-project-accounts.iam.gserviceaccount.com> Fri Jul 09 13:49:41 2021
tree 7a9520c11d21261ea4f6636cb50430b094412448
parent e91c992fa1055a059180a1668a72821372267ebd

Correctly set number of reference buffers in H264 encoder

iNumRefFrame specifies total number of reference buffers to allocate.
For N temporal layers we need at least (N - 1) buffers to store last
encoded frames of all reference temporal layers.

There is no API in OpenH254 encoder to specify exact set of references
to be used to prediction of a given frame. Encoder can theoretically
use all available references.

Note that there is logic in OpenH264 which overrides iNumRefFrame to
max(iNumRefFrame, N - 1): https://source.chromium.org/chromium/chromium/src/+/main:third_party/openh264/src/codec/encoder/core/src/au_set.cpp;drc=8e90a2775c5b9448324fe8fef11d177cb65f36cc;l=122.
I.e., this change has no real effect. It only makes setup more clear.

Bug: none
Change-Id: If4b4970007e1cc55d8f052ea05213ab2e89a878f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/225480
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34445}

modules/video_coding/codecs/h264/h264_encoder_impl.cc

diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
index 733f00f..af03939 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
@@ -16,6 +16,7 @@
 
 #include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
 
+#include <algorithm>
 #include <limits>
 #include <string>
 
@@ -241,7 +242,8 @@
     configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
     configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
     configurations_[i].num_temporal_layers =
-        codec_.simulcastStream[idx].numberOfTemporalLayers;
+        std::max(codec_.H264()->numberOfTemporalLayers,
+                 codec_.simulcastStream[idx].numberOfTemporalLayers);
 
     // Create downscaled image buffers.
     if (i > 0) {
@@ -578,7 +580,13 @@
       encoder_params.iMaxBitrate;
   encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
   if (encoder_params.iTemporalLayerNum > 1) {
-    encoder_params.iNumRefFrame = 1;
+    // iNumRefFrame specifies total number of reference buffers to allocate.
+    // For N temporal layers we need at least (N - 1) buffers to store last
+    // encoded frames of all reference temporal layers.
+    // Note that there is no API in OpenH264 encoder to specify exact set of
+    // references to be used to prediction of a given frame. Encoder can
+    // theoretically use all available reference buffers.
+    encoder_params.iNumRefFrame = encoder_params.iTemporalLayerNum - 1;
   }
   RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
                 << OPENH264_MINOR;

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
index 4c11f6a..dee5b1b 100644
--- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
+++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
@@ -128,6 +128,8 @@
     case kVideoCodecH264:
       ss << "frame_dropping: " << codec.H264().frameDroppingOn;
       ss << "\nkey_frame_interval: " << codec.H264().keyFrameInterval;
+      ss << "\nnum_temporal_layers: "
+         << static_cast<int>(codec.H264().numberOfTemporalLayers);
       break;
     default:
       break;
@@ -214,6 +216,8 @@
     case kVideoCodecH264:
       codec_settings.H264()->frameDroppingOn = frame_dropper_on;
       codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval;
+      codec_settings.H264()->numberOfTemporalLayers =
+          static_cast<uint8_t>(num_temporal_layers);
       break;
     default:
       break;
@@ -236,6 +240,8 @@
     return codec_settings.VP8().numberOfTemporalLayers;
   } else if (codec_settings.codecType == kVideoCodecVP9) {
     return codec_settings.VP9().numberOfTemporalLayers;
+  } else if (codec_settings.codecType == kVideoCodecH264) {
+    return codec_settings.H264().numberOfTemporalLayers;
   } else {
     return 1;
   }

modules/video_coding/utility/simulcast_test_fixture_impl.cc

diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
index 8a27582..6d3195c 100644
--- a/modules/video_coding/utility/simulcast_test_fixture_impl.cc
+++ b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
@@ -590,6 +590,7 @@
     settings_.VP8()->numberOfTemporalLayers = 1;
     temporal_layer_profile = kDefaultTemporalLayerProfile;
   } else {
+    settings_.H264()->numberOfTemporalLayers = 1;
     temporal_layer_profile = kNoTemporalLayerProfile;
   }
   settings_.maxBitrate = 100;