Validate that H.264 resolutions are spec compliant. Bug: chromium:504690157 Change-Id: I1a932dd0b8e71dbe0207d1e44e03bb495dccff2b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/487600 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Auto-Submit: Erik Språng <sprang@webrtc.org> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Cr-Commit-Position: refs/heads/main@{#48153}

diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
index b682013..69db6ff 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -138,6 +138,28 @@
   return std::nullopt;
 }
 
+bool IsValidResolution(int width, int height) {
+  // H.264 Level 5.2 limits:
+  // Max macroblocks per frame (MaxFS) = 36864
+  // Max width/height in macroblocks = Sqrt(MaxFS * 8) = 543
+  // A macroblock is 16x16.
+  const int64_t width_in_mbs = (static_cast<int64_t>(width) + 15) / 16;
+  const int64_t height_in_mbs = (static_cast<int64_t>(height) + 15) / 16;
+
+  if (width_in_mbs * height_in_mbs > 36864) {
+    return false;
+  }
+
+  // Aspect ratio check:
+  // PicWidthInMbs <= Sqrt(MaxFS * 8)
+  // FrameHeightInMbs <= Sqrt(MaxFS * 8)
+  if (width_in_mbs > 543 || height_in_mbs > 543) {
+    return false;
+  }
+
+  return true;
+}
+
 }  // namespace
 
 // Helper method used by H264EncoderImpl::Encode.
@@ -264,6 +286,18 @@
     codec_.simulcastStream[0].height = codec_.height;
   }
 
+  for (int i = 0; i < number_of_streams; ++i) {
+    if (!IsValidResolution(codec_.simulcastStream[i].width,
+                           codec_.simulcastStream[i].height)) {
+      RTC_LOG(LS_ERROR) << "InitEncode: Invalid stream resolution: "
+                        << codec_.simulcastStream[i].width << "x"
+                        << codec_.simulcastStream[i].height;
+      Release();
+      ReportError();
+      return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+    }
+  }
+
   for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
        ++i, --idx) {
     ISVCEncoder* openh264_encoder;

diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc b/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc
index b932278..66f9efe 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc

@@ -203,6 +203,63 @@
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE, encoder.Encode(frame, nullptr));
 }
 
+TEST(H264EncoderImplTest, RejectsTooLargeResolution) {
+  H264EncoderImpl encoder(CreateTestEnvironment(), {});
+  VideoCodec codec_settings;
+  SetDefaultSettings(&codec_settings);
+
+  // Level 5.2 MaxFS is 36864 macroblocks.
+  // 4096 x 2304 = 256 x 144 macroblocks = 36864 macroblocks (Allowed).
+  codec_settings.width = 4096;
+  codec_settings.height = 2304;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            encoder.InitEncode(&codec_settings, kSettings));
+
+  // 4096 x 2320 = 256 x 145 macroblocks = 37120 macroblocks (Too large).
+  codec_settings.width = 4096;
+  codec_settings.height = 2320;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_PARAMETER,
+            encoder.InitEncode(&codec_settings, kSettings));
+}
+
+TEST(H264EncoderImplTest, RejectsSkewedAspectRatioWidth) {
+  H264EncoderImpl encoder(CreateTestEnvironment(), {});
+  VideoCodec codec_settings;
+  SetDefaultSettings(&codec_settings);
+
+  // Level 5.2 Max width in macroblocks is 543 (8688 pixels).
+  // 8688 x 16 = 543 x 1 macroblocks = 543 macroblocks (Allowed).
+  codec_settings.width = 8688;
+  codec_settings.height = 16;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            encoder.InitEncode(&codec_settings, kSettings));
+
+  // 8704 x 16 = 544 x 1 macroblocks = 544 macroblocks (Too wide).
+  codec_settings.width = 8704;
+  codec_settings.height = 16;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_PARAMETER,
+            encoder.InitEncode(&codec_settings, kSettings));
+}
+
+TEST(H264EncoderImplTest, RejectsSkewedAspectRatioHeight) {
+  H264EncoderImpl encoder(CreateTestEnvironment(), {});
+  VideoCodec codec_settings;
+  SetDefaultSettings(&codec_settings);
+
+  // Level 5.2 Max height in macroblocks is 543 (8688 pixels).
+  // 16 x 8688 = 1 x 543 macroblocks = 543 macroblocks (Allowed).
+  codec_settings.width = 16;
+  codec_settings.height = 8688;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            encoder.InitEncode(&codec_settings, kSettings));
+
+  // 16 x 8704 = 1 x 544 macroblocks = 544 macroblocks (Too high).
+  codec_settings.width = 16;
+  codec_settings.height = 8704;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_PARAMETER,
+            encoder.InitEncode(&codec_settings, kSettings));
+}
+
 }  // anonymous namespace
 
 }  // namespace webrtc