Partial frame capture API part 3
Implement utility for applying partial updates to video frames.
Bug: webrtc:10152
Change-Id: I295fa9f792b96bbf1140a13f1f04e4f9deaccd5c
Reviewed-on: https://webrtc-review.googlesource.com/c/120408
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26522}diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc
index 03bbd71..c91aeda 100644
--- a/api/video/video_frame.cc
+++ b/api/video/video_frame.cc
@@ -156,7 +156,6 @@
void VideoFrame::set_video_frame_buffer(
rtc::scoped_refptr<VideoFrameBuffer> buffer) {
- RTC_CHECK(buffer.get());
video_frame_buffer_ = buffer;
}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 4da1e23..9874742 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -171,6 +171,8 @@
sources = [
"overuse_frame_detector.cc",
"overuse_frame_detector.h",
+ "partial_frame_assembler.cc",
+ "partial_frame_assembler.h",
"video_stream_encoder.cc",
"video_stream_encoder.h",
]
@@ -475,6 +477,7 @@
"end_to_end_tests/stats_tests.cc",
"end_to_end_tests/transport_feedback_tests.cc",
"overuse_frame_detector_unittest.cc",
+ "partial_frame_assembler_unittest.cc",
"picture_id_tests.cc",
"quality_scaling_tests.cc",
"quality_threshold_unittest.cc",
diff --git a/video/partial_frame_assembler.cc b/video/partial_frame_assembler.cc
new file mode 100644
index 0000000..e7d1c12
--- /dev/null
+++ b/video/partial_frame_assembler.cc
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/partial_frame_assembler.h"
+
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/ref_counted_object.h"
+
+namespace webrtc {
+
+PartialFrameAssembler::PartialFrameAssembler() = default;
+PartialFrameAssembler::~PartialFrameAssembler() = default;
+
+bool PartialFrameAssembler::ApplyPartialUpdate(
+ const rtc::scoped_refptr<VideoFrameBuffer>& input_buffer,
+ VideoFrame* uncompressed_frame,
+ const VideoFrame::PartialFrameDescription* partial_desc) {
+ const int changed_rect_width = input_buffer ? input_buffer->width() : 0;
+ const int changed_rect_height = input_buffer ? input_buffer->height() : 0;
+ if (partial_desc == nullptr) {
+ // Full update. Copy whole picture to the cached buffer. May need to
+ // resize or create the cache buffer.
+ if (!cached_frame_buffer_ ||
+ cached_frame_buffer_->height() < input_buffer->height() ||
+ cached_frame_buffer_->width() < input_buffer->width()) {
+ cached_frame_buffer_ =
+ I420Buffer::Create(input_buffer->width(), input_buffer->height());
+ }
+ cached_frame_buffer_->PasteFrom(*input_buffer->ToI420().get(), 0, 0);
+ } else {
+ // Have to apply partial input picture to the cached buffer.
+ // Check all possible error situations.
+ if (!cached_frame_buffer_) {
+ RTC_LOG(LS_ERROR) << "Partial picture received but no cached full picture"
+ "present.";
+ return false;
+ }
+ if (partial_desc->offset_x % 2 != 0 || partial_desc->offset_y % 2 != 0) {
+ RTC_LOG(LS_ERROR) << "Partial picture required to be at even offset."
+ " Actual: ("
+ << partial_desc->offset_x << ", "
+ << partial_desc->offset_y << ").";
+ cached_frame_buffer_ = nullptr;
+ return false;
+ }
+ if ((changed_rect_width % 2 != 0 &&
+ changed_rect_width + partial_desc->offset_x <
+ cached_frame_buffer_->width()) ||
+ (changed_rect_height % 2 != 0 &&
+ changed_rect_height + partial_desc->offset_y <
+ cached_frame_buffer_->height())) {
+ RTC_LOG(LS_ERROR) << "Partial picture required to have even dimensions."
+ " Actual: "
+ << input_buffer->width() << "x"
+ << input_buffer->height() << ".";
+ cached_frame_buffer_ = nullptr;
+ return false;
+ }
+ if (partial_desc->offset_x < 0 ||
+ partial_desc->offset_x + changed_rect_width >
+ cached_frame_buffer_->width() ||
+ partial_desc->offset_y < 0 ||
+ partial_desc->offset_y + changed_rect_height >
+ cached_frame_buffer_->height()) {
+ RTC_LOG(LS_ERROR) << "Partial picture is outside of bounds.";
+ cached_frame_buffer_ = nullptr;
+ return false;
+ }
+ // No errors: apply new image to the cache and use the result.
+ if (input_buffer) {
+ cached_frame_buffer_->PasteFrom(*input_buffer->ToI420().get(),
+ partial_desc->offset_x,
+ partial_desc->offset_y);
+ }
+ }
+ // Remove partial frame description, as it doesn't make sense after update
+ // is applied.
+ uncompressed_frame->set_partial_frame_description(absl::nullopt);
+ uncompressed_frame->set_video_frame_buffer(
+ I420Buffer::Copy(*cached_frame_buffer_.get()));
+ return true;
+}
+
+void PartialFrameAssembler::Reset() {
+ cached_frame_buffer_ = nullptr;
+}
+
+} // namespace webrtc
diff --git a/video/partial_frame_assembler.h b/video/partial_frame_assembler.h
new file mode 100644
index 0000000..9106bf8
--- /dev/null
+++ b/video/partial_frame_assembler.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_PARTIAL_FRAME_ASSEMBLER_H_
+#define VIDEO_PARTIAL_FRAME_ASSEMBLER_H_
+
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+
+namespace webrtc {
+
+// Maintains cache of a full resolution frame buffer and applies partial
+// updates to it.
+// This class is not thread-safe.
+class PartialFrameAssembler {
+ public:
+ PartialFrameAssembler();
+ ~PartialFrameAssembler();
+
+ // Applies |input_buffer| to the cached buffer and sets buffer for
+ // |uncompresed_frame| to a full updated image.
+ // Returns false on any error. In that case the buffer will be invalidated
+ // and subsequent updates will also return error until full resolution frame
+ // is processed.
+ bool ApplyPartialUpdate(
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ VideoFrame* uncompressed_frame,
+ const VideoFrame::PartialFrameDescription* partial_desc);
+
+ // Clears internal buffer.
+ void Reset();
+
+ private:
+ rtc::scoped_refptr<I420Buffer> cached_frame_buffer_;
+};
+
+} // namespace webrtc
+#endif // VIDEO_PARTIAL_FRAME_ASSEMBLER_H_
diff --git a/video/partial_frame_assembler_unittest.cc b/video/partial_frame_assembler_unittest.cc
new file mode 100644
index 0000000..25b0f3e
--- /dev/null
+++ b/video/partial_frame_assembler_unittest.cc
@@ -0,0 +1,279 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/partial_frame_assembler.h"
+
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+constexpr uint8_t kCol1 = 100;
+constexpr uint8_t kCol2 = 200;
+
+constexpr int kWidth = 640;
+constexpr int kHeight = 480;
+
+rtc::scoped_refptr<I420Buffer> CreatePicture(int width,
+ int height,
+ uint8_t data) {
+ rtc::scoped_refptr<I420Buffer> buf = I420Buffer::Create(width, height);
+ for (int row = 0; row < height; ++row) {
+ for (int col = 0; col < width; ++col) {
+ int pos_y = row * buf->StrideY() + col;
+ int pos_u = row / 2 * buf->StrideU() + col / 2;
+ int pos_v = row / 2 * buf->StrideV() + col / 2;
+ buf->MutableDataY()[pos_y] = data;
+ buf->MutableDataU()[pos_u] = data;
+ buf->MutableDataV()[pos_v] = data;
+ }
+ }
+ return buf;
+}
+
+VideoFrame CreateFrame(rtc::scoped_refptr<I420Buffer> buf) {
+ VideoFrame frame = VideoFrame(buf, VideoRotation::kVideoRotation_0, 0);
+ frame.set_cache_buffer_for_partial_updates(true);
+ return frame;
+}
+
+bool TestPictureWithOneRect(rtc::scoped_refptr<I420BufferInterface> buf,
+ int offset_x,
+ int offset_y,
+ int rect_width,
+ int rect_height,
+ uint8_t in_rect_data,
+ uint8_t out_rect_data) {
+ for (int row = 0; row < buf->height(); ++row) {
+ for (int col = 0; col < buf->width(); ++col) {
+ int pos_y = row * buf->StrideY() + col;
+ int pos_u = row / 2 * buf->StrideU() + col / 2;
+ int pos_v = row / 2 * buf->StrideV() + col / 2;
+ uint8_t y = buf->DataY()[pos_y];
+ uint8_t u = buf->DataU()[pos_u];
+ uint8_t v = buf->DataV()[pos_v];
+ bool in_rect = col >= offset_x && col < offset_x + rect_width &&
+ row >= offset_y && row < offset_y + rect_height;
+ uint8_t expected_data = in_rect ? in_rect_data : out_rect_data;
+ if (y != expected_data || u != expected_data || v != expected_data)
+ return false;
+ }
+ }
+ return true;
+}
+
+TEST(PartialFrameAssembler, FullPictureUpdates) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Full pic.
+ auto full_pic2 = CreatePicture(kWidth, kHeight, kCol2);
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+ EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0,
+ 0, -1, -1, kCol1, kCol1));
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, nullptr));
+ EXPECT_TRUE(TestPictureWithOneRect(frame2.video_frame_buffer()->ToI420(), 0,
+ 0, -1, -1, kCol2, kCol2));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateFirstFails) {
+ PartialFrameAssembler decompressor;
+
+ // Partial update.
+ auto full_pic1 = CreatePicture(20, 20, kCol1);
+ VideoFrame::PartialFrameDescription desc1{0, 0};
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ EXPECT_FALSE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, &desc1));
+}
+
+TEST(PartialFrameAssembler, PartialUpdate) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update.
+ auto full_pic2 = CreatePicture(10, 20, kCol2);
+ VideoFrame::PartialFrameDescription desc2{30, 40};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+ EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0,
+ 0, -1, -1, kCol1, kCol1));
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+ EXPECT_TRUE(TestPictureWithOneRect(frame2.video_frame_buffer()->ToI420(), 30,
+ 40, 10, 20, kCol2, kCol1));
+}
+
+TEST(PartialFrameAssembler, ProcessesUnchangedUpdate) {
+ PartialFrameAssembler decompressor;
+
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update.
+ auto full_pic2 = CreatePicture(10, 20, kCol2);
+ VideoFrame::PartialFrameDescription desc2{30, 40};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ // Empty update.
+ VideoFrame::PartialFrameDescription desc3{0, 0};
+ VideoFrame frame3 = CreateFrame(nullptr);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+
+ EXPECT_TRUE(decompressor.ApplyPartialUpdate(nullptr, &frame3, &desc3));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateFailsForOddXOffset) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update.
+ auto full_pic2 = CreatePicture(10, 20, kCol2);
+ // Offset is odd.
+ VideoFrame::PartialFrameDescription desc2{31, 40};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_FALSE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateFailsForOddYOffset) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update.
+ auto full_pic2 = CreatePicture(10, 20, kCol2);
+ // Offset is odd.
+ VideoFrame::PartialFrameDescription desc2{30, 41};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_FALSE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateFailsForOddWidth) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update. Odd width.
+ auto full_pic2 = CreatePicture(11, 20, kCol2);
+ VideoFrame::PartialFrameDescription desc2{30, 40};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_FALSE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateWorksForOddWidthAtTheEnd) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic. Odd resolution.
+ auto full_pic1 = CreatePicture(kWidth + 1, kHeight + 1, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update. Odd width.
+ auto full_pic2 = CreatePicture(11, 11, kCol2);
+ VideoFrame::PartialFrameDescription desc2{kWidth + 1 - 11, kHeight + 1 - 11};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+}
+
+TEST(PartialFrameAssembler, PartialUpdateFailsForOddNotAtEnd) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic. Odd resolution.
+ auto full_pic1 = CreatePicture(kWidth + 1, kHeight + 1, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Partial pic update. Odd width.
+ auto full_pic2 = CreatePicture(11, 11, kCol2);
+ VideoFrame::PartialFrameDescription desc2{kWidth + 1 - 11, kHeight + 1 - 20};
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+
+ EXPECT_FALSE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2));
+}
+
+TEST(PartialFrameAssembler, FullPictureUpdatesCanChangeResolution) {
+ PartialFrameAssembler decompressor;
+
+ // Full pic.
+ auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1);
+ VideoFrame frame1 = CreateFrame(full_pic1);
+
+ // Full pic.
+ auto full_pic2 = CreatePicture(kWidth + 100, kHeight + 100, kCol2);
+ VideoFrame frame2 = CreateFrame(full_pic2);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr));
+ EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0,
+ 0, -1, -1, kCol1, kCol1));
+
+ EXPECT_EQ(frame1.video_frame_buffer()->width(), kWidth);
+ EXPECT_EQ(frame1.video_frame_buffer()->height(), kHeight);
+
+ EXPECT_TRUE(
+ decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, nullptr));
+ EXPECT_EQ(frame2.video_frame_buffer()->width(), kWidth + 100);
+ EXPECT_EQ(frame2.video_frame_buffer()->height(), kHeight + 100);
+}
+
+} // namespace
+} // namespace webrtc
