Partial frame capture API part 1
Define new optional struct in VideoFrame to signal that the frame is a
changed part of a whole picture and add a flag to signal that partial
update may be issued by the VideoFrame source.
Also, fix too strict assumptions in FrameBuffers PasteFrom methods.
Also, add ability to set a new buffer in video frame.
Bug: webrtc:10152
Change-Id: Ie0da418fd60bc7a34334329292e0b860ec388788
Reviewed-on: https://webrtc-review.googlesource.com/c/120405
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26489}diff --git a/api/video/i010_buffer.cc b/api/video/i010_buffer.cc
index 8f4541f..7286676 100644
--- a/api/video/i010_buffer.cc
+++ b/api/video/i010_buffer.cc
@@ -243,8 +243,10 @@
// Pasted picture has to be aligned so subsumpled UV plane isn't corrupted.
RTC_CHECK(offset_col % 2 == 0);
RTC_CHECK(offset_row % 2 == 0);
- RTC_CHECK(picture.width() % 2 == 0);
- RTC_CHECK(picture.height() % 2 == 0);
+ RTC_CHECK(picture.width() % 2 == 0 ||
+ picture.width() + offset_col == width());
+ RTC_CHECK(picture.height() % 2 == 0 ||
+ picture.height() + offset_row == height());
libyuv::CopyPlane_16(picture.DataY(), picture.StrideY(),
MutableDataY() + StrideY() * offset_row + offset_col,
diff --git a/api/video/i420_buffer.cc b/api/video/i420_buffer.cc
index ed45b98..46de581 100644
--- a/api/video/i420_buffer.cc
+++ b/api/video/i420_buffer.cc
@@ -237,8 +237,10 @@
// Pasted picture has to be aligned so subsumpled UV plane isn't corrupted.
RTC_CHECK(offset_col % 2 == 0);
RTC_CHECK(offset_row % 2 == 0);
- RTC_CHECK(picture.width() % 2 == 0);
- RTC_CHECK(picture.height() % 2 == 0);
+ RTC_CHECK(picture.width() % 2 == 0 ||
+ picture.width() + offset_col == width());
+ RTC_CHECK(picture.height() % 2 == 0 ||
+ picture.height() + offset_row == height());
libyuv::CopyPlane(picture.DataY(), picture.StrideY(),
MutableDataY() + StrideY() * offset_row + offset_col,
diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc
index 75a30b2..03bbd71 100644
--- a/api/video/video_frame.cc
+++ b/api/video/video_frame.cc
@@ -21,7 +21,9 @@
VideoFrame VideoFrame::Builder::build() {
return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_,
- ntp_time_ms_, rotation_, color_space_);
+ ntp_time_ms_, rotation_, color_space_,
+ partial_frame_description_,
+ cache_buffer_for_partial_updates_);
}
VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
@@ -76,6 +78,18 @@
return *this;
}
+VideoFrame::Builder& VideoFrame::Builder::set_partial_frame_description(
+ const absl::optional<PartialFrameDescription>& description) {
+ partial_frame_description_ = description;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_cache_buffer_for_partial_updates(
+ bool cache_buffer_for_partial_updates) {
+ cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates;
+ return *this;
+}
+
VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us)
@@ -97,20 +111,25 @@
RTC_DCHECK(buffer);
}
-VideoFrame::VideoFrame(uint16_t id,
- const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
- int64_t timestamp_us,
- uint32_t timestamp_rtp,
- int64_t ntp_time_ms,
- VideoRotation rotation,
- const absl::optional<ColorSpace>& color_space)
+VideoFrame::VideoFrame(
+ uint16_t id,
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ int64_t timestamp_us,
+ uint32_t timestamp_rtp,
+ int64_t ntp_time_ms,
+ VideoRotation rotation,
+ const absl::optional<ColorSpace>& color_space,
+ const absl::optional<PartialFrameDescription> partial_frame_description,
+ bool cache_buffer_for_partial_updates)
: id_(id),
video_frame_buffer_(buffer),
timestamp_rtp_(timestamp_rtp),
ntp_time_ms_(ntp_time_ms),
timestamp_us_(timestamp_us),
rotation_(rotation),
- color_space_(color_space) {}
+ color_space_(color_space),
+ partial_frame_description_(partial_frame_description),
+ cache_buffer_for_partial_updates_(cache_buffer_for_partial_updates) {}
VideoFrame::~VideoFrame() = default;
@@ -135,6 +154,12 @@
return video_frame_buffer_;
}
+void VideoFrame::set_video_frame_buffer(
+ rtc::scoped_refptr<VideoFrameBuffer> buffer) {
+ RTC_CHECK(buffer.get());
+ video_frame_buffer_ = buffer;
+}
+
int64_t VideoFrame::render_time_ms() const {
return timestamp_us() / rtc::kNumMicrosecsPerMillisec;
}
diff --git a/api/video/video_frame.h b/api/video/video_frame.h
index de3f0ef..de03d91 100644
--- a/api/video/video_frame.h
+++ b/api/video/video_frame.h
@@ -25,6 +25,14 @@
class RTC_EXPORT VideoFrame {
public:
+ // Describes a partial frame, which contains only a changed region compared
+ // to a previous frame. Shouldn't be set on the fully updated picture.
+ struct PartialFrameDescription {
+ // Coordinates of top-left corner of the changed region in the full picture.
+ int offset_x;
+ int offset_y;
+ };
+
// Preferred way of building VideoFrame objects.
class Builder {
public:
@@ -42,6 +50,10 @@
Builder& set_color_space(const ColorSpace& color_space);
Builder& set_color_space(const ColorSpace* color_space);
Builder& set_id(uint16_t id);
+ Builder& set_partial_frame_description(
+ const absl::optional<PartialFrameDescription>& description);
+ Builder& set_cache_buffer_for_partial_updates(
+ bool cache_buffer_for_partial_updates);
private:
uint16_t id_ = 0;
@@ -51,6 +63,8 @@
int64_t ntp_time_ms_ = 0;
VideoRotation rotation_ = kVideoRotation_0;
absl::optional<ColorSpace> color_space_;
+ absl::optional<PartialFrameDescription> partial_frame_description_;
+ bool cache_buffer_for_partial_updates_;
};
// To be deprecated. Migrate all use to Builder.
@@ -134,13 +148,31 @@
color_space ? absl::make_optional(*color_space) : absl::nullopt;
}
+ const PartialFrameDescription* partial_frame_description() const {
+ return partial_frame_description_ ? &partial_frame_description_.value()
+ : nullptr;
+ }
+ void set_partial_frame_description(
+ const absl::optional<PartialFrameDescription>& description) {
+ partial_frame_description_ = description;
+ }
+
+ void set_cache_buffer_for_partial_updates(
+ bool cache_buffer_for_partial_updates) {
+ cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates;
+ }
+ bool cache_buffer_for_partial_updates() const {
+ return cache_buffer_for_partial_updates_;
+ }
+
// Get render time in milliseconds.
// TODO(nisse): Deprecated. Migrate all users to timestamp_us().
int64_t render_time_ms() const;
- // Return the underlying buffer. Never nullptr for a properly
- // initialized VideoFrame.
+ // Return the underlying buffer. This can only be a nullptr for a partial
+ // update VideoFrame with no changed pixels.
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;
+ void set_video_frame_buffer(rtc::scoped_refptr<VideoFrameBuffer> buffer);
// TODO(nisse): Deprecated.
// Return true if the frame is stored in a texture.
@@ -149,13 +181,16 @@
}
private:
- VideoFrame(uint16_t id,
- const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
- int64_t timestamp_us,
- uint32_t timestamp_rtp,
- int64_t ntp_time_ms,
- VideoRotation rotation,
- const absl::optional<ColorSpace>& color_space);
+ VideoFrame(
+ uint16_t id,
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ int64_t timestamp_us,
+ uint32_t timestamp_rtp,
+ int64_t ntp_time_ms,
+ VideoRotation rotation,
+ const absl::optional<ColorSpace>& color_space,
+ const absl::optional<PartialFrameDescription> partial_frame_description,
+ bool cache_buffer_for_partial_updates_);
uint16_t id_;
// An opaque reference counted handle that stores the pixel data.
@@ -165,6 +200,9 @@
int64_t timestamp_us_;
VideoRotation rotation_;
absl::optional<ColorSpace> color_space_;
+ absl::optional<PartialFrameDescription> partial_frame_description_;
+ // Should be set on all frames, if the source may produce partial updates.
+ bool cache_buffer_for_partial_updates_;
};
} // namespace webrtc
diff --git a/api/video/video_source_interface.h b/api/video/video_source_interface.h
index 9d1641c..e589b7f 100644
--- a/api/video/video_source_interface.h
+++ b/api/video/video_source_interface.h
@@ -42,6 +42,9 @@
absl::optional<int> target_pixel_count;
// Tells the source the maximum framerate the sink wants.
int max_framerate_fps = std::numeric_limits<int>::max();
+
+ // Tells the source that the sink supports partial frame updates.
+ bool partial_frames = false;
};
template <typename VideoFrameT>
