blob: e073fd5e424efaff766881cdb0c68b89a2aa156f [file] [log] [blame]
nisseaf916892017-01-10 15:44:261/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#ifndef API_VIDEO_VIDEO_FRAME_H_
12#define API_VIDEO_VIDEO_FRAME_H_
nisseaf916892017-01-10 15:44:2613
14#include <stdint.h>
Jonas Olssona4d87372019-07-05 17:08:3315
Chen Xingf00bf422019-06-20 08:05:5516#include <utility>
nisseaf916892017-01-10 15:44:2617
Emircan Uysaler800787f2018-07-16 17:01:4918#include "absl/types/optional.h"
Chen Xingf00bf422019-06-20 08:05:5519#include "api/rtp_packet_infos.h"
Mirko Bonadeid9708072019-01-25 19:26:4820#include "api/scoped_refptr.h"
Emircan Uysaler800787f2018-07-16 17:01:4921#include "api/video/color_space.h"
Johannes Kronfbf16832018-11-05 15:13:0222#include "api/video/hdr_metadata.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3123#include "api/video/video_frame_buffer.h"
Yves Gerey665174f2018-06-19 13:03:0524#include "api/video/video_rotation.h"
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:5725#include "rtc_base/checks.h"
Mirko Bonadeiac194142018-10-22 15:08:3726#include "rtc_base/system/rtc_export.h"
nisseaf916892017-01-10 15:44:2627
nisseaf916892017-01-10 15:44:2628namespace webrtc {
29
Mirko Bonadeiac194142018-10-22 15:08:3730class RTC_EXPORT VideoFrame {
nisseaf916892017-01-10 15:44:2631 public:
Mirko Bonadeid4002a72019-11-12 19:11:4832 struct RTC_EXPORT UpdateRect {
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:5733 int offset_x;
34 int offset_y;
35 int width;
36 int height;
Ilya Nikolaevskiy71aee3a2019-02-18 12:01:2637
38 // Makes this UpdateRect a bounding box of this and other rect.
39 void Union(const UpdateRect& other);
40
41 // Makes this UpdateRect an intersection of this and other rect.
42 void Intersect(const UpdateRect& other);
43
44 // Sets everything to 0, making this UpdateRect a zero-size (empty) update.
45 void MakeEmptyUpdate();
46
47 bool IsEmpty() const;
Ilya Nikolaevskiy0660cee2019-11-19 13:57:5748
49 // Per-member equality check. Empty rectangles with different offsets would
50 // be considered different.
51 bool operator==(const UpdateRect& other) const {
52 return other.offset_x == offset_x && other.offset_y == offset_y &&
53 other.width == width && other.height == height;
54 }
55
56 bool operator!=(const UpdateRect& other) const { return !(*this == other); }
57
58 // Scales update_rect given original frame dimensions.
59 // Cropping is applied first, then rect is scaled down.
60 // Update rect is snapped to 2x2 grid due to possible UV subsampling and
61 // then expanded by additional 2 pixels in each direction to accommodate any
62 // possible scaling artifacts.
63 // Note, close but not equal update_rects on original frame may result in
64 // the same scaled update rects.
65 UpdateRect ScaleWithFrame(int frame_width,
66 int frame_height,
67 int crop_x,
68 int crop_y,
69 int crop_width,
70 int crop_height,
71 int scaled_width,
72 int scaled_height) const;
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:5773 };
74
Johannes Kron05f84872020-01-16 13:09:3375 struct RTC_EXPORT ProcessingTime {
76 TimeDelta Elapsed() const { return finish - start; }
77 Timestamp start;
78 Timestamp finish;
79 };
80
Emircan Uysaler800787f2018-07-16 17:01:4981 // Preferred way of building VideoFrame objects.
Mirko Bonadei62a19d02019-11-11 18:59:5482 class RTC_EXPORT Builder {
Emircan Uysaler800787f2018-07-16 17:01:4983 public:
84 Builder();
85 ~Builder();
86
87 VideoFrame build();
88 Builder& set_video_frame_buffer(
89 const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
90 Builder& set_timestamp_ms(int64_t timestamp_ms);
91 Builder& set_timestamp_us(int64_t timestamp_us);
92 Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
93 Builder& set_ntp_time_ms(int64_t ntp_time_ms);
94 Builder& set_rotation(VideoRotation rotation);
Danil Chapovalovb7698942019-02-05 10:32:1995 Builder& set_color_space(const absl::optional<ColorSpace>& color_space);
Johannes Kron4749e4e2018-11-21 09:18:1896 Builder& set_color_space(const ColorSpace* color_space);
Artem Titov1ebfb6a2019-01-03 22:49:3797 Builder& set_id(uint16_t id);
Artem Titov5256d8b2019-12-02 09:34:1298 Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect);
Chen Xingf00bf422019-06-20 08:05:5599 Builder& set_packet_infos(RtpPacketInfos packet_infos);
Emircan Uysaler800787f2018-07-16 17:01:49100
101 private:
Artem Titov1ebfb6a2019-01-03 22:49:37102 uint16_t id_ = 0;
Emircan Uysaler800787f2018-07-16 17:01:49103 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
104 int64_t timestamp_us_ = 0;
105 uint32_t timestamp_rtp_ = 0;
106 int64_t ntp_time_ms_ = 0;
107 VideoRotation rotation_ = kVideoRotation_0;
108 absl::optional<ColorSpace> color_space_;
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:57109 absl::optional<UpdateRect> update_rect_;
Chen Xingf00bf422019-06-20 08:05:55110 RtpPacketInfos packet_infos_;
Emircan Uysaler800787f2018-07-16 17:01:49111 };
112
113 // To be deprecated. Migrate all use to Builder.
nisseaf916892017-01-10 15:44:26114 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
115 webrtc::VideoRotation rotation,
116 int64_t timestamp_us);
nisseaf916892017-01-10 15:44:26117 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
Niels Möller2ac64462018-06-11 09:14:32118 uint32_t timestamp_rtp,
nisseaf916892017-01-10 15:44:26119 int64_t render_time_ms,
120 VideoRotation rotation);
121
122 ~VideoFrame();
123
124 // Support move and copy.
125 VideoFrame(const VideoFrame&);
126 VideoFrame(VideoFrame&&);
127 VideoFrame& operator=(const VideoFrame&);
128 VideoFrame& operator=(VideoFrame&&);
129
130 // Get frame width.
131 int width() const;
nisseaf916892017-01-10 15:44:26132 // Get frame height.
133 int height() const;
kthelgason2bc68642017-02-07 15:02:22134 // Get frame size in pixels.
135 uint32_t size() const;
nisseaf916892017-01-10 15:44:26136
Philipp Hancke54ea85c2021-01-27 17:10:18137 // Get frame ID. Returns 0 if ID is not set. Not guaranteed to be transferred
138 // from the sender to the receiver, but preserved on the sender side. The id
Artem Titov1ebfb6a2019-01-03 22:49:37139 // should be propagated between all frame modifications during its lifetime
140 // from capturing to sending as encoded image. It is intended to be unique
Philipp Hancke54ea85c2021-01-27 17:10:18141 // over a time window of a few minutes for the peer connection to which the
Artem Titov1ebfb6a2019-01-03 22:49:37142 // corresponding video stream belongs to.
143 uint16_t id() const { return id_; }
144 void set_id(uint16_t id) { id_ = id; }
145
nisseaf916892017-01-10 15:44:26146 // System monotonic clock, same timebase as rtc::TimeMicros().
147 int64_t timestamp_us() const { return timestamp_us_; }
148 void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; }
149
150 // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame
151 // merge, timestamps other than timestamp_us will likely be
152 // deprecated.
153
154 // Set frame timestamp (90kHz).
155 void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
156
157 // Get frame timestamp (90kHz).
158 uint32_t timestamp() const { return timestamp_rtp_; }
159
160 // For now, transport_frame_id and rtp timestamp are the same.
161 // TODO(nisse): Must be handled differently for QUIC.
162 uint32_t transport_frame_id() const { return timestamp(); }
163
164 // Set capture ntp time in milliseconds.
165 void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; }
166
167 // Get capture ntp time in milliseconds.
168 int64_t ntp_time_ms() const { return ntp_time_ms_; }
169
170 // Naming convention for Coordination of Video Orientation. Please see
171 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf
172 //
173 // "pending rotation" or "pending" = a frame that has a VideoRotation > 0.
174 //
175 // "not pending" = a frame that has a VideoRotation == 0.
176 //
177 // "apply rotation" = modify a frame from being "pending" to being "not
178 // pending" rotation (a no-op for "unrotated").
179 //
180 VideoRotation rotation() const { return rotation_; }
181 void set_rotation(VideoRotation rotation) { rotation_ = rotation; }
182
Johannes Kronfbf16832018-11-05 15:13:02183 // Get color space when available.
Danil Chapovalovb7698942019-02-05 10:32:19184 const absl::optional<ColorSpace>& color_space() const { return color_space_; }
185 void set_color_space(const absl::optional<ColorSpace>& color_space) {
186 color_space_ = color_space;
Johannes Kronf7f13e02018-12-12 10:17:43187 }
Johannes Kronfbf16832018-11-05 15:13:02188
Johannes Kron111e9812020-10-26 12:54:40189 // max_composition_delay_in_frames() is used in an experiment of a low-latency
190 // renderer algorithm see crbug.com/1138888.
191 absl::optional<int32_t> max_composition_delay_in_frames() const {
192 return max_composition_delay_in_frames_;
193 }
194 void set_max_composition_delay_in_frames(
195 absl::optional<int32_t> max_composition_delay_in_frames) {
196 max_composition_delay_in_frames_ = max_composition_delay_in_frames;
197 }
198
nisseaf916892017-01-10 15:44:26199 // Get render time in milliseconds.
nisse1c0dea82017-01-30 10:43:18200 // TODO(nisse): Deprecated. Migrate all users to timestamp_us().
nisseaf916892017-01-10 15:44:26201 int64_t render_time_ms() const;
202
Ilya Nikolaevskiy871e1442019-02-11 12:35:03203 // Return the underlying buffer. Never nullptr for a properly
204 // initialized VideoFrame.
nisseaf916892017-01-10 15:44:26205 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;
206
Ilya Nikolaevskiy4fc08552019-06-05 13:59:12207 void set_video_frame_buffer(
208 const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
209
nisseaf916892017-01-10 15:44:26210 // TODO(nisse): Deprecated.
211 // Return true if the frame is stored in a texture.
212 bool is_texture() const {
magjed3f075492017-06-01 17:02:26213 return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative;
nisseaf916892017-01-10 15:44:26214 }
215
Ilya Nikolaevskiy9560d7d2019-10-30 10:19:47216 bool has_update_rect() const { return update_rect_.has_value(); }
217
218 // Returns update_rect set by the builder or set_update_rect() or whole frame
219 // rect if no update rect is available.
220 UpdateRect update_rect() const {
221 return update_rect_.value_or(UpdateRect{0, 0, width(), height()});
222 }
223
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:57224 // Rectangle must be within the frame dimensions.
225 void set_update_rect(const VideoFrame::UpdateRect& update_rect) {
226 RTC_DCHECK_GE(update_rect.offset_x, 0);
227 RTC_DCHECK_GE(update_rect.offset_y, 0);
228 RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width());
229 RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height());
230 update_rect_ = update_rect;
231 }
232
Ilya Nikolaevskiy9560d7d2019-10-30 10:19:47233 void clear_update_rect() { update_rect_ = absl::nullopt; }
234
Chen Xingf00bf422019-06-20 08:05:55235 // Get information about packets used to assemble this video frame. Might be
236 // empty if the information isn't available.
237 const RtpPacketInfos& packet_infos() const { return packet_infos_; }
238 void set_packet_infos(RtpPacketInfos value) {
239 packet_infos_ = std::move(value);
240 }
241
Johannes Kron05f84872020-01-16 13:09:33242 const absl::optional<ProcessingTime> processing_time() const {
243 return processing_time_;
244 }
245 void set_processing_time(const ProcessingTime& processing_time) {
246 processing_time_ = processing_time;
247 }
248
nisseaf916892017-01-10 15:44:26249 private:
Ilya Nikolaevskiy871e1442019-02-11 12:35:03250 VideoFrame(uint16_t id,
251 const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
252 int64_t timestamp_us,
253 uint32_t timestamp_rtp,
254 int64_t ntp_time_ms,
255 VideoRotation rotation,
Ilya Nikolaevskiy6aca0b72019-02-13 10:55:57256 const absl::optional<ColorSpace>& color_space,
Chen Xingf00bf422019-06-20 08:05:55257 const absl::optional<UpdateRect>& update_rect,
Markus Handell026f64f2019-11-21 14:07:19258 RtpPacketInfos packet_infos);
Emircan Uysaler800787f2018-07-16 17:01:49259
Artem Titov1ebfb6a2019-01-03 22:49:37260 uint16_t id_;
Markus Handell026f64f2019-11-21 14:07:19261 // An opaque reference counted handle that stores the pixel data.
nisseaf916892017-01-10 15:44:26262 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
263 uint32_t timestamp_rtp_;
264 int64_t ntp_time_ms_;
265 int64_t timestamp_us_;
266 VideoRotation rotation_;
Emircan Uysaler800787f2018-07-16 17:01:49267 absl::optional<ColorSpace> color_space_;
Johannes Kron111e9812020-10-26 12:54:40268 absl::optional<int32_t> max_composition_delay_in_frames_;
Ilya Nikolaevskiy9560d7d2019-10-30 10:19:47269 // Updated since the last frame area. If present it means that the bounding
270 // box of all the changes is within the rectangular area and is close to it.
271 // If absent, it means that there's no information about the change at all and
272 // update_rect() will return a rectangle corresponding to the entire frame.
273 absl::optional<UpdateRect> update_rect_;
Chen Xingf00bf422019-06-20 08:05:55274 // Information about packets used to assemble this video frame. This is needed
275 // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's
276 // MediaStreamTrack, in order to implement getContributingSources(). See:
277 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
278 RtpPacketInfos packet_infos_;
Johannes Kron05f84872020-01-16 13:09:33279 // Processing timestamps of the frame. For received video frames these are the
280 // timestamps when the frame is sent to the decoder and the decoded image
281 // returned from the decoder.
282 // Currently, not set for locally captured video frames.
283 absl::optional<ProcessingTime> processing_time_;
nisseaf916892017-01-10 15:44:26284};
285
286} // namespace webrtc
287
Mirko Bonadei92ea95e2017-09-15 04:47:31288#endif // API_VIDEO_VIDEO_FRAME_H_