nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #ifndef API_VIDEO_VIDEO_FRAME_H_ |
| 12 | #define API_VIDEO_VIDEO_FRAME_H_ |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 13 | |
| 14 | #include <stdint.h> |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 15 | |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 16 | #include <utility> |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 17 | |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 18 | #include "absl/types/optional.h" |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 19 | #include "api/rtp_packet_infos.h" |
Mirko Bonadei | d970807 | 2019-01-25 19:26:48 | [diff] [blame] | 20 | #include "api/scoped_refptr.h" |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 21 | #include "api/video/color_space.h" |
Johannes Kron | fbf1683 | 2018-11-05 15:13:02 | [diff] [blame] | 22 | #include "api/video/hdr_metadata.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 23 | #include "api/video/video_frame_buffer.h" |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 24 | #include "api/video/video_rotation.h" |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 25 | #include "rtc_base/checks.h" |
Mirko Bonadei | ac19414 | 2018-10-22 15:08:37 | [diff] [blame] | 26 | #include "rtc_base/system/rtc_export.h" |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 27 | |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 28 | namespace webrtc { |
| 29 | |
Mirko Bonadei | ac19414 | 2018-10-22 15:08:37 | [diff] [blame] | 30 | class RTC_EXPORT VideoFrame { |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 31 | public: |
Mirko Bonadei | d4002a7 | 2019-11-12 19:11:48 | [diff] [blame] | 32 | struct RTC_EXPORT UpdateRect { |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 33 | int offset_x; |
| 34 | int offset_y; |
| 35 | int width; |
| 36 | int height; |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 12:01:26 | [diff] [blame] | 37 | |
| 38 | // Makes this UpdateRect a bounding box of this and other rect. |
| 39 | void Union(const UpdateRect& other); |
| 40 | |
| 41 | // Makes this UpdateRect an intersection of this and other rect. |
| 42 | void Intersect(const UpdateRect& other); |
| 43 | |
| 44 | // Sets everything to 0, making this UpdateRect a zero-size (empty) update. |
| 45 | void MakeEmptyUpdate(); |
| 46 | |
| 47 | bool IsEmpty() const; |
Ilya Nikolaevskiy | 0660cee | 2019-11-19 13:57:57 | [diff] [blame] | 48 | |
| 49 | // Per-member equality check. Empty rectangles with different offsets would |
| 50 | // be considered different. |
| 51 | bool operator==(const UpdateRect& other) const { |
| 52 | return other.offset_x == offset_x && other.offset_y == offset_y && |
| 53 | other.width == width && other.height == height; |
| 54 | } |
| 55 | |
| 56 | bool operator!=(const UpdateRect& other) const { return !(*this == other); } |
| 57 | |
| 58 | // Scales update_rect given original frame dimensions. |
| 59 | // Cropping is applied first, then rect is scaled down. |
| 60 | // Update rect is snapped to 2x2 grid due to possible UV subsampling and |
| 61 | // then expanded by additional 2 pixels in each direction to accommodate any |
| 62 | // possible scaling artifacts. |
| 63 | // Note, close but not equal update_rects on original frame may result in |
| 64 | // the same scaled update rects. |
| 65 | UpdateRect ScaleWithFrame(int frame_width, |
| 66 | int frame_height, |
| 67 | int crop_x, |
| 68 | int crop_y, |
| 69 | int crop_width, |
| 70 | int crop_height, |
| 71 | int scaled_width, |
| 72 | int scaled_height) const; |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 73 | }; |
| 74 | |
Johannes Kron | 05f8487 | 2020-01-16 13:09:33 | [diff] [blame] | 75 | struct RTC_EXPORT ProcessingTime { |
| 76 | TimeDelta Elapsed() const { return finish - start; } |
| 77 | Timestamp start; |
| 78 | Timestamp finish; |
| 79 | }; |
| 80 | |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 81 | // Preferred way of building VideoFrame objects. |
Mirko Bonadei | 62a19d0 | 2019-11-11 18:59:54 | [diff] [blame] | 82 | class RTC_EXPORT Builder { |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 83 | public: |
| 84 | Builder(); |
| 85 | ~Builder(); |
| 86 | |
| 87 | VideoFrame build(); |
| 88 | Builder& set_video_frame_buffer( |
| 89 | const rtc::scoped_refptr<VideoFrameBuffer>& buffer); |
| 90 | Builder& set_timestamp_ms(int64_t timestamp_ms); |
| 91 | Builder& set_timestamp_us(int64_t timestamp_us); |
| 92 | Builder& set_timestamp_rtp(uint32_t timestamp_rtp); |
| 93 | Builder& set_ntp_time_ms(int64_t ntp_time_ms); |
| 94 | Builder& set_rotation(VideoRotation rotation); |
Danil Chapovalov | b769894 | 2019-02-05 10:32:19 | [diff] [blame] | 95 | Builder& set_color_space(const absl::optional<ColorSpace>& color_space); |
Johannes Kron | 4749e4e | 2018-11-21 09:18:18 | [diff] [blame] | 96 | Builder& set_color_space(const ColorSpace* color_space); |
Artem Titov | 1ebfb6a | 2019-01-03 22:49:37 | [diff] [blame] | 97 | Builder& set_id(uint16_t id); |
Artem Titov | 5256d8b | 2019-12-02 09:34:12 | [diff] [blame] | 98 | Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect); |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 99 | Builder& set_packet_infos(RtpPacketInfos packet_infos); |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 100 | |
| 101 | private: |
Artem Titov | 1ebfb6a | 2019-01-03 22:49:37 | [diff] [blame] | 102 | uint16_t id_ = 0; |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 103 | rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; |
| 104 | int64_t timestamp_us_ = 0; |
| 105 | uint32_t timestamp_rtp_ = 0; |
| 106 | int64_t ntp_time_ms_ = 0; |
| 107 | VideoRotation rotation_ = kVideoRotation_0; |
| 108 | absl::optional<ColorSpace> color_space_; |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 109 | absl::optional<UpdateRect> update_rect_; |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 110 | RtpPacketInfos packet_infos_; |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 111 | }; |
| 112 | |
| 113 | // To be deprecated. Migrate all use to Builder. |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 114 | VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, |
| 115 | webrtc::VideoRotation rotation, |
| 116 | int64_t timestamp_us); |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 117 | VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, |
Niels Möller | 2ac6446 | 2018-06-11 09:14:32 | [diff] [blame] | 118 | uint32_t timestamp_rtp, |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 119 | int64_t render_time_ms, |
| 120 | VideoRotation rotation); |
| 121 | |
| 122 | ~VideoFrame(); |
| 123 | |
| 124 | // Support move and copy. |
| 125 | VideoFrame(const VideoFrame&); |
| 126 | VideoFrame(VideoFrame&&); |
| 127 | VideoFrame& operator=(const VideoFrame&); |
| 128 | VideoFrame& operator=(VideoFrame&&); |
| 129 | |
| 130 | // Get frame width. |
| 131 | int width() const; |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 132 | // Get frame height. |
| 133 | int height() const; |
kthelgason | 2bc6864 | 2017-02-07 15:02:22 | [diff] [blame] | 134 | // Get frame size in pixels. |
| 135 | uint32_t size() const; |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 136 | |
Philipp Hancke | 54ea85c | 2021-01-27 17:10:18 | [diff] [blame] | 137 | // Get frame ID. Returns 0 if ID is not set. Not guaranteed to be transferred |
| 138 | // from the sender to the receiver, but preserved on the sender side. The id |
Artem Titov | 1ebfb6a | 2019-01-03 22:49:37 | [diff] [blame] | 139 | // should be propagated between all frame modifications during its lifetime |
| 140 | // from capturing to sending as encoded image. It is intended to be unique |
Philipp Hancke | 54ea85c | 2021-01-27 17:10:18 | [diff] [blame] | 141 | // over a time window of a few minutes for the peer connection to which the |
Artem Titov | 1ebfb6a | 2019-01-03 22:49:37 | [diff] [blame] | 142 | // corresponding video stream belongs to. |
| 143 | uint16_t id() const { return id_; } |
| 144 | void set_id(uint16_t id) { id_ = id; } |
| 145 | |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 146 | // System monotonic clock, same timebase as rtc::TimeMicros(). |
| 147 | int64_t timestamp_us() const { return timestamp_us_; } |
| 148 | void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; } |
| 149 | |
| 150 | // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame |
| 151 | // merge, timestamps other than timestamp_us will likely be |
| 152 | // deprecated. |
| 153 | |
| 154 | // Set frame timestamp (90kHz). |
| 155 | void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } |
| 156 | |
| 157 | // Get frame timestamp (90kHz). |
| 158 | uint32_t timestamp() const { return timestamp_rtp_; } |
| 159 | |
| 160 | // For now, transport_frame_id and rtp timestamp are the same. |
| 161 | // TODO(nisse): Must be handled differently for QUIC. |
| 162 | uint32_t transport_frame_id() const { return timestamp(); } |
| 163 | |
| 164 | // Set capture ntp time in milliseconds. |
| 165 | void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; } |
| 166 | |
| 167 | // Get capture ntp time in milliseconds. |
| 168 | int64_t ntp_time_ms() const { return ntp_time_ms_; } |
| 169 | |
| 170 | // Naming convention for Coordination of Video Orientation. Please see |
| 171 | // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf |
| 172 | // |
| 173 | // "pending rotation" or "pending" = a frame that has a VideoRotation > 0. |
| 174 | // |
| 175 | // "not pending" = a frame that has a VideoRotation == 0. |
| 176 | // |
| 177 | // "apply rotation" = modify a frame from being "pending" to being "not |
| 178 | // pending" rotation (a no-op for "unrotated"). |
| 179 | // |
| 180 | VideoRotation rotation() const { return rotation_; } |
| 181 | void set_rotation(VideoRotation rotation) { rotation_ = rotation; } |
| 182 | |
Johannes Kron | fbf1683 | 2018-11-05 15:13:02 | [diff] [blame] | 183 | // Get color space when available. |
Danil Chapovalov | b769894 | 2019-02-05 10:32:19 | [diff] [blame] | 184 | const absl::optional<ColorSpace>& color_space() const { return color_space_; } |
| 185 | void set_color_space(const absl::optional<ColorSpace>& color_space) { |
| 186 | color_space_ = color_space; |
Johannes Kron | f7f13e0 | 2018-12-12 10:17:43 | [diff] [blame] | 187 | } |
Johannes Kron | fbf1683 | 2018-11-05 15:13:02 | [diff] [blame] | 188 | |
Johannes Kron | 111e981 | 2020-10-26 12:54:40 | [diff] [blame] | 189 | // max_composition_delay_in_frames() is used in an experiment of a low-latency |
| 190 | // renderer algorithm see crbug.com/1138888. |
| 191 | absl::optional<int32_t> max_composition_delay_in_frames() const { |
| 192 | return max_composition_delay_in_frames_; |
| 193 | } |
| 194 | void set_max_composition_delay_in_frames( |
| 195 | absl::optional<int32_t> max_composition_delay_in_frames) { |
| 196 | max_composition_delay_in_frames_ = max_composition_delay_in_frames; |
| 197 | } |
| 198 | |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 199 | // Get render time in milliseconds. |
nisse | 1c0dea8 | 2017-01-30 10:43:18 | [diff] [blame] | 200 | // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 201 | int64_t render_time_ms() const; |
| 202 | |
Ilya Nikolaevskiy | 871e144 | 2019-02-11 12:35:03 | [diff] [blame] | 203 | // Return the underlying buffer. Never nullptr for a properly |
| 204 | // initialized VideoFrame. |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 205 | rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const; |
| 206 | |
Ilya Nikolaevskiy | 4fc0855 | 2019-06-05 13:59:12 | [diff] [blame] | 207 | void set_video_frame_buffer( |
| 208 | const rtc::scoped_refptr<VideoFrameBuffer>& buffer); |
| 209 | |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 210 | // TODO(nisse): Deprecated. |
| 211 | // Return true if the frame is stored in a texture. |
| 212 | bool is_texture() const { |
magjed | 3f07549 | 2017-06-01 17:02:26 | [diff] [blame] | 213 | return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative; |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 214 | } |
| 215 | |
Ilya Nikolaevskiy | 9560d7d | 2019-10-30 10:19:47 | [diff] [blame] | 216 | bool has_update_rect() const { return update_rect_.has_value(); } |
| 217 | |
| 218 | // Returns update_rect set by the builder or set_update_rect() or whole frame |
| 219 | // rect if no update rect is available. |
| 220 | UpdateRect update_rect() const { |
| 221 | return update_rect_.value_or(UpdateRect{0, 0, width(), height()}); |
| 222 | } |
| 223 | |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 224 | // Rectangle must be within the frame dimensions. |
| 225 | void set_update_rect(const VideoFrame::UpdateRect& update_rect) { |
| 226 | RTC_DCHECK_GE(update_rect.offset_x, 0); |
| 227 | RTC_DCHECK_GE(update_rect.offset_y, 0); |
| 228 | RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width()); |
| 229 | RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height()); |
| 230 | update_rect_ = update_rect; |
| 231 | } |
| 232 | |
Ilya Nikolaevskiy | 9560d7d | 2019-10-30 10:19:47 | [diff] [blame] | 233 | void clear_update_rect() { update_rect_ = absl::nullopt; } |
| 234 | |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 235 | // Get information about packets used to assemble this video frame. Might be |
| 236 | // empty if the information isn't available. |
| 237 | const RtpPacketInfos& packet_infos() const { return packet_infos_; } |
| 238 | void set_packet_infos(RtpPacketInfos value) { |
| 239 | packet_infos_ = std::move(value); |
| 240 | } |
| 241 | |
Johannes Kron | 05f8487 | 2020-01-16 13:09:33 | [diff] [blame] | 242 | const absl::optional<ProcessingTime> processing_time() const { |
| 243 | return processing_time_; |
| 244 | } |
| 245 | void set_processing_time(const ProcessingTime& processing_time) { |
| 246 | processing_time_ = processing_time; |
| 247 | } |
| 248 | |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 249 | private: |
Ilya Nikolaevskiy | 871e144 | 2019-02-11 12:35:03 | [diff] [blame] | 250 | VideoFrame(uint16_t id, |
| 251 | const rtc::scoped_refptr<VideoFrameBuffer>& buffer, |
| 252 | int64_t timestamp_us, |
| 253 | uint32_t timestamp_rtp, |
| 254 | int64_t ntp_time_ms, |
| 255 | VideoRotation rotation, |
Ilya Nikolaevskiy | 6aca0b7 | 2019-02-13 10:55:57 | [diff] [blame] | 256 | const absl::optional<ColorSpace>& color_space, |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 257 | const absl::optional<UpdateRect>& update_rect, |
Markus Handell | 026f64f | 2019-11-21 14:07:19 | [diff] [blame] | 258 | RtpPacketInfos packet_infos); |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 259 | |
Artem Titov | 1ebfb6a | 2019-01-03 22:49:37 | [diff] [blame] | 260 | uint16_t id_; |
Markus Handell | 026f64f | 2019-11-21 14:07:19 | [diff] [blame] | 261 | // An opaque reference counted handle that stores the pixel data. |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 262 | rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; |
| 263 | uint32_t timestamp_rtp_; |
| 264 | int64_t ntp_time_ms_; |
| 265 | int64_t timestamp_us_; |
| 266 | VideoRotation rotation_; |
Emircan Uysaler | 800787f | 2018-07-16 17:01:49 | [diff] [blame] | 267 | absl::optional<ColorSpace> color_space_; |
Johannes Kron | 111e981 | 2020-10-26 12:54:40 | [diff] [blame] | 268 | absl::optional<int32_t> max_composition_delay_in_frames_; |
Ilya Nikolaevskiy | 9560d7d | 2019-10-30 10:19:47 | [diff] [blame] | 269 | // Updated since the last frame area. If present it means that the bounding |
| 270 | // box of all the changes is within the rectangular area and is close to it. |
| 271 | // If absent, it means that there's no information about the change at all and |
| 272 | // update_rect() will return a rectangle corresponding to the entire frame. |
| 273 | absl::optional<UpdateRect> update_rect_; |
Chen Xing | f00bf42 | 2019-06-20 08:05:55 | [diff] [blame] | 274 | // Information about packets used to assemble this video frame. This is needed |
| 275 | // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's |
| 276 | // MediaStreamTrack, in order to implement getContributingSources(). See: |
| 277 | // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources |
| 278 | RtpPacketInfos packet_infos_; |
Johannes Kron | 05f8487 | 2020-01-16 13:09:33 | [diff] [blame] | 279 | // Processing timestamps of the frame. For received video frames these are the |
| 280 | // timestamps when the frame is sent to the decoder and the decoded image |
| 281 | // returned from the decoder. |
| 282 | // Currently, not set for locally captured video frames. |
| 283 | absl::optional<ProcessingTime> processing_time_; |
nisse | af91689 | 2017-01-10 15:44:26 | [diff] [blame] | 284 | }; |
| 285 | |
| 286 | } // namespace webrtc |
| 287 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 288 | #endif // API_VIDEO_VIDEO_FRAME_H_ |