Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2020 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #ifndef API_FRAME_TRANSFORMER_INTERFACE_H_ |
| 12 | #define API_FRAME_TRANSFORMER_INTERFACE_H_ |
| 13 | |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 14 | #include <cstdint> |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 15 | #include <memory> |
Philipp Hancke | 0bace22 | 2023-10-23 09:57:43 | [diff] [blame] | 16 | #include <string> |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 17 | |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 18 | #include "absl/types/optional.h" |
| 19 | #include "api/array_view.h" |
Harald Alvestrand | e8a2b3c | 2023-10-31 13:30:30 | [diff] [blame] | 20 | #include "api/ref_count.h" |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 21 | #include "api/scoped_refptr.h" |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 22 | #include "api/units/timestamp.h" |
Marina Ciocea | cdc89b4 | 2020-05-14 18:01:02 | [diff] [blame] | 23 | #include "api/video/video_frame_metadata.h" |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 24 | #include "rtc_base/system/rtc_export.h" |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 25 | |
| 26 | namespace webrtc { |
| 27 | |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 28 | // Owns the frame payload data. |
| 29 | class TransformableFrameInterface { |
| 30 | public: |
Tony Herre | a45c705 | 2024-05-16 11:38:25 | [diff] [blame] | 31 | // Only a known list of internal implementations of transformable frames are |
| 32 | // permitted to allow internal downcasting. This is enforced via the |
| 33 | // internally-constructable Passkey. |
| 34 | // TODO: bugs.webrtc.org/339815768 - Remove this passkey once the |
| 35 | // downcasts are removed. |
| 36 | class Passkey; |
| 37 | RTC_EXPORT explicit TransformableFrameInterface(Passkey); |
| 38 | |
| 39 | TransformableFrameInterface(TransformableFrameInterface&&) = default; |
| 40 | TransformableFrameInterface& operator=(TransformableFrameInterface&&) = |
| 41 | default; |
| 42 | |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 43 | virtual ~TransformableFrameInterface() = default; |
| 44 | |
| 45 | // Returns the frame payload data. The data is valid until the next non-const |
| 46 | // method call. |
| 47 | virtual rtc::ArrayView<const uint8_t> GetData() const = 0; |
| 48 | |
Artem Titov | 0e61fdd | 2021-07-25 19:50:14 | [diff] [blame] | 49 | // Copies `data` into the owned frame payload data. |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 50 | virtual void SetData(rtc::ArrayView<const uint8_t> data) = 0; |
| 51 | |
Olga Sharonova | 92e9ff6 | 2021-09-02 07:58:21 | [diff] [blame] | 52 | virtual uint8_t GetPayloadType() const = 0; |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 53 | virtual uint32_t GetSsrc() const = 0; |
Philipp Hancke | 2ace42f | 2021-08-24 08:40:12 | [diff] [blame] | 54 | virtual uint32_t GetTimestamp() const = 0; |
Tony Herre | b4062e5 | 2023-06-29 09:02:22 | [diff] [blame] | 55 | virtual void SetRTPTimestamp(uint32_t timestamp) = 0; |
| 56 | |
Palak Agarwal | a09f21b | 2023-02-22 13:46:23 | [diff] [blame] | 57 | // TODO(https://bugs.webrtc.org/14878): Change this to pure virtual after it |
| 58 | // is implemented everywhere. |
| 59 | virtual absl::optional<Timestamp> GetCaptureTimeIdentifier() const { |
| 60 | return absl::nullopt; |
| 61 | } |
Tony Herre | 8fb41a3 | 2021-09-24 12:05:20 | [diff] [blame] | 62 | |
| 63 | enum class Direction { |
| 64 | kUnknown, |
| 65 | kReceiver, |
| 66 | kSender, |
| 67 | }; |
| 68 | // TODO(crbug.com/1250638): Remove this distinction between receiver and |
| 69 | // sender frames to allow received frames to be directly re-transmitted on |
| 70 | // other PeerConnectionss. |
| 71 | virtual Direction GetDirection() const { return Direction::kUnknown; } |
Philipp Hancke | 3e3881a | 2023-11-06 10:28:30 | [diff] [blame] | 72 | virtual std::string GetMimeType() const = 0; |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 73 | }; |
| 74 | |
| 75 | class TransformableVideoFrameInterface : public TransformableFrameInterface { |
| 76 | public: |
Tony Herre | a45c705 | 2024-05-16 11:38:25 | [diff] [blame] | 77 | RTC_EXPORT explicit TransformableVideoFrameInterface(Passkey passkey); |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 78 | virtual ~TransformableVideoFrameInterface() = default; |
| 79 | virtual bool IsKeyFrame() const = 0; |
| 80 | |
Tony Herre | 6d262c5 | 2023-02-24 11:20:28 | [diff] [blame] | 81 | virtual VideoFrameMetadata Metadata() const = 0; |
| 82 | |
Tony Herre | c381c33 | 2023-04-13 07:37:55 | [diff] [blame] | 83 | virtual void SetMetadata(const VideoFrameMetadata&) = 0; |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 84 | }; |
| 85 | |
Marina Ciocea | 4862320 | 2020-04-01 08:19:44 | [diff] [blame] | 86 | // Extends the TransformableFrameInterface to expose audio-specific information. |
| 87 | class TransformableAudioFrameInterface : public TransformableFrameInterface { |
| 88 | public: |
Tony Herre | a45c705 | 2024-05-16 11:38:25 | [diff] [blame] | 89 | RTC_EXPORT explicit TransformableAudioFrameInterface(Passkey passkey); |
Marina Ciocea | 4862320 | 2020-04-01 08:19:44 | [diff] [blame] | 90 | virtual ~TransformableAudioFrameInterface() = default; |
| 91 | |
Sergio Garcia Murillo | 15dfc5a | 2022-10-11 10:34:44 | [diff] [blame] | 92 | virtual rtc::ArrayView<const uint32_t> GetContributingSources() const = 0; |
Tony Herre | 097a4de | 2023-06-19 15:13:08 | [diff] [blame] | 93 | |
Tony Herre | 7aa7972 | 2024-01-29 07:31:14 | [diff] [blame] | 94 | virtual const absl::optional<uint16_t> SequenceNumber() const = 0; |
Tony Herre | fc68f1f | 2023-06-22 13:32:24 | [diff] [blame] | 95 | |
Palak Agarwal | 0505115 | 2023-09-20 08:01:18 | [diff] [blame] | 96 | virtual absl::optional<uint64_t> AbsoluteCaptureTimestamp() const = 0; |
Tony Herre | fc68f1f | 2023-06-22 13:32:24 | [diff] [blame] | 97 | |
| 98 | enum class FrameType { kEmptyFrame, kAudioFrameSpeech, kAudioFrameCN }; |
| 99 | |
| 100 | // TODO(crbug.com/1456628): Change this to pure virtual after it |
| 101 | // is implemented everywhere. |
| 102 | virtual FrameType Type() const { return FrameType::kEmptyFrame; } |
Tony Herre | 64437e8 | 2024-04-29 13:13:48 | [diff] [blame] | 103 | |
| 104 | // Audio level in -dBov. Values range from 0 to 127, representing 0 to -127 |
| 105 | // dBov. 127 represents digital silence. Only present on remote frames if |
| 106 | // the audio level header extension was included. |
| 107 | virtual absl::optional<uint8_t> AudioLevel() const = 0; |
Lionel Koenig GĂ©las | b446251 | 2024-07-30 13:47:41 | [diff] [blame] | 108 | |
| 109 | // Timestamp at which the packet has been first seen on the network interface. |
| 110 | // Only defined for received audio packet. |
| 111 | virtual absl::optional<Timestamp> ReceiveTime() const = 0; |
Marina Ciocea | 4862320 | 2020-04-01 08:19:44 | [diff] [blame] | 112 | }; |
| 113 | |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 114 | // Objects implement this interface to be notified with the transformed frame. |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 115 | class TransformedFrameCallback : public RefCountInterface { |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 116 | public: |
| 117 | virtual void OnTransformedFrame( |
Marina Ciocea | 81be421 | 2020-05-05 14:03:54 | [diff] [blame] | 118 | std::unique_ptr<TransformableFrameInterface> frame) = 0; |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 119 | |
Tony Herre | 6e95605 | 2023-11-16 13:59:54 | [diff] [blame] | 120 | // Request to no longer be called on each frame, instead having frames be |
| 121 | // sent directly to OnTransformedFrame without additional work. |
| 122 | // TODO(crbug.com/1502781): Make pure virtual once all mocks have |
| 123 | // implementations. |
| 124 | virtual void StartShortCircuiting() {} |
| 125 | |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 126 | protected: |
| 127 | ~TransformedFrameCallback() override = default; |
| 128 | }; |
| 129 | |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 130 | // Transforms encoded frames. The transformed frame is sent in a callback using |
| 131 | // the TransformedFrameCallback interface (see above). |
Harald Alvestrand | 6431a64 | 2024-06-04 21:29:14 | [diff] [blame] | 132 | class FrameTransformerInterface : public RefCountInterface { |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 133 | public: |
Artem Titov | 0e61fdd | 2021-07-25 19:50:14 | [diff] [blame] | 134 | // Transforms `frame` using the implementing class' processing logic. |
Marina Ciocea | c24b6b7 | 2020-03-30 12:51:10 | [diff] [blame] | 135 | virtual void Transform( |
Marina Ciocea | 81be421 | 2020-05-05 14:03:54 | [diff] [blame] | 136 | std::unique_ptr<TransformableFrameInterface> transformable_frame) = 0; |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 137 | |
| 138 | virtual void RegisterTransformedFrameCallback( |
Marina Ciocea | fdabfbc | 2020-04-10 16:40:11 | [diff] [blame] | 139 | rtc::scoped_refptr<TransformedFrameCallback>) {} |
| 140 | virtual void RegisterTransformedFrameSinkCallback( |
| 141 | rtc::scoped_refptr<TransformedFrameCallback>, |
| 142 | uint32_t ssrc) {} |
| 143 | virtual void UnregisterTransformedFrameCallback() {} |
| 144 | virtual void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) {} |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 145 | |
| 146 | protected: |
| 147 | ~FrameTransformerInterface() override = default; |
| 148 | }; |
| 149 | |
Harald Alvestrand | b0e7057 | 2024-04-23 14:04:18 | [diff] [blame] | 150 | // An interface implemented by classes that can host a transform. |
| 151 | // Currently this is implemented by the RTCRtpSender and RTCRtpReceiver. |
| 152 | class FrameTransformerHost { |
| 153 | public: |
| 154 | virtual ~FrameTransformerHost() {} |
| 155 | virtual void SetFrameTransformer( |
| 156 | rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) = 0; |
| 157 | // TODO: bugs.webrtc.org/15929 - To be added: |
| 158 | // virtual AddIncomingMediaType(RtpCodec codec) = 0; |
| 159 | // virtual AddOutgoingMediaType(RtpCodec codec) = 0; |
| 160 | }; |
| 161 | |
Tony Herre | a45c705 | 2024-05-16 11:38:25 | [diff] [blame] | 162 | //------------------------------------------------------------------------------ |
| 163 | // Implementation details follow |
| 164 | //------------------------------------------------------------------------------ |
| 165 | class TransformableFrameInterface::Passkey { |
| 166 | public: |
| 167 | ~Passkey() = default; |
| 168 | |
| 169 | private: |
| 170 | // Explicit list of allowed internal implmentations of |
| 171 | // TransformableFrameInterface. |
| 172 | friend class TransformableOutgoingAudioFrame; |
| 173 | friend class TransformableIncomingAudioFrame; |
| 174 | friend class TransformableVideoSenderFrame; |
| 175 | friend class TransformableVideoReceiverFrame; |
| 176 | |
| 177 | friend class MockTransformableFrame; |
| 178 | friend class MockTransformableAudioFrame; |
| 179 | friend class MockTransformableVideoFrame; |
| 180 | Passkey() = default; |
| 181 | }; |
| 182 | |
Marina Ciocea | e3e07bf | 2020-02-27 15:28:51 | [diff] [blame] | 183 | } // namespace webrtc |
| 184 | |
| 185 | #endif // API_FRAME_TRANSFORMER_INTERFACE_H_ |