blob: b1f5426dd391308bd0cbf61451e9a284159a6db3 [file] [log] [blame]
Marina Cioceae3e07bf2020-02-27 15:28:511/*
2 * Copyright 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef API_FRAME_TRANSFORMER_INTERFACE_H_
12#define API_FRAME_TRANSFORMER_INTERFACE_H_
13
Harald Alvestrand6431a642024-06-04 21:29:1414#include <cstdint>
Marina Cioceae3e07bf2020-02-27 15:28:5115#include <memory>
Philipp Hancke0bace222023-10-23 09:57:4316#include <string>
Marina Cioceae3e07bf2020-02-27 15:28:5117
Harald Alvestrand6431a642024-06-04 21:29:1418#include "absl/types/optional.h"
19#include "api/array_view.h"
Harald Alvestrande8a2b3c2023-10-31 13:30:3020#include "api/ref_count.h"
Marina Cioceae3e07bf2020-02-27 15:28:5121#include "api/scoped_refptr.h"
Harald Alvestrand6431a642024-06-04 21:29:1422#include "api/units/timestamp.h"
Marina Cioceacdc89b42020-05-14 18:01:0223#include "api/video/video_frame_metadata.h"
Harald Alvestrand6431a642024-06-04 21:29:1424#include "rtc_base/system/rtc_export.h"
Marina Cioceae3e07bf2020-02-27 15:28:5125
26namespace webrtc {
27
Marina Cioceac24b6b72020-03-30 12:51:1028// Owns the frame payload data.
29class TransformableFrameInterface {
30 public:
Tony Herrea45c7052024-05-16 11:38:2531 // Only a known list of internal implementations of transformable frames are
32 // permitted to allow internal downcasting. This is enforced via the
33 // internally-constructable Passkey.
34 // TODO: bugs.webrtc.org/339815768 - Remove this passkey once the
35 // downcasts are removed.
36 class Passkey;
37 RTC_EXPORT explicit TransformableFrameInterface(Passkey);
38
39 TransformableFrameInterface(TransformableFrameInterface&&) = default;
40 TransformableFrameInterface& operator=(TransformableFrameInterface&&) =
41 default;
42
Marina Cioceac24b6b72020-03-30 12:51:1043 virtual ~TransformableFrameInterface() = default;
44
45 // Returns the frame payload data. The data is valid until the next non-const
46 // method call.
47 virtual rtc::ArrayView<const uint8_t> GetData() const = 0;
48
Artem Titov0e61fdd2021-07-25 19:50:1449 // Copies `data` into the owned frame payload data.
Marina Cioceac24b6b72020-03-30 12:51:1050 virtual void SetData(rtc::ArrayView<const uint8_t> data) = 0;
51
Olga Sharonova92e9ff62021-09-02 07:58:2152 virtual uint8_t GetPayloadType() const = 0;
Marina Cioceac24b6b72020-03-30 12:51:1053 virtual uint32_t GetSsrc() const = 0;
Philipp Hancke2ace42f2021-08-24 08:40:1254 virtual uint32_t GetTimestamp() const = 0;
Tony Herreb4062e52023-06-29 09:02:2255 virtual void SetRTPTimestamp(uint32_t timestamp) = 0;
56
Palak Agarwala09f21b2023-02-22 13:46:2357 // TODO(https://bugs.webrtc.org/14878): Change this to pure virtual after it
58 // is implemented everywhere.
59 virtual absl::optional<Timestamp> GetCaptureTimeIdentifier() const {
60 return absl::nullopt;
61 }
Tony Herre8fb41a32021-09-24 12:05:2062
63 enum class Direction {
64 kUnknown,
65 kReceiver,
66 kSender,
67 };
68 // TODO(crbug.com/1250638): Remove this distinction between receiver and
69 // sender frames to allow received frames to be directly re-transmitted on
70 // other PeerConnectionss.
71 virtual Direction GetDirection() const { return Direction::kUnknown; }
Philipp Hancke3e3881a2023-11-06 10:28:3072 virtual std::string GetMimeType() const = 0;
Marina Cioceac24b6b72020-03-30 12:51:1073};
74
75class TransformableVideoFrameInterface : public TransformableFrameInterface {
76 public:
Tony Herrea45c7052024-05-16 11:38:2577 RTC_EXPORT explicit TransformableVideoFrameInterface(Passkey passkey);
Marina Cioceac24b6b72020-03-30 12:51:1078 virtual ~TransformableVideoFrameInterface() = default;
79 virtual bool IsKeyFrame() const = 0;
80
Tony Herre6d262c52023-02-24 11:20:2881 virtual VideoFrameMetadata Metadata() const = 0;
82
Tony Herrec381c332023-04-13 07:37:5583 virtual void SetMetadata(const VideoFrameMetadata&) = 0;
Marina Cioceac24b6b72020-03-30 12:51:1084};
85
Marina Ciocea48623202020-04-01 08:19:4486// Extends the TransformableFrameInterface to expose audio-specific information.
87class TransformableAudioFrameInterface : public TransformableFrameInterface {
88 public:
Tony Herrea45c7052024-05-16 11:38:2589 RTC_EXPORT explicit TransformableAudioFrameInterface(Passkey passkey);
Marina Ciocea48623202020-04-01 08:19:4490 virtual ~TransformableAudioFrameInterface() = default;
91
Sergio Garcia Murillo15dfc5a2022-10-11 10:34:4492 virtual rtc::ArrayView<const uint32_t> GetContributingSources() const = 0;
Tony Herre097a4de2023-06-19 15:13:0893
Tony Herre7aa79722024-01-29 07:31:1494 virtual const absl::optional<uint16_t> SequenceNumber() const = 0;
Tony Herrefc68f1f2023-06-22 13:32:2495
Palak Agarwal05051152023-09-20 08:01:1896 virtual absl::optional<uint64_t> AbsoluteCaptureTimestamp() const = 0;
Tony Herrefc68f1f2023-06-22 13:32:2497
98 enum class FrameType { kEmptyFrame, kAudioFrameSpeech, kAudioFrameCN };
99
100 // TODO(crbug.com/1456628): Change this to pure virtual after it
101 // is implemented everywhere.
102 virtual FrameType Type() const { return FrameType::kEmptyFrame; }
Tony Herre64437e82024-04-29 13:13:48103
104 // Audio level in -dBov. Values range from 0 to 127, representing 0 to -127
105 // dBov. 127 represents digital silence. Only present on remote frames if
106 // the audio level header extension was included.
107 virtual absl::optional<uint8_t> AudioLevel() const = 0;
Lionel Koenig GĂ©lasb4462512024-07-30 13:47:41108
109 // Timestamp at which the packet has been first seen on the network interface.
110 // Only defined for received audio packet.
111 virtual absl::optional<Timestamp> ReceiveTime() const = 0;
Marina Ciocea48623202020-04-01 08:19:44112};
113
Marina Cioceae3e07bf2020-02-27 15:28:51114// Objects implement this interface to be notified with the transformed frame.
Harald Alvestrand6431a642024-06-04 21:29:14115class TransformedFrameCallback : public RefCountInterface {
Marina Cioceae3e07bf2020-02-27 15:28:51116 public:
117 virtual void OnTransformedFrame(
Marina Ciocea81be4212020-05-05 14:03:54118 std::unique_ptr<TransformableFrameInterface> frame) = 0;
Marina Cioceae3e07bf2020-02-27 15:28:51119
Tony Herre6e956052023-11-16 13:59:54120 // Request to no longer be called on each frame, instead having frames be
121 // sent directly to OnTransformedFrame without additional work.
122 // TODO(crbug.com/1502781): Make pure virtual once all mocks have
123 // implementations.
124 virtual void StartShortCircuiting() {}
125
Marina Cioceae3e07bf2020-02-27 15:28:51126 protected:
127 ~TransformedFrameCallback() override = default;
128};
129
Marina Cioceac24b6b72020-03-30 12:51:10130// Transforms encoded frames. The transformed frame is sent in a callback using
131// the TransformedFrameCallback interface (see above).
Harald Alvestrand6431a642024-06-04 21:29:14132class FrameTransformerInterface : public RefCountInterface {
Marina Cioceae3e07bf2020-02-27 15:28:51133 public:
Artem Titov0e61fdd2021-07-25 19:50:14134 // Transforms `frame` using the implementing class' processing logic.
Marina Cioceac24b6b72020-03-30 12:51:10135 virtual void Transform(
Marina Ciocea81be4212020-05-05 14:03:54136 std::unique_ptr<TransformableFrameInterface> transformable_frame) = 0;
Marina Cioceae3e07bf2020-02-27 15:28:51137
138 virtual void RegisterTransformedFrameCallback(
Marina Cioceafdabfbc2020-04-10 16:40:11139 rtc::scoped_refptr<TransformedFrameCallback>) {}
140 virtual void RegisterTransformedFrameSinkCallback(
141 rtc::scoped_refptr<TransformedFrameCallback>,
142 uint32_t ssrc) {}
143 virtual void UnregisterTransformedFrameCallback() {}
144 virtual void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) {}
Marina Cioceae3e07bf2020-02-27 15:28:51145
146 protected:
147 ~FrameTransformerInterface() override = default;
148};
149
Harald Alvestrandb0e70572024-04-23 14:04:18150// An interface implemented by classes that can host a transform.
151// Currently this is implemented by the RTCRtpSender and RTCRtpReceiver.
152class FrameTransformerHost {
153 public:
154 virtual ~FrameTransformerHost() {}
155 virtual void SetFrameTransformer(
156 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) = 0;
157 // TODO: bugs.webrtc.org/15929 - To be added:
158 // virtual AddIncomingMediaType(RtpCodec codec) = 0;
159 // virtual AddOutgoingMediaType(RtpCodec codec) = 0;
160};
161
Tony Herrea45c7052024-05-16 11:38:25162//------------------------------------------------------------------------------
163// Implementation details follow
164//------------------------------------------------------------------------------
165class TransformableFrameInterface::Passkey {
166 public:
167 ~Passkey() = default;
168
169 private:
170 // Explicit list of allowed internal implmentations of
171 // TransformableFrameInterface.
172 friend class TransformableOutgoingAudioFrame;
173 friend class TransformableIncomingAudioFrame;
174 friend class TransformableVideoSenderFrame;
175 friend class TransformableVideoReceiverFrame;
176
177 friend class MockTransformableFrame;
178 friend class MockTransformableAudioFrame;
179 friend class MockTransformableVideoFrame;
180 Passkey() = default;
181};
182
Marina Cioceae3e07bf2020-02-27 15:28:51183} // namespace webrtc
184
185#endif // API_FRAME_TRANSFORMER_INTERFACE_H_