blob: 192e239535b3419efd63d37c4a76da67213a5432 [file] [log] [blame]
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h"
#include <cstdio>
#include <memory>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "api/call/transport.h"
#include "api/test/mock_frame_transformer.h"
#include "api/test/mock_transformable_video_frame.h"
#include "api/units/timestamp.h"
#include "call/video_receive_stream.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "rtc_base/event.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::NiceMock;
using ::testing::NotNull;
using ::testing::Return;
using ::testing::SaveArg;
const int kFirstSeqNum = 1;
const int kLastSeqNum = 2;
std::unique_ptr<RtpFrameObject> CreateRtpFrameObject(
const RTPVideoHeader& video_header,
std::vector<uint32_t> csrcs) {
RtpPacketInfo packet_info(/*ssrc=*/123, csrcs, /*rtc_timestamp=*/0,
/*receive_time=*/Timestamp::Seconds(123456));
return std::make_unique<RtpFrameObject>(
kFirstSeqNum, kLastSeqNum, /*markerBit=*/true,
/*times_nacked=*/3, /*first_packet_received_time=*/4,
/*last_packet_received_time=*/5, /*rtp_timestamp=*/6, /*ntp_time_ms=*/7,
VideoSendTiming(), /*payload_type=*/8, video_header.codec,
kVideoRotation_0, VideoContentType::UNSPECIFIED, video_header,
absl::nullopt, RtpPacketInfos({packet_info}),
EncodedImageBuffer::Create(0));
}
std::unique_ptr<RtpFrameObject> CreateRtpFrameObject() {
return CreateRtpFrameObject(RTPVideoHeader(), /*csrcs=*/{});
}
class TestRtpVideoFrameReceiver : public RtpVideoFrameReceiver {
public:
TestRtpVideoFrameReceiver() {}
~TestRtpVideoFrameReceiver() override = default;
MOCK_METHOD(void,
ManageFrame,
(std::unique_ptr<RtpFrameObject> frame),
(override));
};
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
RegisterTransformedFrameCallbackSinkOnInit) {
TestRtpVideoFrameReceiver receiver;
auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>());
SimulatedClock clock(0);
auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111));
EXPECT_CALL(*frame_transformer,
RegisterTransformedFrameSinkCallback(testing::_, 1111));
delegate->Init();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
UnregisterTransformedFrameSinkCallbackOnReset) {
TestRtpVideoFrameReceiver receiver;
auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>());
SimulatedClock clock(0);
auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111));
EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameSinkCallback(1111));
delegate->Reset();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) {
TestRtpVideoFrameReceiver receiver;
auto frame_transformer(
rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111));
auto frame = CreateRtpFrameObject();
EXPECT_CALL(*frame_transformer, Transform);
delegate->TransformFrame(std::move(frame));
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
ManageFrameOnTransformedFrame) {
rtc::AutoThread main_thread_;
TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer(
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
std::vector<uint32_t> csrcs = {234, 345, 456};
auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111);
rtc::scoped_refptr<TransformedFrameCallback> callback;
EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
.WillOnce(SaveArg<0>(&callback));
delegate->Init();
ASSERT_TRUE(callback);
EXPECT_CALL(receiver, ManageFrame)
.WillOnce([&](std::unique_ptr<RtpFrameObject> frame) {
EXPECT_EQ(frame->Csrcs(), csrcs);
EXPECT_EQ(frame->first_seq_num(), kFirstSeqNum);
EXPECT_EQ(frame->last_seq_num(), kLastSeqNum);
});
ON_CALL(*mock_frame_transformer, Transform)
.WillByDefault(
[&callback](std::unique_ptr<TransformableFrameInterface> frame) {
EXPECT_STRCASEEQ("video/Generic", frame->GetMimeType().c_str());
callback->OnTransformedFrame(std::move(frame));
});
delegate->TransformFrame(CreateRtpFrameObject(RTPVideoHeader(), csrcs));
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TransformableFrameMetadataHasCorrectValue) {
TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(0);
auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
1111);
delegate->Init();
RTPVideoHeader video_header;
video_header.width = 1280u;
video_header.height = 720u;
RTPVideoHeader::GenericDescriptorInfo& generic =
video_header.generic.emplace();
generic.frame_id = 10;
generic.temporal_index = 3;
generic.spatial_index = 2;
generic.decode_target_indications = {DecodeTargetIndication::kSwitch};
generic.dependencies = {5};
std::vector<uint32_t> csrcs = {234, 345, 456};
// Check that the transformable frame passed to the frame transformer has the
// correct metadata.
EXPECT_CALL(*mock_frame_transformer, Transform)
.WillOnce([&](std::unique_ptr<TransformableFrameInterface>
transformable_frame) {
auto frame =
absl::WrapUnique(static_cast<TransformableVideoFrameInterface*>(
transformable_frame.release()));
ASSERT_TRUE(frame);
auto metadata = frame->Metadata();
EXPECT_EQ(metadata.GetWidth(), 1280u);
EXPECT_EQ(metadata.GetHeight(), 720u);
EXPECT_EQ(metadata.GetFrameId(), 10);
EXPECT_EQ(metadata.GetTemporalIndex(), 3);
EXPECT_EQ(metadata.GetSpatialIndex(), 2);
EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5));
EXPECT_THAT(metadata.GetDecodeTargetIndications(),
ElementsAre(DecodeTargetIndication::kSwitch));
EXPECT_EQ(metadata.GetCsrcs(), csrcs);
});
// The delegate creates a transformable frame from the RtpFrameObject.
delegate->TransformFrame(CreateRtpFrameObject(video_header, csrcs));
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TransformableFrameMetadataHasCorrectValueAfterSetMetadata) {
rtc::AutoThread main_thread;
TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(1000);
auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
1111);
rtc::scoped_refptr<TransformedFrameCallback> callback;
EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
.WillOnce(SaveArg<0>(&callback));
delegate->Init();
ASSERT_TRUE(callback);
RTPVideoHeader video_header;
RTPVideoHeader::GenericDescriptorInfo& generic =
video_header.generic.emplace();
generic.frame_id = 10;
generic.dependencies = {5};
std::vector<uint32_t> csrcs = {234, 345, 456};
// Checks that the recieved RTPFrameObject has the new metadata.
EXPECT_CALL(receiver, ManageFrame)
.WillOnce([&](std::unique_ptr<RtpFrameObject> frame) {
const absl::optional<RTPVideoHeader::GenericDescriptorInfo>&
descriptor = frame->GetRtpVideoHeader().generic;
if (!descriptor.has_value()) {
ADD_FAILURE() << "GenericDescriptorInfo in RTPVideoHeader doesn't "
"have a value.";
} else {
EXPECT_EQ(descriptor->frame_id, 20);
EXPECT_THAT(descriptor->dependencies, ElementsAre(15));
}
EXPECT_EQ(frame->Csrcs(), csrcs);
});
// Sets new metadata to the transformable frame.
ON_CALL(*mock_frame_transformer, Transform)
.WillByDefault([&](std::unique_ptr<TransformableFrameInterface>
transformable_frame) {
ASSERT_THAT(transformable_frame, NotNull());
auto& video_frame = static_cast<TransformableVideoFrameInterface&>(
*transformable_frame);
VideoFrameMetadata metadata = video_frame.Metadata();
EXPECT_EQ(metadata.GetFrameId(), 10);
EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5));
EXPECT_EQ(metadata.GetCsrcs(), csrcs);
metadata.SetFrameId(20);
metadata.SetFrameDependencies(std::vector<int64_t>{15});
video_frame.SetMetadata(metadata);
callback->OnTransformedFrame(std::move(transformable_frame));
});
// The delegate creates a transformable frame from the RtpFrameObject.
delegate->TransformFrame(CreateRtpFrameObject(video_header, csrcs));
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
SenderFramesAreConvertedToReceiverFrames) {
rtc::AutoThread main_thread_;
TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(/*initial_timestamp_us=*/12345000);
auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111);
auto mock_sender_frame =
std::make_unique<NiceMock<MockTransformableVideoFrame>>();
ON_CALL(*mock_sender_frame, GetDirection)
.WillByDefault(Return(TransformableFrameInterface::Direction::kSender));
VideoFrameMetadata metadata;
metadata.SetCodec(kVideoCodecVP8);
metadata.SetRTPVideoHeaderCodecSpecifics(RTPVideoHeaderVP8());
ON_CALL(*mock_sender_frame, Metadata).WillByDefault(Return(metadata));
rtc::scoped_refptr<EncodedImageBufferInterface> buffer =
EncodedImageBuffer::Create(1);
ON_CALL(*mock_sender_frame, GetData)
.WillByDefault(Return(rtc::ArrayView<const uint8_t>(*buffer)));
rtc::scoped_refptr<TransformedFrameCallback> callback;
EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
.WillOnce(SaveArg<0>(&callback));
delegate->Init();
ASSERT_TRUE(callback);
EXPECT_CALL(receiver, ManageFrame)
.WillOnce([&](std::unique_ptr<RtpFrameObject> frame) {
EXPECT_EQ(frame->codec_type(), metadata.GetCodec());
EXPECT_EQ(frame->ReceivedTime(), 12345);
});
callback->OnTransformedFrame(std::move(mock_sender_frame));
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
ManageFrameFromDifferentReceiver) {
rtc::AutoThread main_thread_;
std::vector<uint32_t> csrcs = {234, 345, 456};
const int frame_id = 11;
TestRtpVideoFrameReceiver receiver1;
auto mock_frame_transformer1(
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
auto delegate1 =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver1, &clock, mock_frame_transformer1, rtc::Thread::Current(),
/*remote_ssrc*/ 1111);
TestRtpVideoFrameReceiver receiver2;
auto mock_frame_transformer2(
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
auto delegate2 =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver2, &clock, mock_frame_transformer2, rtc::Thread::Current(),
/*remote_ssrc*/ 1111);
delegate1->Init();
rtc::scoped_refptr<TransformedFrameCallback> callback_for_2;
EXPECT_CALL(*mock_frame_transformer2, RegisterTransformedFrameSinkCallback)
.WillOnce(SaveArg<0>(&callback_for_2));
delegate2->Init();
ASSERT_TRUE(callback_for_2);
// Expect a call on receiver2's ManageFrame with sequence numbers overwritten
// with the frame's ID.
EXPECT_CALL(receiver2, ManageFrame)
.WillOnce([&](std::unique_ptr<RtpFrameObject> frame) {
EXPECT_EQ(frame->Csrcs(), csrcs);
EXPECT_EQ(frame->first_seq_num(), frame_id);
EXPECT_EQ(frame->last_seq_num(), frame_id);
});
// When the frame transformer for receiver 1 receives the frame to transform,
// pipe it over to the callback for receiver 2.
ON_CALL(*mock_frame_transformer1, Transform)
.WillByDefault([&callback_for_2](
std::unique_ptr<TransformableFrameInterface> frame) {
callback_for_2->OnTransformedFrame(std::move(frame));
});
std::unique_ptr<RtpFrameObject> untransformed_frame =
CreateRtpFrameObject(RTPVideoHeader(), csrcs);
untransformed_frame->SetId(frame_id);
delegate1->TransformFrame(std::move(untransformed_frame));
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
ShortCircuitingSkipsTransform) {
rtc::AutoThread main_thread_;
TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(0);
auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
1111);
delegate->Init();
delegate->StartShortCircuiting();
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
// Will not call the actual transformer.
EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
// Will pass the frame straight to the reciever.
EXPECT_CALL(receiver, ManageFrame);
delegate->TransformFrame(CreateRtpFrameObject());
}
} // namespace
} // namespace webrtc