Remove VideoSender and fold code into VideoStreamEncoder
This CL moves the functionality in VideoSender into VideoStreamEncoder
and simplifies the code where possible, given what we know of the
encoder state and that we now run on the encoder queue.
The intent here is to make it easier to remove the next parts, the
encoder database and generic encoder wrapper.
Bug: webrtc:10164
Change-Id: I8c108ccbe5db97cd9fd1e84228134709af845ea3
Reviewed-on: https://webrtc-review.googlesource.com/c/123540
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26813}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index f4490cb..a321f32 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -153,7 +153,6 @@
"video_coding_impl.cc",
"video_coding_impl.h",
"video_receiver.cc",
- "video_sender.cc",
]
if (!build_with_chromium && is_clang) {
@@ -873,7 +872,6 @@
"video_codec_initializer_unittest.cc",
"video_packet_buffer_unittest.cc",
"video_receiver_unittest.cc",
- "video_sender_unittest.cc",
]
if (rtc_use_h264) {
sources += [
diff --git a/modules/video_coding/video_sender.cc b/modules/video_coding/video_sender.cc
deleted file mode 100644
index 4493dd7..0000000
--- a/modules/video_coding/video_sender.cc
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <stddef.h>
-#include <stdint.h>
-#include <vector>
-
-#include "api/scoped_refptr.h"
-#include "api/video/video_bitrate_allocation.h"
-#include "api/video/video_bitrate_allocator.h"
-#include "api/video/video_frame.h"
-#include "api/video/video_frame_buffer.h"
-#include "api/video_codecs/video_codec.h"
-#include "api/video_codecs/video_encoder.h"
-#include "common_types.h" // NOLINT(build/include)
-#include "modules/video_coding/encoder_database.h"
-#include "modules/video_coding/generic_encoder.h"
-#include "modules/video_coding/include/video_codec_interface.h"
-#include "modules/video_coding/include/video_coding_defines.h"
-#include "modules/video_coding/include/video_error_codes.h"
-#include "modules/video_coding/internal_defines.h"
-#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
-#include "modules/video_coding/video_coding_impl.h"
-#include "rtc_base/checks.h"
-#include "rtc_base/critical_section.h"
-#include "rtc_base/logging.h"
-#include "rtc_base/sequenced_task_checker.h"
-#include "system_wrappers/include/clock.h"
-#include "system_wrappers/include/field_trial.h"
-
-namespace webrtc {
-namespace vcm {
-
-VideoSender::VideoSender(Clock* clock,
- EncodedImageCallback* post_encode_callback)
- : _encoder(nullptr),
- _encodedFrameCallback(post_encode_callback),
- _codecDataBase(&_encodedFrameCallback),
- current_codec_(),
- encoder_has_internal_source_(false),
- next_frame_types_(1, kVideoFrameDelta) {
- // Allow VideoSender to be created on one thread but used on another, post
- // construction. This is currently how this class is being used by at least
- // one external project (diffractor).
- sequenced_checker_.Detach();
-}
-
-VideoSender::~VideoSender() {}
-
-// Register the send codec to be used.
-int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
- uint32_t numberOfCores,
- uint32_t maxPayloadSize) {
- RTC_DCHECK(sequenced_checker_.CalledSequentially());
- rtc::CritScope lock(&encoder_crit_);
- if (sendCodec == nullptr) {
- return VCM_PARAMETER_ERROR;
- }
-
- bool ret =
- _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize);
-
- // Update encoder regardless of result to make sure that we're not holding on
- // to a deleted instance.
- _encoder = _codecDataBase.GetEncoder();
- // Cache the current codec here so they can be fetched from this thread
- // without requiring the _sendCritSect lock.
- current_codec_ = *sendCodec;
-
- if (!ret) {
- RTC_LOG(LS_ERROR) << "Failed to initialize set encoder with codec type '"
- << sendCodec->codecType << "'.";
- return VCM_CODEC_ERROR;
- }
-
- // SetSendCodec succeeded, _encoder should be set.
- RTC_DCHECK(_encoder);
-
- {
- rtc::CritScope cs(¶ms_crit_);
- next_frame_types_.clear();
- next_frame_types_.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
- kVideoFrameKey);
- // Cache InternalSource() to have this available from IntraFrameRequest()
- // without having to acquire encoder_crit_ (avoid blocking on encoder use).
- encoder_has_internal_source_ = _encoder->InternalSource();
- }
-
- RTC_LOG(LS_VERBOSE) << " max bitrate " << sendCodec->maxBitrate
- << " start bitrate " << sendCodec->startBitrate
- << " max frame rate " << sendCodec->maxFramerate
- << " max payload size " << maxPayloadSize;
- return VCM_OK;
-}
-
-// Register an external decoder object.
-// This can not be used together with external decoder callbacks.
-void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
- bool internalSource /*= false*/) {
- RTC_DCHECK(sequenced_checker_.CalledSequentially());
-
- rtc::CritScope lock(&encoder_crit_);
-
- if (externalEncoder == nullptr) {
- _codecDataBase.DeregisterExternalEncoder();
- {
- // Make sure the VCM doesn't use the de-registered codec
- rtc::CritScope params_lock(¶ms_crit_);
- _encoder = nullptr;
- encoder_has_internal_source_ = false;
- }
- return;
- }
- _codecDataBase.RegisterExternalEncoder(externalEncoder,
- internalSource);
-}
-
-int32_t VideoSender::SetChannelParameters(
- const VideoBitrateAllocation& bitrate_allocation,
- uint32_t framerate_fps) {
- bool encoder_has_internal_source;
- {
- rtc::CritScope cs(¶ms_crit_);
- encoder_has_internal_source = encoder_has_internal_source_;
- }
-
- {
- rtc::CritScope cs(&encoder_crit_);
- if (_encoder) {
- // |target_bitrate == 0 | means that the network is down or the send pacer
- // is full. We currently only report this if the encoder has an internal
- // source. If the encoder does not have an internal source, higher levels
- // are expected to not call AddVideoFrame. We do this since its unclear
- // how current encoder implementations behave when given a zero target
- // bitrate.
- // TODO(perkj): Make sure all known encoder implementations handle zero
- // target bitrate and remove this check.
- if (!encoder_has_internal_source &&
- bitrate_allocation.get_sum_bps() == 0) {
- return VCM_OK;
- }
-
- if (framerate_fps == 0) {
- // No frame rate estimate available, use default.
- framerate_fps = current_codec_.maxFramerate;
- }
- if (_encoder != nullptr)
- _encoder->SetEncoderParameters(bitrate_allocation, framerate_fps);
- }
- }
-
- return VCM_OK;
-}
-
-// Add one raw video frame to the encoder, blocking.
-int32_t VideoSender::AddVideoFrame(
- const VideoFrame& videoFrame,
- const CodecSpecificInfo* codecSpecificInfo,
- absl::optional<VideoEncoder::EncoderInfo> encoder_info) {
- std::vector<FrameType> next_frame_types;
- bool encoder_has_internal_source = false;
- {
- rtc::CritScope lock(¶ms_crit_);
- next_frame_types = next_frame_types_;
- encoder_has_internal_source = encoder_has_internal_source_;
- }
- rtc::CritScope lock(&encoder_crit_);
- if (_encoder == nullptr)
- return VCM_UNINITIALIZED;
- // TODO(pbos): Make sure setting send codec is synchronized with video
- // processing so frame size always matches.
- if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
- videoFrame.height())) {
- RTC_LOG(LS_ERROR)
- << "Incoming frame doesn't match set resolution. Dropping.";
- return VCM_PARAMETER_ERROR;
- }
- VideoFrame converted_frame = videoFrame;
- const VideoFrameBuffer::Type buffer_type =
- converted_frame.video_frame_buffer()->type();
- const bool is_buffer_type_supported =
- buffer_type == VideoFrameBuffer::Type::kI420 ||
- (buffer_type == VideoFrameBuffer::Type::kNative &&
- encoder_info->supports_native_handle);
- if (!is_buffer_type_supported) {
- // This module only supports software encoding.
- // TODO(pbos): Offload conversion from the encoder thread.
- rtc::scoped_refptr<I420BufferInterface> converted_buffer(
- converted_frame.video_frame_buffer()->ToI420());
-
- if (!converted_buffer) {
- RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
- return VCM_PARAMETER_ERROR;
- }
-
- // UpdatedRect is not propagated because buffer was converted,
- // therefore we can't guarantee that pixels outside of UpdateRect didn't
- // change comparing to the previous frame.
- converted_frame = VideoFrame::Builder()
- .set_video_frame_buffer(converted_buffer)
- .set_timestamp_rtp(converted_frame.timestamp())
- .set_timestamp_ms(converted_frame.render_time_ms())
- .set_rotation(converted_frame.rotation())
- .set_id(converted_frame.id())
- .build();
- }
- int32_t ret =
- _encoder->Encode(converted_frame, codecSpecificInfo, next_frame_types);
- if (ret < 0) {
- RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
- return ret;
- }
-
- {
- rtc::CritScope lock(¶ms_crit_);
- // Change all keyframe requests to encode delta frames the next time.
- for (size_t i = 0; i < next_frame_types_.size(); ++i) {
- // Check for equality (same requested as before encoding) to not
- // accidentally drop a keyframe request while encoding.
- if (next_frame_types[i] == next_frame_types_[i])
- next_frame_types_[i] = kVideoFrameDelta;
- }
- }
- return VCM_OK;
-}
-
-int32_t VideoSender::IntraFrameRequest(size_t stream_index) {
- {
- rtc::CritScope lock(¶ms_crit_);
- if (stream_index >= next_frame_types_.size()) {
- return -1;
- }
- next_frame_types_[stream_index] = kVideoFrameKey;
- if (!encoder_has_internal_source_)
- return VCM_OK;
- }
- // TODO(pbos): Remove when InternalSource() is gone. Both locks have to be
- // held here for internal consistency, since _encoder could be removed while
- // not holding encoder_crit_. Checks have to be performed again since
- // params_crit_ was dropped to not cause lock-order inversions with
- // encoder_crit_.
- rtc::CritScope lock(&encoder_crit_);
- rtc::CritScope params_lock(¶ms_crit_);
- if (stream_index >= next_frame_types_.size())
- return -1;
- if (_encoder != nullptr && _encoder->InternalSource()) {
- // Try to request the frame if we have an external encoder with
- // internal source since AddVideoFrame never will be called.
- if (_encoder->RequestFrame(next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) {
- // Try to remove just-performed keyframe request, if stream still exists.
- next_frame_types_[stream_index] = kVideoFrameDelta;
- }
- }
- return VCM_OK;
-}
-
-} // namespace vcm
-} // namespace webrtc
diff --git a/modules/video_coding/video_sender_unittest.cc b/modules/video_coding/video_sender_unittest.cc
deleted file mode 100644
index 11a69be..0000000
--- a/modules/video_coding/video_sender_unittest.cc
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <memory>
-#include <vector>
-
-#include "absl/memory/memory.h"
-#include "api/test/mock_video_encoder.h"
-#include "api/video/i420_buffer.h"
-#include "api/video_codecs/vp8_temporal_layers.h"
-#include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/include/mock/mock_vcm_callbacks.h"
-#include "modules/video_coding/include/video_coding.h"
-#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
-#include "modules/video_coding/utility/simulcast_rate_allocator.h"
-#include "modules/video_coding/video_coding_impl.h"
-#include "system_wrappers/include/clock.h"
-#include "test/frame_generator.h"
-#include "test/gtest.h"
-#include "test/testsupport/file_utils.h"
-#include "test/video_codec_settings.h"
-
-using ::testing::_;
-using ::testing::AllOf;
-using ::testing::ElementsAre;
-using ::testing::ElementsAreArray;
-using ::testing::Field;
-using ::testing::NiceMock;
-using ::testing::Pointee;
-using ::testing::Return;
-using ::testing::FloatEq;
-using std::vector;
-using webrtc::test::FrameGenerator;
-
-namespace webrtc {
-namespace vcm {
-namespace {
-static const int kDefaultHeight = 720;
-static const int kDefaultWidth = 1280;
-static const int kMaxNumberOfTemporalLayers = 3;
-static const int kNumberOfLayers = 3;
-static const int kNumberOfStreams = 3;
-static const int kUnusedPayloadType = 10;
-
-struct Vp8StreamInfo {
- float framerate_fps[kMaxNumberOfTemporalLayers];
- int bitrate_kbps[kMaxNumberOfTemporalLayers];
-};
-
-MATCHER_P(MatchesVp8StreamInfo, expected, "") {
- bool res = true;
- for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
- if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
- *result_listener << " framerate_fps[" << tl
- << "] = " << arg.framerate_fps[tl] << " (expected "
- << expected.framerate_fps[tl] << ") ";
- res = false;
- }
- if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
- *result_listener << " bitrate_kbps[" << tl
- << "] = " << arg.bitrate_kbps[tl] << " (expected "
- << expected.bitrate_kbps[tl] << ") ";
- res = false;
- }
- }
- return res;
-}
-
-class EmptyFrameGenerator : public FrameGenerator {
- public:
- EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {}
- VideoFrame* NextFrame() override {
- frame_ = absl::make_unique<VideoFrame>(
- VideoFrame::Builder()
- .set_video_frame_buffer(I420Buffer::Create(width_, height_))
- .set_rotation(webrtc::kVideoRotation_0)
- .set_timestamp_us(0)
- .build());
- return frame_.get();
- }
-
- private:
- const int width_;
- const int height_;
- std::unique_ptr<VideoFrame> frame_;
-};
-
-class EncodedImageCallbackImpl : public EncodedImageCallback {
- public:
- explicit EncodedImageCallbackImpl(Clock* clock)
- : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {}
-
- virtual ~EncodedImageCallbackImpl() {}
-
- Result OnEncodedImage(const EncodedImage& encoded_image,
- const CodecSpecificInfo* codec_specific_info,
- const RTPFragmentationHeader* fragmentation) override {
- assert(codec_specific_info);
- frame_data_.push_back(
- FrameData(encoded_image.size(), *codec_specific_info));
- return Result(Result::OK, encoded_image.Timestamp());
- }
-
- void Reset() {
- frame_data_.clear();
- start_time_ms_ = clock_->TimeInMilliseconds();
- }
-
- float FramerateFpsWithinTemporalLayer(int temporal_layer) {
- return CountFramesWithinTemporalLayer(temporal_layer) *
- (1000.0 / interval_ms());
- }
-
- float BitrateKbpsWithinTemporalLayer(int temporal_layer) {
- return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 /
- interval_ms();
- }
-
- Vp8StreamInfo CalculateVp8StreamInfo() {
- Vp8StreamInfo info;
- for (int tl = 0; tl < 3; ++tl) {
- info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
- info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
- }
- return info;
- }
-
- private:
- struct FrameData {
- FrameData() : payload_size(0) {}
-
- FrameData(size_t payload_size, const CodecSpecificInfo& codec_specific_info)
- : payload_size(payload_size),
- codec_specific_info(codec_specific_info) {}
-
- size_t payload_size;
- CodecSpecificInfo codec_specific_info;
- };
-
- int64_t interval_ms() {
- int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_);
- EXPECT_GT(diff, 0);
- return diff;
- }
-
- int CountFramesWithinTemporalLayer(int temporal_layer) {
- int frames = 0;
- for (size_t i = 0; i < frame_data_.size(); ++i) {
- EXPECT_EQ(kVideoCodecVP8, frame_data_[i].codec_specific_info.codecType);
- const uint8_t temporal_idx =
- frame_data_[i].codec_specific_info.codecSpecific.VP8.temporalIdx;
- if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
- frames++;
- }
- return frames;
- }
-
- size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) {
- size_t payload_size = 0;
- for (size_t i = 0; i < frame_data_.size(); ++i) {
- EXPECT_EQ(kVideoCodecVP8, frame_data_[i].codec_specific_info.codecType);
- const uint8_t temporal_idx =
- frame_data_[i].codec_specific_info.codecSpecific.VP8.temporalIdx;
- if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
- payload_size += frame_data_[i].payload_size;
- }
- return payload_size;
- }
-
- Clock* clock_;
- int64_t start_time_ms_;
- vector<FrameData> frame_data_;
-};
-
-class TestVideoSender : public ::testing::Test {
- protected:
- // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
- // a special case (e.g. frame rate in media optimization).
- TestVideoSender() : clock_(1000), encoded_frame_callback_(&clock_) {}
-
- void SetUp() override {
- sender_.reset(new VideoSender(&clock_, &encoded_frame_callback_));
- }
-
- void AddFrame() {
- assert(generator_.get());
- sender_->AddVideoFrame(*generator_->NextFrame(), nullptr,
- encoder_ ? absl::optional<VideoEncoder::EncoderInfo>(
- encoder_->GetEncoderInfo())
- : absl::nullopt);
- }
-
- SimulatedClock clock_;
- EncodedImageCallbackImpl encoded_frame_callback_;
- // Used by subclassing tests, need to outlive sender_.
- std::unique_ptr<VideoEncoder> encoder_;
- std::unique_ptr<VideoSender> sender_;
- std::unique_ptr<FrameGenerator> generator_;
-};
-
-class TestVideoSenderWithMockEncoder : public TestVideoSender {
- public:
- TestVideoSenderWithMockEncoder() {}
- ~TestVideoSenderWithMockEncoder() override {}
-
- protected:
- void SetUp() override {
- TestVideoSender::SetUp();
- sender_->RegisterExternalEncoder(&encoder_, false);
- webrtc::test::CodecSettings(kVideoCodecVP8, &settings_);
- settings_.numberOfSimulcastStreams = kNumberOfStreams;
- ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100,
- &settings_.simulcastStream[0]);
- ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500,
- &settings_.simulcastStream[1]);
- ConfigureStream(kDefaultWidth, kDefaultHeight, 1200,
- &settings_.simulcastStream[2]);
- settings_.plType = kUnusedPayloadType; // Use the mocked encoder.
- generator_.reset(
- new EmptyFrameGenerator(settings_.width, settings_.height));
- EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
- rate_allocator_.reset(new DefaultVideoBitrateAllocator(settings_));
- }
-
- void TearDown() override { sender_.reset(); }
-
- void ExpectIntraRequest(int stream) {
- ExpectEncodeWithFrameTypes(stream, false);
- }
-
- void ExpectInitialKeyFrames() { ExpectEncodeWithFrameTypes(-1, true); }
-
- void ExpectEncodeWithFrameTypes(int intra_request_stream, bool first_frame) {
- if (intra_request_stream == -1) {
- // No intra request expected, keyframes on first frame.
- FrameType frame_type = first_frame ? kVideoFrameKey : kVideoFrameDelta;
- EXPECT_CALL(
- encoder_,
- Encode(_, _,
- Pointee(ElementsAre(frame_type, frame_type, frame_type))))
- .Times(1)
- .WillRepeatedly(Return(0));
- return;
- }
- ASSERT_FALSE(first_frame);
- ASSERT_GE(intra_request_stream, 0);
- ASSERT_LT(intra_request_stream, kNumberOfStreams);
- std::vector<FrameType> frame_types(kNumberOfStreams, kVideoFrameDelta);
- frame_types[intra_request_stream] = kVideoFrameKey;
- EXPECT_CALL(
- encoder_,
- Encode(_, _,
- Pointee(ElementsAreArray(&frame_types[0], frame_types.size()))))
- .Times(1)
- .WillRepeatedly(Return(0));
- }
-
- static void ConfigureStream(int width,
- int height,
- int max_bitrate,
- SimulcastStream* stream) {
- assert(stream);
- stream->width = width;
- stream->height = height;
- stream->maxBitrate = max_bitrate;
- stream->numberOfTemporalLayers = kNumberOfLayers;
- stream->qpMax = 45;
- }
-
- VideoCodec settings_;
- NiceMock<MockVideoEncoder> encoder_;
- std::unique_ptr<DefaultVideoBitrateAllocator> rate_allocator_;
-};
-
-TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) {
- // Initial request should be all keyframes.
- ExpectInitialKeyFrames();
- AddFrame();
- EXPECT_EQ(0, sender_->IntraFrameRequest(0));
- ExpectIntraRequest(0);
- AddFrame();
- ExpectIntraRequest(-1);
- AddFrame();
-
- EXPECT_EQ(0, sender_->IntraFrameRequest(1));
- ExpectIntraRequest(1);
- AddFrame();
- ExpectIntraRequest(-1);
- AddFrame();
-
- EXPECT_EQ(0, sender_->IntraFrameRequest(2));
- ExpectIntraRequest(2);
- AddFrame();
- ExpectIntraRequest(-1);
- AddFrame();
-
- EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
- ExpectIntraRequest(-1);
- AddFrame();
-}
-
-TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) {
- // Let actual fps be half of max, so it can be distinguished from default.
- const uint32_t kActualFrameRate = settings_.maxFramerate / 2;
- const int64_t kFrameIntervalMs = 1000 / kActualFrameRate;
- const uint32_t new_bitrate_kbps = settings_.startBitrate + 300;
-
- // Initial frame rate is taken from config, as we have no data yet.
- VideoBitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation(
- new_bitrate_kbps * 1000, settings_.maxFramerate);
- EXPECT_CALL(encoder_,
- SetRateAllocation(new_rate_allocation, settings_.maxFramerate))
- .Times(1)
- .WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
- AddFrame();
- clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
-
- // Expect no call to encoder_.SetRates if the new bitrate is zero.
- EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0);
- sender_->SetChannelParameters(VideoBitrateAllocation(),
- settings_.maxFramerate);
- AddFrame();
-}
-
-TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) {
- // De-register current external encoder.
- sender_->RegisterExternalEncoder(nullptr, false);
- // Register encoder with internal capture.
- sender_->RegisterExternalEncoder(&encoder_, true);
- EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
- // Initial request should be all keyframes.
- ExpectInitialKeyFrames();
- AddFrame();
- ExpectIntraRequest(0);
- EXPECT_EQ(0, sender_->IntraFrameRequest(0));
- ExpectIntraRequest(1);
- EXPECT_EQ(0, sender_->IntraFrameRequest(1));
- ExpectIntraRequest(2);
- EXPECT_EQ(0, sender_->IntraFrameRequest(2));
- // No requests expected since these indices are out of bounds.
- EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
-}
-
-TEST_F(TestVideoSenderWithMockEncoder, TestEncoderParametersForInternalSource) {
- // De-register current external encoder.
- sender_->RegisterExternalEncoder(nullptr, false);
- // Register encoder with internal capture.
- sender_->RegisterExternalEncoder(&encoder_, true);
- EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
- // Update encoder bitrate parameters. We expect that to immediately call
- // SetRates on the encoder without waiting for AddFrame processing.
- const uint32_t new_bitrate_kbps = settings_.startBitrate + 300;
- VideoBitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation(
- new_bitrate_kbps * 1000, settings_.maxFramerate);
- EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _))
- .Times(1)
- .WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
-}
-
-TEST_F(TestVideoSenderWithMockEncoder,
- NoRedundantSetChannelParameterOrSetRatesCalls) {
- const int64_t kRateStatsWindowMs = 2000;
- const uint32_t kInputFps = 20;
- int64_t start_time = clock_.TimeInMilliseconds();
- // Expect initial call to SetChannelParameters. Rates are initialized through
- // InitEncode and expects no additional call before the framerate (or bitrate)
- // updates.
- sender_->SetChannelParameters(
- rate_allocator_->GetAllocation(settings_.startBitrate * 1000, kInputFps),
- kInputFps);
- while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
- AddFrame();
- clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
- }
-
- // Call to SetChannelParameters with changed bitrate should call encoder
- // SetRates but not encoder SetChannelParameters (that are unchanged).
- uint32_t new_bitrate_bps = 2 * settings_.startBitrate * 1000;
- VideoBitrateAllocation new_rate_allocation =
- rate_allocator_->GetAllocation(new_bitrate_bps, kInputFps);
- EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps))
- .Times(1)
- .WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, kInputFps);
- AddFrame();
-}
-
-class TestVideoSenderWithVp8 : public TestVideoSender {
- public:
- TestVideoSenderWithVp8()
- : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {}
-
- void SetUp() override {
- TestVideoSender::SetUp();
-
- const char* input_video = "foreman_cif";
- const int width = 352;
- const int height = 288;
- generator_ = FrameGenerator::CreateFromYuvFile(
- std::vector<std::string>(1, test::ResourcePath(input_video, "yuv")),
- width, height, 1);
-
- codec_ = MakeVp8VideoCodec(width, height, 3);
- codec_.minBitrate = 10;
- codec_.startBitrate = codec_bitrate_kbps_;
- codec_.maxBitrate = codec_bitrate_kbps_;
-
- rate_allocator_.reset(new SimulcastRateAllocator(codec_));
-
- encoder_ = VP8Encoder::Create();
- sender_->RegisterExternalEncoder(encoder_.get(), false);
- EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200));
- }
-
- static VideoCodec MakeVp8VideoCodec(int width,
- int height,
- int temporal_layers) {
- VideoCodec codec;
- webrtc::test::CodecSettings(kVideoCodecVP8, &codec);
- codec.width = width;
- codec.height = height;
- codec.VP8()->numberOfTemporalLayers = temporal_layers;
- return codec;
- }
-
- void InsertFrames(float framerate, float seconds) {
- for (int i = 0; i < seconds * framerate; ++i) {
- clock_.AdvanceTimeMilliseconds(1000.0f / framerate);
- AddFrame();
- // SetChannelParameters needs to be called frequently to propagate
- // framerate from the media optimization into the encoder.
- const VideoBitrateAllocation bitrate_allocation =
- rate_allocator_->GetAllocation(available_bitrate_kbps_ * 1000,
- static_cast<uint32_t>(framerate));
- if (i != 0) {
- EXPECT_EQ(VCM_OK,
- sender_->SetChannelParameters(
- bitrate_allocation, static_cast<uint32_t>(framerate)));
- }
- }
- }
-
- Vp8StreamInfo SimulateWithFramerate(float framerate) {
- const float short_simulation_interval = 5.0;
- const float long_simulation_interval = 10.0;
- // It appears that this 5 seconds simulation is needed to allow
- // bitrate and framerate to stabilize.
- InsertFrames(framerate, short_simulation_interval);
- encoded_frame_callback_.Reset();
-
- InsertFrames(framerate, long_simulation_interval);
- return encoded_frame_callback_.CalculateVp8StreamInfo();
- }
-
- protected:
- VideoCodec codec_;
- int codec_bitrate_kbps_;
- int available_bitrate_kbps_;
- std::unique_ptr<SimulcastRateAllocator> rate_allocator_;
-};
-
-#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
-#define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy
-#else
-#define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
-#endif
-TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
- const int low_b =
- codec_bitrate_kbps_ *
- webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 0);
- const int mid_b =
- codec_bitrate_kbps_ *
- webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 1);
- const int high_b =
- codec_bitrate_kbps_ *
- webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 2);
- {
- Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
- EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
- }
- {
- Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
- EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
- }
-}
-
-} // namespace
-} // namespace vcm
-} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 6066ac5..8dcb8d9 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -366,7 +366,6 @@
sink_(nullptr),
settings_(settings),
rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
- video_sender_(Clock::GetRealTimeClock(), this),
overuse_detector_(std::move(overuse_detector)),
encoder_stats_observer_(encoder_stats_observer),
max_framerate_(-1),
@@ -393,9 +392,14 @@
force_disable_frame_dropper_(false),
input_framerate_(kFrameRateAvergingWindowSizeMs, 1000),
pending_frame_drops_(0),
+ generic_encoder_(nullptr),
+ generic_encoder_callback_(this),
+ codec_database_(&generic_encoder_callback_),
+ next_frame_types_(1, kVideoFrameDelta),
encoder_queue_("EncoderQueue") {
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK(overuse_detector_);
+ RTC_DCHECK_GE(number_of_cores, 1);
}
VideoStreamEncoder::~VideoStreamEncoder() {
@@ -412,7 +416,8 @@
overuse_detector_->StopCheckForOveruse();
rate_allocator_.reset();
bitrate_observer_ = nullptr;
- video_sender_.RegisterExternalEncoder(nullptr, false);
+ codec_database_.DeregisterExternalEncoder();
+ generic_encoder_ = nullptr;
quality_scaler_ = nullptr;
shutdown_event_.Set();
});
@@ -502,10 +507,11 @@
RTC_DCHECK(sink_);
RTC_LOG(LS_INFO) << "ConfigureEncoder requested.";
- max_data_payload_length_ = max_data_payload_length;
pending_encoder_creation_ =
- (!encoder_ || encoder_config_.video_format != config.video_format);
+ (!encoder_ || encoder_config_.video_format != config.video_format ||
+ max_data_payload_length_ != max_data_payload_length);
encoder_config_ = std::move(config);
+ max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
// Reconfigure the encoder now if the encoder has an internal source or
@@ -514,11 +520,13 @@
// The codec configuration depends on incoming video frame size.
if (last_frame_info_) {
ReconfigureEncoder();
- } else if (settings_.encoder_factory
- ->QueryVideoEncoder(encoder_config_.video_format)
- .has_internal_source) {
- last_frame_info_ = VideoFrameInfo(176, 144, false);
- ReconfigureEncoder();
+ } else {
+ codec_info_ = settings_.encoder_factory->QueryVideoEncoder(
+ encoder_config_.video_format);
+ if (HasInternalSource()) {
+ last_frame_info_ = VideoFrameInfo(176, 144, false);
+ ReconfigureEncoder();
+ }
}
}
@@ -574,6 +582,8 @@
std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
codec.expect_encode_from_texture = last_frame_info_->is_texture;
+ // Make sure the start bit rate is sane...
+ RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
// Inform source about max configured framerate.
@@ -589,7 +599,8 @@
// encoder_->InitEncode().
if (pending_encoder_creation_) {
if (encoder_) {
- video_sender_.RegisterExternalEncoder(nullptr, false);
+ codec_database_.DeregisterExternalEncoder();
+ generic_encoder_ = nullptr;
}
encoder_ = settings_.encoder_factory->CreateVideoEncoder(
@@ -598,19 +609,29 @@
// or just discard incoming frames?
RTC_CHECK(encoder_);
- const webrtc::VideoEncoderFactory::CodecInfo info =
- settings_.encoder_factory->QueryVideoEncoder(
- encoder_config_.video_format);
+ codec_info_ = settings_.encoder_factory->QueryVideoEncoder(
+ encoder_config_.video_format);
- video_sender_.RegisterExternalEncoder(encoder_.get(),
- info.has_internal_source);
+ codec_database_.RegisterExternalEncoder(encoder_.get(),
+ HasInternalSource());
}
- // RegisterSendCodec implies an unconditional call to
- // encoder_->InitEncode().
- bool success = video_sender_.RegisterSendCodec(
- &codec, number_of_cores_,
- static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
- if (!success) {
+
+ // SetSendCodec implies an unconditional call to encoder_->InitEncode().
+ bool success = codec_database_.SetSendCodec(&codec, number_of_cores_,
+ max_data_payload_length_);
+ generic_encoder_ = codec_database_.GetEncoder();
+
+ if (success) {
+ RTC_DCHECK(generic_encoder_);
+ next_frame_types_.clear();
+ next_frame_types_.resize(
+ std::max(static_cast<int>(codec.numberOfSimulcastStreams), 1),
+ kVideoFrameKey);
+ RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate
+ << " start bitrate " << codec.startBitrate
+ << " max frame rate " << codec.maxFramerate
+ << " max payload size " << max_data_payload_length_;
+ } else {
RTC_LOG(LS_ERROR) << "Failed to configure encoder.";
rate_allocator_.reset();
}
@@ -642,7 +663,6 @@
frame_dropper_.Reset();
frame_dropper_.SetRates(codec.startBitrate, max_framerate_);
- uint32_t framerate_fps = GetInputFramerateFps();
// Force-disable frame dropper if either:
// * We have screensharing with layers.
// * "WebRTC-FrameDropper" field trial is "Disabled".
@@ -659,11 +679,10 @@
if (rate_allocator_ && last_observed_bitrate_bps_ > 0) {
// We have a new rate allocator instance and already configured target
// bitrate. Update the rate allocation and notify observsers.
- VideoBitrateAllocation bitrate_allocation =
- GetBitrateAllocationAndNotifyObserver(last_observed_bitrate_bps_,
- framerate_fps);
-
- video_sender_.SetChannelParameters(bitrate_allocation, framerate_fps);
+ const uint32_t framerate_fps = GetInputFramerateFps();
+ SetEncoderRates(GetBitrateAllocationAndNotifyObserver(
+ last_observed_bitrate_bps_, framerate_fps),
+ framerate_fps);
}
encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams);
@@ -871,8 +890,35 @@
uint32_t VideoStreamEncoder::GetInputFramerateFps() {
const uint32_t default_fps = max_framerate_ != -1 ? max_framerate_ : 30;
- return input_framerate_.Rate(clock_->TimeInMilliseconds())
- .value_or(default_fps);
+ absl::optional<uint32_t> input_fps =
+ input_framerate_.Rate(clock_->TimeInMilliseconds());
+ if (!input_fps || *input_fps == 0) {
+ return default_fps;
+ }
+ return *input_fps;
+}
+
+void VideoStreamEncoder::SetEncoderRates(
+ const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps) {
+ if (!generic_encoder_) {
+ return;
+ }
+
+ // |bitrate_allocation| is 0 it means that the network is down or the send
+ // pacer is full. We currently only report this if the encoder has an internal
+ // source. If the encoder does not have an internal source, higher levels
+ // are expected to not call AddVideoFrame. We do this since its unclear
+ // how current encoder implementations behave when given a zero target
+ // bitrate.
+ // TODO(perkj): Make sure all known encoder implementations handle zero
+ // target bitrate and remove this check.
+ if (!HasInternalSource() && bitrate_allocation.get_sum_bps() == 0) {
+ return;
+ }
+
+ RTC_DCHECK_GT(framerate_fps, 0);
+ generic_encoder_->SetEncoderParameters(bitrate_allocation, framerate_fps);
}
void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
@@ -901,9 +947,11 @@
// from GetScalingSettings should enable or disable the frame drop.
// Update input frame rate before we start using it. If we update it after
- // any potential frame drop we are going to artifically increase frame sizes.
- input_framerate_.Update(1u, clock_->TimeInMilliseconds());
+ // any potential frame drop we are going to artificially increase frame sizes.
+ // Poll the rate before updating, otherwise we risk the rate being estimated
+ // a little too high at the start of the call when then window is small.
uint32_t framerate_fps = GetInputFramerateFps();
+ input_framerate_.Update(1u, clock_->TimeInMilliseconds());
int64_t now_ms = clock_->TimeInMilliseconds();
if (pending_encoder_reconfiguration_) {
@@ -912,10 +960,9 @@
} else if (!last_parameters_update_ms_ ||
now_ms - *last_parameters_update_ms_ >=
vcm::VCMProcessTimer::kDefaultProcessIntervalMs) {
- video_sender_.SetChannelParameters(
- GetBitrateAllocationAndNotifyObserver(last_observed_bitrate_bps_,
- framerate_fps),
- framerate_fps);
+ SetEncoderRates(GetBitrateAllocationAndNotifyObserver(
+ last_observed_bitrate_bps_, framerate_fps),
+ framerate_fps);
last_parameters_update_ms_.emplace(now_ms);
}
@@ -1069,7 +1116,48 @@
}
encoder_info_ = info;
- video_sender_.AddVideoFrame(out_frame, nullptr, encoder_info_);
+ RTC_DCHECK(generic_encoder_);
+ RTC_DCHECK(codec_database_.MatchesCurrentResolution(out_frame.width(),
+ out_frame.height()));
+ const VideoFrameBuffer::Type buffer_type =
+ out_frame.video_frame_buffer()->type();
+ const bool is_buffer_type_supported =
+ buffer_type == VideoFrameBuffer::Type::kI420 ||
+ (buffer_type == VideoFrameBuffer::Type::kNative &&
+ encoder_info_.supports_native_handle);
+
+ if (!is_buffer_type_supported) {
+ // This module only supports software encoding.
+ rtc::scoped_refptr<I420BufferInterface> converted_buffer(
+ out_frame.video_frame_buffer()->ToI420());
+
+ if (!converted_buffer) {
+ RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
+ return;
+ }
+
+ // UpdatedRect is not propagated because buffer was converted,
+ // therefore we can't guarantee that pixels outside of UpdateRect didn't
+ // change comparing to the previous frame.
+ out_frame = VideoFrame::Builder()
+ .set_video_frame_buffer(converted_buffer)
+ .set_timestamp_rtp(out_frame.timestamp())
+ .set_timestamp_ms(out_frame.render_time_ms())
+ .set_rotation(out_frame.rotation())
+ .set_id(out_frame.id())
+ .build();
+ }
+ const int32_t encode_status =
+ generic_encoder_->Encode(out_frame, nullptr, next_frame_types_);
+ if (encode_status < 0) {
+ RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: "
+ << encode_status;
+ return;
+ }
+
+ for (auto& it : next_frame_types_) {
+ it = kVideoFrameDelta;
+ }
}
void VideoStreamEncoder::SendKeyFrame() {
@@ -1079,7 +1167,18 @@
}
RTC_DCHECK_RUN_ON(&encoder_queue_);
TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
- video_sender_.IntraFrameRequest(0);
+ RTC_DCHECK(!next_frame_types_.empty());
+ next_frame_types_[0] = kVideoFrameKey;
+ if (HasInternalSource()) {
+ // Try to request the frame if we have an external encoder with
+ // internal source since AddVideoFrame never will be called.
+ RTC_DCHECK(generic_encoder_);
+ if (generic_encoder_->RequestFrame(next_frame_types_) ==
+ WEBRTC_VIDEO_CODEC_OK) {
+ // Try to remove just-performed keyframe request, if stream still exists.
+ next_frame_types_[0] = kVideoFrameDelta;
+ }
+ }
}
EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
@@ -1187,10 +1286,9 @@
uint32_t framerate_fps = GetInputFramerateFps();
frame_dropper_.SetRates((bitrate_bps + 500) / 1000, framerate_fps);
-
- VideoBitrateAllocation bitrate_allocation =
- GetBitrateAllocationAndNotifyObserver(bitrate_bps, framerate_fps);
- video_sender_.SetChannelParameters(bitrate_allocation, framerate_fps);
+ SetEncoderRates(
+ GetBitrateAllocationAndNotifyObserver(bitrate_bps, framerate_fps),
+ framerate_fps);
encoder_start_bitrate_bps_ =
bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_;
@@ -1480,7 +1578,7 @@
frame_dropper_.Fill(frame_size, !keyframe);
}
- if (encoder_info_.has_internal_source) {
+ if (HasInternalSource()) {
// Update frame dropper after the fact for internal sources.
input_framerate_.Update(1u, clock_->TimeInMilliseconds());
frame_dropper_.Leak(GetInputFramerateFps());
@@ -1501,6 +1599,12 @@
}
}
+bool VideoStreamEncoder::HasInternalSource() const {
+ // TODO(sprang): Checking both info from encoder and from encoder factory
+ // until we have deprecated and removed the encoder factory info.
+ return codec_info_.has_internal_source || encoder_info_.has_internal_source;
+}
+
// Class holding adaptation information.
VideoStreamEncoder::AdaptCounter::AdaptCounter() {
fps_counters_.resize(kScaleReasonSize);
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 861028f..11aa570 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -139,6 +139,8 @@
const uint32_t target_bitrate_bps,
uint32_t framerate_fps) RTC_RUN_ON(&encoder_queue_);
uint32_t GetInputFramerateFps() RTC_RUN_ON(&encoder_queue_);
+ void SetEncoderRates(const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps) RTC_RUN_ON(&encoder_queue_);
// Class holding adaptation information.
class AdaptCounter final {
@@ -185,6 +187,7 @@
void RunPostEncode(EncodedImage encoded_image,
int64_t time_sent_us,
int temporal_index);
+ bool HasInternalSource() const RTC_RUN_ON(&encoder_queue_);
rtc::Event shutdown_event_;
@@ -201,7 +204,6 @@
const VideoStreamEncoderSettings settings_;
const RateControlSettings rate_control_settings_;
- vcm::VideoSender video_sender_ RTC_GUARDED_BY(&encoder_queue_);
const std::unique_ptr<OveruseFrameDetector> overuse_detector_
RTC_PT_GUARDED_BY(&encoder_queue_);
std::unique_ptr<QualityScaler> quality_scaler_ RTC_GUARDED_BY(&encoder_queue_)
@@ -283,6 +285,7 @@
RTC_GUARDED_BY(&encoder_queue_);
VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_);
+ VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_);
FrameDropper frame_dropper_ RTC_GUARDED_BY(&encoder_queue_);
// If frame dropper is not force disabled, frame dropping might still be
@@ -300,6 +303,16 @@
std::unique_ptr<EncoderBitrateAdjuster> bitrate_adjuster_
RTC_GUARDED_BY(&encoder_queue_);
+ // TODO(webrtc:10164): Refactor/remove these VCM classes.
+ // |generic_encoder_| instance is owned by |codec_database_|.
+ VCMGenericEncoder* generic_encoder_ RTC_GUARDED_BY(&encoder_queue_);
+ VCMEncodedFrameCallback generic_encoder_callback_
+ RTC_GUARDED_BY(&encoder_queue_);
+ VCMEncoderDataBase codec_database_ RTC_GUARDED_BY(&encoder_queue_);
+ // TODO(sprang): Change actually support keyframe per simulcast stream, or
+ // turn this into a simple bool |pending_keyframe_request_|.
+ std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(&encoder_queue_);
+
// All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members.
rtc::TaskQueue encoder_queue_;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index b419e60..9bf9cfd 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -49,8 +49,9 @@
const int kMinBalancedFramerateFps = 7;
const int64_t kFrameTimeoutMs = 100;
const size_t kMaxPayloadLength = 1440;
-const int kTargetBitrateBps = 1000000;
-const int kLowTargetBitrateBps = kTargetBitrateBps / 10;
+const uint32_t kTargetBitrateBps = 1000000;
+const uint32_t kSimulcastTargetBitrateBps = 3150000;
+const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
const int kMaxInitialFramedrop = 4;
const int kDefaultFramerate = 30;
const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
@@ -342,7 +343,8 @@
VideoEncoderConfig video_encoder_config;
video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
video_encoder_config.number_of_streams = num_streams;
- video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
+ video_encoder_config.max_bitrate_bps =
+ num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
video_encoder_config.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
kDefaultFramerate);
@@ -584,16 +586,42 @@
rate_factor_ = rate_factor;
}
- uint32_t GetLastFramerate() {
+ uint32_t GetLastFramerate() const {
rtc::CritScope lock(&local_crit_sect_);
return last_framerate_;
}
- VideoFrame::UpdateRect GetLastUpdateRect() {
+ VideoFrame::UpdateRect GetLastUpdateRect() const {
rtc::CritScope lock(&local_crit_sect_);
return last_update_rect_;
}
+ const std::vector<FrameType>& LastFrameTypes() const {
+ rtc::CritScope lock(&local_crit_sect_);
+ return last_frame_types_;
+ }
+
+ void InjectFrame(const VideoFrame& input_image, bool keyframe) {
+ const std::vector<FrameType> frame_type = {keyframe ? kVideoFrameKey
+ : kVideoFrameDelta};
+ {
+ rtc::CritScope lock(&local_crit_sect_);
+ last_frame_types_ = frame_type;
+ }
+ FakeEncoder::Encode(input_image, nullptr, &frame_type);
+ }
+
+ void ExpectNullFrame() {
+ rtc::CritScope lock(&local_crit_sect_);
+ expect_null_frame_ = true;
+ }
+
+ absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
+ auto allocation = last_bitrate_allocation_;
+ last_bitrate_allocation_.reset();
+ return allocation;
+ }
+
private:
int32_t Encode(const VideoFrame& input_image,
const CodecSpecificInfo* codec_specific_info,
@@ -601,9 +629,16 @@
bool block_encode;
{
rtc::CritScope lock(&local_crit_sect_);
- EXPECT_GT(input_image.timestamp(), timestamp_);
- EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
- EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
+ if (expect_null_frame_) {
+ EXPECT_EQ(input_image.timestamp(), 0u);
+ EXPECT_EQ(input_image.width(), 1);
+ last_frame_types_ = *frame_types;
+ expect_null_frame_ = false;
+ } else {
+ EXPECT_GT(input_image.timestamp(), timestamp_);
+ EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
+ EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
+ }
timestamp_ = input_image.timestamp();
ntp_time_ms_ = input_image.ntp_time_ms();
@@ -612,6 +647,7 @@
block_encode = block_next_encode_;
block_next_encode_ = false;
last_update_rect_ = input_image.update_rect();
+ last_frame_types_ = *frame_types;
}
int32_t result =
FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
@@ -658,6 +694,7 @@
}
}
last_framerate_ = framerate;
+ last_bitrate_allocation_ = rate_allocation;
return FakeEncoder::SetRateAllocation(adjusted_rate_allocation,
framerate);
}
@@ -677,8 +714,11 @@
bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
+ absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
VideoFrame::UpdateRect last_update_rect_
RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
+ std::vector<FrameType> last_frame_types_;
+ bool expect_null_frame_ = false;
};
class TestSink : public VideoStreamEncoder::EncoderSink {
@@ -744,6 +784,11 @@
return min_transmit_bitrate_bps_;
}
+ void SetNumExpectedLayers(size_t num_layers) {
+ rtc::CritScope lock(&crit_);
+ num_expected_layers_ = num_layers;
+ }
+
private:
Result OnEncodedImage(
const EncodedImage& encoded_image,
@@ -751,10 +796,18 @@
const RTPFragmentationHeader* fragmentation) override {
rtc::CritScope lock(&crit_);
EXPECT_TRUE(expect_frames_);
- last_timestamp_ = encoded_image.Timestamp();
+ uint32_t timestamp = encoded_image.Timestamp();
+ if (last_timestamp_ != timestamp) {
+ num_received_layers_ = 1;
+ } else {
+ ++num_received_layers_;
+ }
+ last_timestamp_ = timestamp;
last_width_ = encoded_image._encodedWidth;
last_height_ = encoded_image._encodedHeight;
- encoded_frame_event_.Set();
+ if (num_received_layers_ == num_expected_layers_) {
+ encoded_frame_event_.Set();
+ }
return Result(Result::OK, last_timestamp_);
}
@@ -773,6 +826,8 @@
uint32_t last_timestamp_ = 0;
uint32_t last_height_ = 0;
uint32_t last_width_ = 0;
+ size_t num_expected_layers_ = 1;
+ size_t num_received_layers_ = 0;
bool expect_frames_ = true;
int number_of_reconfigurations_ = 0;
int min_transmit_bitrate_bps_ = 0;
@@ -2168,28 +2223,38 @@
.Times(1);
video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
- const int64_t kStartTimeMs = 1;
video_source_.IncomingCapturedFrame(
- CreateFrame(kStartTimeMs, codec_width_, codec_height_));
- WaitForEncodedFrame(kStartTimeMs);
+ CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
+ WaitForEncodedFrame(rtc::TimeMillis());
+ absl::optional<VideoBitrateAllocation> bitrate_allocation =
+ fake_encoder_.GetAndResetLastBitrateAllocation();
+ // Check that encoder has been updated too, not just allocation observer.
+ EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
+ fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
// Not called on second frame.
EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
.Times(0);
video_source_.IncomingCapturedFrame(
- CreateFrame(kStartTimeMs + 1, codec_width_, codec_height_));
- WaitForEncodedFrame(kStartTimeMs + 1);
+ CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
+ WaitForEncodedFrame(rtc::TimeMillis());
+ fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
// Called after a process interval.
const int64_t kProcessIntervalMs =
vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
- fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec *
- (kProcessIntervalMs + (1000 / kDefaultFps)));
EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
.Times(1);
- video_source_.IncomingCapturedFrame(CreateFrame(
- kStartTimeMs + kProcessIntervalMs, codec_width_, codec_height_));
- WaitForEncodedFrame(kStartTimeMs + kProcessIntervalMs);
+ const int64_t start_time_ms = rtc::TimeMillis();
+ while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
+ WaitForEncodedFrame(rtc::TimeMillis());
+ fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
+ }
+
+ // Since rates are unchanged, encoder should not be reconfigured.
+ EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
video_stream_encoder_->Stop();
}
@@ -3429,4 +3494,93 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // First frame is always keyframe.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameKey}));
+
+ // Insert delta frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameDelta}));
+
+ // Request next frame be a key-frame.
+ video_stream_encoder_->SendKeyFrame();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+ WaitForEncodedFrame(3);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameKey}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
+ // Setup simulcast with three streams.
+ ResetEncoder("VP8", 3, 1, 1, false);
+ video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0);
+ // Wait for all three layers before triggering event.
+ sink_.SetNumExpectedLayers(3);
+
+ // First frame is always keyframe.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAreArray(
+ {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey}));
+
+ // Insert delta frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAreArray(
+ {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta}));
+
+ // Request next frame be a key-frame.
+ // Only first stream is configured to produce key-frame.
+ video_stream_encoder_->SendKeyFrame();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+ WaitForEncodedFrame(3);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAreArray(
+ {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
+ // Configure internal source factory and setup test again.
+ encoder_factory_.SetHasInternalSource(true);
+ ResetEncoder("VP8", 1, 1, 1, false);
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Call encoder directly, simulating internal source where encoded frame
+ // callback in VideoStreamEncoder is called despite no OnFrame().
+ fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
+ EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameKey}));
+
+ const std::vector<FrameType> kDeltaFrame = {kVideoFrameDelta};
+ // Need to set timestamp manually since manually for injected frame.
+ VideoFrame frame = CreateFrame(101, nullptr);
+ frame.set_timestamp(101);
+ fake_encoder_.InjectFrame(frame, false);
+ EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameDelta}));
+
+ // Request key-frame. The forces a dummy frame down into the encoder.
+ fake_encoder_.ExpectNullFrame();
+ video_stream_encoder_->SendKeyFrame();
+ EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ testing::ElementsAre(FrameType{kVideoFrameKey}));
+
+ video_stream_encoder_->Stop();
+}
} // namespace webrtc
