Enable capturing from camera in PC framework
Bug: webrtc:10138
Change-Id: Idcf10331b9f5208010b2bd29324e0fc1341db2d3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/156241
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29431}
diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index c6a5409..5ab0f68 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -275,6 +275,8 @@
":test_peer",
":video_quality_analyzer_injection_helper",
"../..:field_trial",
+ "../..:platform_video_capturer",
+ "../..:video_test_common",
"../../../api:audio_quality_analyzer_api",
"../../../api:libjingle_peerconnection_api",
"../../../api:media_stream_interface",
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index 2605461..fcef1fe 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -159,7 +159,8 @@
const webrtc::VideoFrame& frame) {
rtc::CritScope crit(&lock_);
auto it = frame_stats_.find(frame.id());
- RTC_DCHECK(it != frame_stats_.end());
+ RTC_DCHECK(it != frame_stats_.end())
+ << "Frame id=" << frame.id() << " not found";
frame_counters_.pre_encoded++;
stream_frame_counters_[it->second.stream_label].pre_encoded++;
it->second.pre_encode_time = Now();
diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
index b7d87b8..436418b 100644
--- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
+++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
@@ -11,6 +11,7 @@
#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
#include <utility>
+#include <vector>
#include "absl/memory/memory.h"
#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h"
@@ -38,6 +39,37 @@
test::VideoFrameWriter* video_writer_;
};
+class AnalyzingFramePreprocessor
+ : public test::TestVideoCapturer::FramePreprocessor {
+ public:
+ AnalyzingFramePreprocessor(
+ std::string stream_label,
+ VideoQualityAnalyzerInterface* analyzer,
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks)
+ : stream_label_(std::move(stream_label)),
+ analyzer_(analyzer),
+ sinks_(std::move(sinks)) {}
+ ~AnalyzingFramePreprocessor() override = default;
+
+ VideoFrame Preprocess(const VideoFrame& source_frame) override {
+ // Copy VideoFrame to be able to set id on it.
+ VideoFrame frame = source_frame;
+ uint16_t frame_id = analyzer_->OnFrameCaptured(stream_label_, frame);
+ frame.set_id(frame_id);
+
+ for (auto& sink : sinks_) {
+ sink->OnFrame(frame);
+ }
+ return frame;
+ }
+
+ private:
+ const std::string stream_label_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+ const std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>
+ sinks_;
+};
+
// Intercepts generated frames and passes them also to video quality analyzer
// and to provided sinks.
class AnalyzingFrameGenerator final : public test::FrameGenerator {
@@ -142,10 +174,9 @@
analyzer_.get());
}
-std::unique_ptr<test::FrameGenerator>
-VideoQualityAnalyzerInjectionHelper::WrapFrameGenerator(
+std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor(
const VideoConfig& config,
- std::unique_ptr<test::FrameGenerator> delegate,
test::VideoFrameWriter* writer) const {
std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks;
if (writer) {
@@ -156,9 +187,8 @@
test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(),
config.width, config.height)));
}
- return std::make_unique<AnalyzingFrameGenerator>(
- std::move(*config.stream_label), std::move(delegate), analyzer_.get(),
- std::move(sinks));
+ return std::make_unique<AnalyzingFramePreprocessor>(
+ std::move(*config.stream_label), analyzer_.get(), std::move(sinks));
}
std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>
diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
index eb07a5d..4918768 100644
--- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
+++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
@@ -25,6 +25,7 @@
#include "test/frame_generator.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
#include "test/pc/e2e/analyzer/video/id_generator.h"
+#include "test/test_video_capturer.h"
#include "test/testsupport/video_frame_writer.h"
namespace webrtc {
@@ -54,16 +55,15 @@
std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory(
std::unique_ptr<VideoDecoderFactory> delegate) const;
- // Wraps frame generator, so video quality analyzer will gain access to the
- // captured frames. If |writer| in not nullptr, will dump captured frames
- // with provided writer.
- std::unique_ptr<test::FrameGenerator> WrapFrameGenerator(
- const VideoConfig& config,
- std::unique_ptr<test::FrameGenerator> delegate,
- test::VideoFrameWriter* writer) const;
- // Creates sink, that will allow video quality analyzer to get access to the
- // rendered frames. If |writer| in not nullptr, will dump rendered frames
- // with provided writer.
+ // Creates VideoFrame preprocessor, that will allow video quality analyzer to
+ // get access to the captured frames. If |writer| in not nullptr, will dump
+ // captured frames with provided writer.
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ CreateFramePreprocessor(const VideoConfig& config,
+ test::VideoFrameWriter* writer) const;
+ // Creates sink, that will allow video quality analyzer to get access to
+ // the rendered frames. If |writer| in not nullptr, will dump rendered
+ // frames with provided writer.
std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink(
const VideoConfig& config,
test::VideoFrameWriter* writer) const;
diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc
index eecb5ae..aa656c8 100644
--- a/test/pc/e2e/peer_connection_quality_test.cc
+++ b/test/pc/e2e/peer_connection_quality_test.cc
@@ -30,9 +30,11 @@
#include "rtc_base/numerics/safe_conversions.h"
#include "system_wrappers/include/cpu_info.h"
#include "system_wrappers/include/field_trial.h"
+#include "test/frame_generator_capturer.h"
#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include "test/pc/e2e/stats_poller.h"
+#include "test/platform_video_capturer.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
@@ -67,7 +69,9 @@
<< "; video_config.input_file_name="
<< video_config.input_file_name.has_value()
<< "; video_config.screen_share_config="
- << video_config.screen_share_config.has_value() << ";";
+ << video_config.screen_share_config.has_value()
+ << "; video_config.capturing_device_index="
+ << video_config.capturing_device_index.has_value() << ";";
return builder.str();
}
@@ -420,7 +424,7 @@
// Audio dumps.
RTC_CHECK(!alice_);
RTC_CHECK(!bob_);
- // Ensuring that FrameGeneratorCapturerVideoTrackSource and VideoFrameWriter
+ // Ensuring that TestVideoCapturerVideoTrackSource and VideoFrameWriter
// are destroyed on the right thread.
RTC_CHECK(alice_video_sources_.empty());
RTC_CHECK(bob_video_sources_.empty());
@@ -436,7 +440,8 @@
for (auto* p : params) {
for (auto& video_config : p->video_configs) {
if (!video_config.generator && !video_config.input_file_name &&
- !video_config.screen_share_config) {
+ !video_config.screen_share_config &&
+ !video_config.capturing_device_index) {
video_config.generator = VideoGeneratorType::kDefault;
}
if (!video_config.stream_label) {
@@ -485,15 +490,16 @@
video_labels.insert(video_config.stream_label.value()).second;
RTC_CHECK(inserted) << "Duplicate video_config.stream_label="
<< video_config.stream_label.value();
- RTC_CHECK(video_config.generator || video_config.input_file_name ||
- video_config.screen_share_config)
- << VideoConfigSourcePresenceToString(video_config);
- RTC_CHECK(!(video_config.input_file_name && video_config.generator))
- << VideoConfigSourcePresenceToString(video_config);
- RTC_CHECK(
- !(video_config.input_file_name && video_config.screen_share_config))
- << VideoConfigSourcePresenceToString(video_config);
- RTC_CHECK(!(video_config.screen_share_config && video_config.generator))
+ int input_sources_count = 0;
+ if (video_config.generator)
+ ++input_sources_count;
+ if (video_config.input_file_name)
+ ++input_sources_count;
+ if (video_config.screen_share_config)
+ ++input_sources_count;
+ if (video_config.capturing_device_index)
+ ++input_sources_count;
+ RTC_CHECK_EQ(input_sources_count, 1)
<< VideoConfigSourcePresenceToString(video_config);
if (video_config.screen_share_config) {
@@ -680,37 +686,27 @@
TearDownCall();
}
-std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
PeerConnectionE2EQualityTest::MaybeAddMedia(TestPeer* peer) {
MaybeAddAudio(peer);
return MaybeAddVideo(peer);
}
-std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
PeerConnectionE2EQualityTest::MaybeAddVideo(TestPeer* peer) {
// Params here valid because of pre-run validation.
Params* params = peer->params();
- std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>> out;
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> out;
for (auto video_config : params->video_configs) {
- // Create video generator.
- std::unique_ptr<test::FrameGenerator> frame_generator =
- CreateFrameGenerator(video_config);
-
- // Wrap it to inject video quality analyzer and enable dump of input video
- // if required.
+ // Setup input video source into peer connection.
test::VideoFrameWriter* writer =
MaybeCreateVideoWriter(video_config.input_dump_file_name, video_config);
- frame_generator =
- video_quality_analyzer_injection_helper_->WrapFrameGenerator(
- video_config, std::move(frame_generator), writer);
-
- // Setup FrameGenerator into peer connection.
- auto capturer = std::make_unique<test::FrameGeneratorCapturer>(
- clock_, std::move(frame_generator), video_config.fps,
- *task_queue_factory_);
- capturer->Init();
- rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource> source =
- new rtc::RefCountedObject<FrameGeneratorCapturerVideoTrackSource>(
+ std::unique_ptr<test::TestVideoCapturer> capturer = CreateVideoCapturer(
+ video_config,
+ video_quality_analyzer_injection_helper_->CreateFramePreprocessor(
+ video_config, writer));
+ rtc::scoped_refptr<TestVideoCapturerVideoTrackSource> source =
+ new rtc::RefCountedObject<TestVideoCapturerVideoTrackSource>(
std::move(capturer),
/*is_screencast=*/video_config.screen_share_config &&
video_config.screen_share_config->use_text_content_hint);
@@ -740,9 +736,24 @@
return out;
}
-std::unique_ptr<test::FrameGenerator>
-PeerConnectionE2EQualityTest::CreateFrameGenerator(
- const VideoConfig& video_config) {
+std::unique_ptr<test::TestVideoCapturer>
+PeerConnectionE2EQualityTest::CreateVideoCapturer(
+ const VideoConfig& video_config,
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ frame_preprocessor) {
+ if (video_config.capturing_device_index) {
+ std::unique_ptr<test::TestVideoCapturer> capturer =
+ test::CreateVideoCapturer(video_config.width, video_config.height,
+ video_config.fps,
+ *video_config.capturing_device_index);
+ capturer->SetFramePreprocessor(std::move(frame_preprocessor));
+ RTC_CHECK(capturer)
+ << "Failed to obtain input stream from capturing device #"
+ << *video_config.capturing_device_index;
+ return capturer;
+ }
+
+ std::unique_ptr<test::FrameGenerator> frame_generator = nullptr;
if (video_config.generator) {
absl::optional<test::FrameGenerator::OutputType> frame_generator_type =
absl::nullopt;
@@ -753,22 +764,28 @@
} else if (video_config.generator == VideoGeneratorType::kI010) {
frame_generator_type = test::FrameGenerator::OutputType::kI010;
}
- return test::FrameGenerator::CreateSquareGenerator(
+ frame_generator = test::FrameGenerator::CreateSquareGenerator(
static_cast<int>(video_config.width),
static_cast<int>(video_config.height), frame_generator_type,
absl::nullopt);
}
if (video_config.input_file_name) {
- return test::FrameGenerator::CreateFromYuvFile(
+ frame_generator = test::FrameGenerator::CreateFromYuvFile(
std::vector<std::string>(/*count=*/1,
video_config.input_file_name.value()),
video_config.width, video_config.height, /*frame_repeat_count=*/1);
}
if (video_config.screen_share_config) {
- return CreateScreenShareFrameGenerator(video_config);
+ frame_generator = CreateScreenShareFrameGenerator(video_config);
}
- RTC_NOTREACHED() << "Unsupported video_config input source";
- return nullptr;
+ RTC_CHECK(frame_generator) << "Unsupported video_config input source";
+
+ auto capturer = std::make_unique<test::FrameGeneratorCapturer>(
+ clock_, std::move(frame_generator), video_config.fps,
+ *task_queue_factory_);
+ capturer->SetFramePreprocessor(std::move(frame_preprocessor));
+ capturer->Init();
+ return capturer;
}
std::unique_ptr<test::FrameGenerator>
@@ -957,8 +974,8 @@
}
void PeerConnectionE2EQualityTest::StartVideo(
- const std::vector<
- rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>& sources) {
+ const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>&
+ sources) {
for (auto& source : sources) {
if (source->state() != MediaSourceInterface::SourceState::kLive) {
source->Start();
diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h
index 43a2f94..dea5713 100644
--- a/test/pc/e2e/peer_connection_quality_test.h
+++ b/test/pc/e2e/peer_connection_quality_test.h
@@ -20,7 +20,7 @@
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
-#include "pc/test/frame_generator_capturer_video_track_source.h"
+#include "pc/video_track_source.h"
#include "rtc_base/task_queue_for_test.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "rtc_base/thread.h"
@@ -159,6 +159,33 @@
std::unique_ptr<Params> params_;
};
+class TestVideoCapturerVideoTrackSource : public VideoTrackSource {
+ public:
+ TestVideoCapturerVideoTrackSource(
+ std::unique_ptr<test::TestVideoCapturer> video_capturer,
+ bool is_screencast)
+ : VideoTrackSource(/*remote=*/false),
+ video_capturer_(std::move(video_capturer)),
+ is_screencast_(is_screencast) {}
+
+ ~TestVideoCapturerVideoTrackSource() = default;
+
+ void Start() { SetState(kLive); }
+
+ void Stop() { SetState(kMuted); }
+
+ bool is_screencast() const override { return is_screencast_; }
+
+ protected:
+ rtc::VideoSourceInterface<VideoFrame>* source() override {
+ return video_capturer_.get();
+ }
+
+ private:
+ std::unique_ptr<test::TestVideoCapturer> video_capturer_;
+ const bool is_screencast_;
+};
+
class PeerConnectionE2EQualityTest
: public PeerConnectionE2EQualityTestFixture {
public:
@@ -230,12 +257,14 @@
// Have to be run on the signaling thread.
void SetupCallOnSignalingThread(const RunParams& run_params);
void TearDownCallOnSignalingThread();
- std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
MaybeAddMedia(TestPeer* peer);
- std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
MaybeAddVideo(TestPeer* peer);
- std::unique_ptr<test::FrameGenerator> CreateFrameGenerator(
- const VideoConfig& video_config);
+ std::unique_ptr<test::TestVideoCapturer> CreateVideoCapturer(
+ const VideoConfig& video_config,
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ frame_preprocessor);
std::unique_ptr<test::FrameGenerator> CreateScreenShareFrameGenerator(
const VideoConfig& video_config);
void MaybeAddAudio(TestPeer* peer);
@@ -244,8 +273,8 @@
void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor);
void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor);
void StartVideo(
- const std::vector<
- rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>& sources);
+ const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>&
+ sources);
void TearDownCall();
test::VideoFrameWriter* MaybeCreateVideoWriter(
absl::optional<std::string> file_name,
@@ -270,9 +299,9 @@
std::vector<std::unique_ptr<QualityMetricsReporter>>
quality_metrics_reporters_;
- std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
alice_video_sources_;
- std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
bob_video_sources_;
std::vector<std::unique_ptr<test::VideoFrameWriter>> video_writers_;
std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>
diff --git a/test/test_video_capturer.cc b/test/test_video_capturer.cc
index fa43f9f..a894cec 100644
--- a/test/test_video_capturer.cc
+++ b/test/test_video_capturer.cc
@@ -19,15 +19,16 @@
namespace webrtc {
namespace test {
-TestVideoCapturer::TestVideoCapturer() = default;
TestVideoCapturer::~TestVideoCapturer() = default;
-void TestVideoCapturer::OnFrame(const VideoFrame& frame) {
+void TestVideoCapturer::OnFrame(const VideoFrame& original_frame) {
int cropped_width = 0;
int cropped_height = 0;
int out_width = 0;
int out_height = 0;
+ VideoFrame frame = MaybePreprocess(original_frame);
+
if (!video_adapter_.AdaptFrameResolution(
frame.width(), frame.height(), frame.timestamp_us() * 1000,
&cropped_width, &cropped_height, &out_width, &out_height)) {
@@ -75,5 +76,14 @@
wants.target_pixel_count, wants.max_pixel_count, wants.max_framerate_fps);
}
+VideoFrame TestVideoCapturer::MaybePreprocess(const VideoFrame& frame) {
+ rtc::CritScope crit(&lock_);
+ if (preprocessor_ != nullptr) {
+ return preprocessor_->Preprocess(frame);
+ } else {
+ return frame;
+ }
+}
+
} // namespace test
} // namespace webrtc
diff --git a/test/test_video_capturer.h b/test/test_video_capturer.h
index 0f1886b..114767a 100644
--- a/test/test_video_capturer.h
+++ b/test/test_video_capturer.h
@@ -18,18 +18,29 @@
#include "api/video/video_source_interface.h"
#include "media/base/video_adapter.h"
#include "media/base/video_broadcaster.h"
+#include "rtc_base/critical_section.h"
namespace webrtc {
namespace test {
class TestVideoCapturer : public rtc::VideoSourceInterface<VideoFrame> {
public:
- TestVideoCapturer();
+ class FramePreprocessor {
+ public:
+ virtual ~FramePreprocessor() = default;
+
+ virtual VideoFrame Preprocess(const VideoFrame& frame) = 0;
+ };
+
~TestVideoCapturer() override;
void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override;
void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override;
+ void SetFramePreprocessor(std::unique_ptr<FramePreprocessor> preprocessor) {
+ rtc::CritScope crit(&lock_);
+ preprocessor_ = std::move(preprocessor);
+ }
protected:
void OnFrame(const VideoFrame& frame);
@@ -37,7 +48,10 @@
private:
void UpdateVideoAdapter();
+ VideoFrame MaybePreprocess(const VideoFrame& frame);
+ rtc::CriticalSection lock_;
+ std::unique_ptr<FramePreprocessor> preprocessor_ RTC_GUARDED_BY(lock_);
rtc::VideoBroadcaster broadcaster_;
cricket::VideoAdapter video_adapter_;
};
