Introduce peer connection end-2-end quality test fixture interface.
Also introduce interface for video quality analyze and mock interface,
that then will be extended for audio quality analyze.
Bug: webrtc:10138
Change-Id: I0e3957fb2af1b12e796f154765580ddf562c7814
Reviewed-on: https://webrtc-review.googlesource.com/c/116500
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Yves Gerey <yvesg@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26157}diff --git a/test/BUILD.gn b/test/BUILD.gn
index b9f3d0b..ffa6d01 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -21,6 +21,7 @@
":test_renderer",
":test_support",
":video_test_common",
+ "pc/e2e/api:peer_connection_quality_test_fixture_api",
]
if (rtc_include_tests) {
diff --git a/test/pc/e2e/api/BUILD.gn b/test/pc/e2e/api/BUILD.gn
new file mode 100644
index 0000000..f9fc538
--- /dev/null
+++ b/test/pc/e2e/api/BUILD.gn
@@ -0,0 +1,53 @@
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../../webrtc.gni")
+
+rtc_source_set("video_quality_analyzer_api") {
+ visibility = [ "*" ]
+ sources = [
+ "video_quality_analyzer_interface.h",
+ ]
+
+ deps = [
+ "../../../../api/video:encoded_image",
+ "../../../../api/video:video_frame",
+ "../../../../api/video_codecs:video_codecs_api",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_source_set("audio_quality_analyzer_api") {
+ visibility = [ "*" ]
+ sources = [
+ "audio_quality_analyzer_interface.h",
+ ]
+
+ deps = []
+}
+
+rtc_source_set("peer_connection_quality_test_fixture_api") {
+ visibility = [ "*" ]
+ sources = [
+ "peerconnection_quality_test_fixture.h",
+ ]
+
+ deps = [
+ ":audio_quality_analyzer_api",
+ ":video_quality_analyzer_api",
+ "../../../../api:callfactory_api",
+ "../../../../api:fec_controller_api",
+ "../../../../api:libjingle_peerconnection_api",
+ "../../../../api:simulated_network_api",
+ "../../../../api/transport:network_control",
+ "../../../../api/video_codecs:video_codecs_api",
+ "../../../../logging:rtc_event_log_api",
+ "../../../../rtc_base:rtc_base",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
diff --git a/test/pc/e2e/api/audio_quality_analyzer_interface.h b/test/pc/e2e/api/audio_quality_analyzer_interface.h
new file mode 100644
index 0000000..25f7410
--- /dev/null
+++ b/test/pc/e2e/api/audio_quality_analyzer_interface.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_
+#define TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_
+
+namespace webrtc {
+
+class AudioQualityAnalyzerInterface {};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_
diff --git a/test/pc/e2e/api/peerconnection_quality_test_fixture.h b/test/pc/e2e/api/peerconnection_quality_test_fixture.h
new file mode 100644
index 0000000..1640340
--- /dev/null
+++ b/test/pc/e2e/api/peerconnection_quality_test_fixture.h
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
+#define TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "api/asyncresolverfactory.h"
+#include "api/call/callfactoryinterface.h"
+#include "api/fec_controller.h"
+#include "api/media_transport_interface.h"
+#include "api/peerconnectioninterface.h"
+#include "api/test/simulated_network.h"
+#include "api/transport/network_control.h"
+#include "api/video_codecs/video_decoder_factory.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_factory.h"
+#include "logging/rtc_event_log/rtc_event_log_factory_interface.h"
+#include "rtc_base/network.h"
+#include "rtc_base/rtccertificategenerator.h"
+#include "rtc_base/sslcertificate.h"
+#include "rtc_base/thread.h"
+#include "test/pc/e2e/api/audio_quality_analyzer_interface.h"
+#include "test/pc/e2e/api/video_quality_analyzer_interface.h"
+
+namespace webrtc {
+
+// TODO(titovartem) move to API when it will be stabilized.
+class PeerConnectionE2EQualityTestFixture {
+ public:
+ struct PeerConnectionFactoryComponents {
+ std::unique_ptr<CallFactoryInterface> call_factory;
+ std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory;
+ std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory;
+ std::unique_ptr<NetworkControllerFactoryInterface>
+ network_controller_factory;
+ std::unique_ptr<MediaTransportFactory> media_transport_factory;
+
+ // Will be passed to MediaEngineInterface, that will be used in
+ // PeerConnectionFactory.
+ std::unique_ptr<VideoEncoderFactory> video_encoder_factory;
+ std::unique_ptr<VideoDecoderFactory> video_decoder_factory;
+ };
+
+ struct PeerConnectionComponents {
+ std::unique_ptr<rtc::NetworkManager> network_manager;
+ std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory;
+ std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator;
+ std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier;
+ };
+
+ struct InjectableComponents {
+ explicit InjectableComponents(rtc::Thread* network_thread)
+ : network_thread(network_thread) {}
+
+ rtc::Thread* network_thread;
+
+ std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies;
+ std::unique_ptr<PeerConnectionComponents> pc_dependencies;
+ };
+
+ struct ScreenShareConfig {
+ // If true, slides will be generated programmatically.
+ bool generate_slides;
+ int32_t slide_change_interval;
+ // If equal to 0, no scrolling will be applied.
+ int32_t scroll_duration;
+ // If empty, default set of slides will be used.
+ std::vector<std::string> slides_yuv_file_names;
+ };
+
+ struct VideoConfig {
+ size_t width;
+ size_t height;
+ int32_t fps;
+ // Have to be unique among all specified configs for all peers in the call.
+ absl::optional<std::string> stream_label;
+ // Only single from 3 next fields can be specified.
+ // If specified generator with this name will be used as input.
+ absl::optional<std::string> generator_name;
+ // If specified this file will be used as input.
+ absl::optional<std::string> input_file_name;
+ // If specified screen share video stream will be created as input.
+ absl::optional<ScreenShareConfig> screen_share_config;
+ // If specified the input stream will be also copied to specified file.
+ absl::optional<std::string> input_dump_file_name;
+ // If specified this file will be used as output on the receiver side for
+ // this stream. If multiple streams will be produced by input stream,
+ // output files will be appended with indexes.
+ absl::optional<std::string> output_file_name;
+ };
+
+ struct AudioConfig {
+ enum Mode {
+ kGenerated,
+ kFile,
+ };
+ Mode mode;
+ // Have to be specified only if mode = kFile
+ absl::optional<std::string> input_file_name;
+ // If specified the input stream will be also copied to specified file.
+ absl::optional<std::string> input_dump_file_name;
+ // If specified the output stream will be copied to specified file.
+ absl::optional<std::string> output_file_name;
+ // Audio options to use.
+ cricket::AudioOptions audio_options;
+ };
+
+ struct Params {
+ // If |video_configs| is empty - no video should be added to the test call.
+ std::vector<VideoConfig> video_configs;
+ // If |audio_config| is presented audio stream will be configured
+ absl::optional<AudioConfig> audio_config;
+
+ PeerConnectionInterface::RTCConfiguration rtc_configuration;
+ };
+
+ struct Analyzers {
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer;
+ std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer;
+ };
+
+ virtual void Run() = 0;
+ virtual ~PeerConnectionE2EQualityTestFixture() = default;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
diff --git a/test/pc/e2e/api/video_quality_analyzer_interface.h b/test/pc/e2e/api/video_quality_analyzer_interface.h
new file mode 100644
index 0000000..ae4a428
--- /dev/null
+++ b/test/pc/e2e/api/video_quality_analyzer_interface.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_
+#define TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_
+
+#include <memory>
+#include <string>
+
+#include "absl/types/optional.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "api/video_codecs/video_encoder.h"
+
+namespace webrtc {
+
+class VideoQualityAnalyzerInterface {
+ public:
+ virtual ~VideoQualityAnalyzerInterface() = default;
+
+ // Will be called by framework before test. |threads_count| is number of
+ // threads, that analyzer can use for heavy calculations. Analyzer can perform
+ // simple calculations on the calling thread in each method, but should
+ // remember, that is the same thread, that is used in video pipeline.
+ virtual void Start(uint16_t threads_count) {}
+
+ // Will be called when frame was generated from the input stream.
+ // Returns frame id, that will be set by framework to the frame.
+ virtual uint16_t OnFrameCaptured(std::string stream_label,
+ const VideoFrame& frame) = 0;
+ // Will be called before calling the real encoder.
+ virtual void OnFramePreEncode(const VideoFrame& frame) {}
+ // Will be called for each EncodedImage received from encoder. Single
+ // VideoFrame can produce multiple EncodedImages. Each encoded image will
+ // have id from VideoFrame.
+ virtual void OnFrameEncoded(uint16_t frame_id,
+ const EncodedImage& encoded_image) {}
+ // Will be called for each frame dropped by encoder.
+ virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) {}
+ // Will be called before calling the real decoder.
+ virtual void OnFrameReceived(uint16_t frame_id,
+ const EncodedImage& encoded_image) {}
+ // Will be called after decoding the frame. |decode_time_ms| is a decode
+ // time provided by decoder itself. If decoder doesn’t produce such
+ // information can be omitted.
+ virtual void OnFrameDecoded(const VideoFrame& frame,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) {}
+ // Will be called when frame will be obtained from PeerConnection stack.
+ virtual void OnFrameRendered(const VideoFrame& frame) {}
+ // Will be called if real encoder return not WEBRTC_VIDEO_CODEC_OK.
+ virtual void OnEncoderError(const VideoFrame& frame, int32_t error_code) {}
+ // Will be called if real decoder return not WEBRTC_VIDEO_CODEC_OK.
+ virtual void OnDecoderError(uint16_t frame_id, int32_t error_code) {}
+
+ // Tells analyzer, that analysis complete and it should calculate final
+ // statistics.
+ virtual void Stop() {}
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_
