Add VideoProcessorIntegrationTest for MediaCodec implementations.
* Add VideoProcessorIntegrationTestMediaCodec tests, including
a standard foreman test and a forced SW fallback test.
* Morph PlotVideoProcessorIntegrationTest into
VideoProcessorIntegrationTestParameterized. This test is intended
to be patched locally, depending on what metric is of interest.
It is run on the bots to ensure that it doesn't break.
* Remove the plot_videoprocessor_integrationtest binary. The test
above is instead moved to the modules_tests binary.
BUG=webrtc:6634
Review-Url: https://codereview.webrtc.org/3011043002
Cr-Original-Commit-Position: refs/heads/master@{#19727}
Cr-Mirrored-From: https://chromium.googlesource.com/external/webrtc
Cr-Mirrored-Commit: df232992597bd2865fdbede3e9d7e45445e4f122
diff --git a/BUILD.gn b/BUILD.gn
index 210d208..b11016c 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -307,7 +307,6 @@
"modules/remote_bitrate_estimator:bwe_simulations_tests",
"modules/rtp_rtcp:test_packet_masks_metrics",
"modules/video_capture:video_capture_internal_impl",
- "modules/video_coding:plot_videoprocessor_integrationtest",
"ortc:ortc_unittests",
"pc:peerconnection_unittests",
"pc:rtc_pc_unittests",
diff --git a/modules/BUILD.gn b/modules/BUILD.gn
index dd8e54b..ead7855 100644
--- a/modules/BUILD.gn
+++ b/modules/BUILD.gn
@@ -77,7 +77,13 @@
data = modules_tests_resources
if (is_android) {
- deps += [ "//testing/android/native_test:native_test_native_code" ]
+ deps += [
+ # NOTE(brandtr): Including Java classes seems only to be possible from
+ # rtc_test targets. Therefore we include this target here, instead of
+ # in video_coding_modules_tests, where it is actually used.
+ "../sdk/android:libjingle_peerconnection_java",
+ "//testing/android/native_test:native_test_native_code",
+ ]
shard_timeout = 900
}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 3bf221b..3d3a3af 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -366,55 +366,20 @@
]
}
- rtc_source_set("video_coding_videoprocessor_integration_test") {
- testonly = true
+ video_coding_modules_tests_resources = [
+ "../../../resources/foreman_128x96.yuv",
+ "../../../resources/foreman_160x120.yuv",
+ "../../../resources/foreman_176x144.yuv",
+ "../../../resources/foreman_320x240.yuv",
+ ]
- sources = [
- "codecs/test/videoprocessor_integrationtest.cc",
- "codecs/test/videoprocessor_integrationtest.h",
- ]
-
- deps = [
- ":video_codecs_test_framework",
- ":video_coding",
- ":video_coding_utility",
- ":webrtc_h264",
- ":webrtc_vp8",
- ":webrtc_vp9",
- "../..:webrtc_common",
- "../../media:rtc_media",
- "../../rtc_base:rtc_base_approved",
- "../../rtc_base:rtc_task_queue",
- "../../system_wrappers:system_wrappers",
- "../../test:test_support",
- "../../test:video_test_common",
- "../../test:video_test_support",
- ]
-
- if (is_android) {
- sources += [
- "codecs/test/android_test_initializer.cc",
- "codecs/test/android_test_initializer.h",
+ if (is_ios || is_mac) {
+ bundle_data("video_coding_modules_tests_resources_bundle_data") {
+ testonly = true
+ sources = video_coding_modules_tests_resources
+ outputs = [
+ "{{bundle_resources_dir}}/{{source_file_part}}",
]
-
- deps += [
- "../../rtc_base:rtc_base_approved",
- "../../sdk/android:libjingle_peerconnection_jni",
- "//base",
- ]
- }
-
- if (is_ios || is_mac) {
- deps += [
- ":codec_test_objc",
- "../../media:rtc_media_base",
- "../../sdk:videotoolbox_objc",
- ]
- }
-
- if (!build_with_chromium && is_clang) {
- # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
- suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
}
}
@@ -429,105 +394,63 @@
}
sources = [
"codecs/h264/test/h264_impl_unittest.cc",
+ "codecs/test/videoprocessor_integrationtest.cc",
+ "codecs/test/videoprocessor_integrationtest.h",
"codecs/test/videoprocessor_integrationtest_libvpx.cc",
"codecs/test/videoprocessor_integrationtest_openh264.cc",
+ "codecs/test/videoprocessor_integrationtest_parameterized.cc",
"codecs/vp8/test/vp8_impl_unittest.cc",
"codecs/vp9/test/vp9_impl_unittest.cc",
]
deps = [
":video_codecs_test_framework",
- ":video_coding_videoprocessor_integration_test",
+ ":video_coding",
+ ":video_coding_utility",
":webrtc_h264",
":webrtc_vp8",
":webrtc_vp9",
+ "../..:webrtc_common",
"../../api:optional",
"../../api:video_frame_api",
- "../../common_video:common_video",
+ "../../common_video",
+ "../../media:rtc_audio_video",
"../../rtc_base:rtc_base_approved",
+ "../../system_wrappers",
"../../test:field_trial",
"../../test:test_support",
"../../test:video_test_common",
+ "../../test:video_test_support",
"../video_capture",
]
- if (is_ios || is_mac) {
- deps += [ ":codec_test_objc" ]
- }
-
- if (rtc_use_h264) {
- defines = [ "WEBRTC_USE_H264" ]
- }
-
- if (!build_with_chromium && is_clang) {
- # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
- suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
- }
- }
-
- plot_videoprocessor_integrationtest_resources = [
- "../../../resources/foreman_128x96.yuv",
- "../../../resources/foreman_160x120.yuv",
- "../../../resources/foreman_176x144.yuv",
- "../../../resources/foreman_320x240.yuv",
- "../../../resources/foreman_cif.yuv",
- ]
-
- if (is_ios || is_mac) {
- bundle_data("plot_videoprocessor_integrationtest_bundle_data") {
- testonly = true
- sources = plot_videoprocessor_integrationtest_resources
- outputs = [
- "{{bundle_resources_dir}}/{{source_file_part}}",
- ]
- }
- }
-
- # This executable is meant for local codec perf testing and should not be run
- # on the trybots/buildbots, hence the existence of this special build target.
- rtc_test("plot_videoprocessor_integrationtest") {
- testonly = true
-
- sources = [
- "codecs/test/plot_videoprocessor_integrationtest.cc",
- ]
-
- deps = [
- ":video_coding_videoprocessor_integration_test",
- "../../test:test_main",
- "../../test:video_test_common",
- "../video_capture",
- ]
-
- data = plot_videoprocessor_integrationtest_resources
+ data = video_coding_modules_tests_resources
if (is_android) {
- deps += [
- "../../rtc_base:rtc_base_approved",
-
- # TODO(brandtr): Figure out if the java dep below could be moved into
- # :video_coding_videoprocessor_integration_test, where it belongs.
- # When that is done, support for Android HW codecs can be added to the
- # modules_tests target as well.
- "../../sdk/android:libjingle_peerconnection_java",
- "../../sdk/android:libjingle_peerconnection_jni",
- "//base",
- "//testing/android/native_test:native_test_support",
+ sources += [
+ "codecs/test/android_test_initializer.cc",
+ "codecs/test/android_test_initializer.h",
+ "codecs/test/videoprocessor_integrationtest_mediacodec.cc",
]
- shard_timeout = 900
+ deps += [
+ "../../sdk/android:libjingle_peerconnection_jni",
+ "//base",
+ ]
}
if (is_ios || is_mac) {
deps += [
":codec_test_objc",
- ":plot_videoprocessor_integrationtest_bundle_data",
+ ":video_coding_modules_tests_resources_bundle_data",
+ "../../media:rtc_media_base",
+ "../../sdk:videotoolbox_objc",
]
}
- # TODO(brandtr): Remove this define when the modules_tests target properly
- # loads the Java classes mentioned above.
- defines = [ "WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED" ]
+ if (rtc_use_h264) {
+ defines = [ "WEBRTC_USE_H264" ]
+ }
if (!build_with_chromium && is_clang) {
# Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
diff --git a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
index d598ffb..0a8b094 100755
--- a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
+++ b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py
@@ -18,8 +18,9 @@
import matplotlib.pyplot as plt
# Log events.
-EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.'
-EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.'
+EVENT_START = \
+ 'RUN ] CodecSettings/VideoProcessorIntegrationTestParameterized.'
+EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.'
# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
BITRATE = ('Target bitrate', 'target bitrate (kbps)')
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
index cbc2518..5d54caa 100644
--- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
@@ -71,8 +71,7 @@
} // namespace
VideoProcessorIntegrationTest::VideoProcessorIntegrationTest() {
-#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) && \
- defined(WEBRTC_ANDROID)
+#if defined(WEBRTC_ANDROID)
InitializeAndroidObjects();
#endif
}
@@ -276,7 +275,6 @@
void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() {
std::unique_ptr<cricket::WebRtcVideoEncoderFactory> encoder_factory;
if (config_.hw_encoder) {
-#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED)
#if defined(WEBRTC_ANDROID)
encoder_factory.reset(new jni::MediaCodecVideoEncoderFactory());
#elif defined(WEBRTC_IOS)
@@ -286,13 +284,11 @@
#else
RTC_NOTREACHED() << "Only support HW encoder on Android and iOS.";
#endif
-#endif // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED
} else {
encoder_factory.reset(new cricket::InternalEncoderFactory());
}
if (config_.hw_decoder) {
-#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED)
#if defined(WEBRTC_ANDROID)
decoder_factory_.reset(new jni::MediaCodecVideoDecoderFactory());
#elif defined(WEBRTC_IOS)
@@ -302,7 +298,6 @@
#else
RTC_NOTREACHED() << "Only support HW decoder on Android and iOS.";
#endif
-#endif // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED
} else {
decoder_factory_.reset(new cricket::InternalDecoderFactory());
}
diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
new file mode 100644
index 0000000..47734d7
--- /dev/null
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
+
+#include <vector>
+
+#include "webrtc/test/field_trial.h"
+#include "webrtc/test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace test {
+
+#if defined(WEBRTC_ANDROID)
+
+namespace {
+const int kForemanNumFrames = 300;
+const std::nullptr_t kNoVisualizationParams = nullptr;
+} // namespace
+
+class VideoProcessorIntegrationTestMediaCodec
+ : public VideoProcessorIntegrationTest {
+ protected:
+ VideoProcessorIntegrationTestMediaCodec() {
+ config_.filename = "foreman_cif";
+ config_.input_filename = ResourcePath(config_.filename, "yuv");
+ config_.output_filename =
+ TempFilename(OutputPath(), "videoprocessor_integrationtest_mediacodec");
+ config_.verbose = false;
+ config_.hw_encoder = true;
+ config_.hw_decoder = true;
+ }
+};
+
+TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) {
+ SetCodecSettings(&config_, kVideoCodecVP8, 1, false, false, false, false,
+ false, 352, 288);
+
+ RateProfile rate_profile;
+ SetRateProfile(&rate_profile, 0, 500, 30, 0); // Start below |low_kbps|.
+ rate_profile.frame_index_rate_update[1] = kForemanNumFrames + 1;
+ rate_profile.num_frames = kForemanNumFrames;
+
+ // The thresholds below may have to be tweaked to let even poor MediaCodec
+ // implementations pass. If this test fails on the bots, disable it and
+ // ping brandtr@.
+ std::vector<RateControlThresholds> rc_thresholds;
+ AddRateControlThresholds(5, 95, 20, 10, 10, 0, 1, &rc_thresholds);
+
+ QualityThresholds quality_thresholds(30.0, 15.0, 0.90, 0.40);
+
+ ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
+ kNoVisualizationParams);
+}
+
+TEST_F(VideoProcessorIntegrationTestMediaCodec,
+ Foreman240p100kbpsVp8WithForcedSwFallback) {
+ ScopedFieldTrials override_field_trials(
+ "WebRTC-VP8-Forced-Fallback-Encoder/Enabled-150,175,10000,1/");
+
+ config_.filename = "foreman_320x240";
+ config_.input_filename = ResourcePath(config_.filename, "yuv");
+ config_.sw_fallback_encoder = true;
+ SetCodecSettings(&config_, kVideoCodecVP8, 1, false, false, false, false,
+ false, 320, 240);
+
+ RateProfile rate_profile;
+ SetRateProfile(&rate_profile, 0, 100, 10, 0); // Start below |low_kbps|.
+ SetRateProfile(&rate_profile, 1, 100, 10, 80); // Fallback in this bucket.
+ SetRateProfile(&rate_profile, 2, 200, 10, 200); // Switch back here.
+ rate_profile.frame_index_rate_update[3] = kForemanNumFrames + 1;
+ rate_profile.num_frames = kForemanNumFrames;
+
+ // The thresholds below may have to be tweaked to let even poor MediaCodec
+ // implementations pass. If this test fails on the bots, disable it and
+ // ping brandtr@.
+ std::vector<RateControlThresholds> rc_thresholds;
+ AddRateControlThresholds(0, 50, 75, 70, 10, 0, 1, &rc_thresholds);
+ AddRateControlThresholds(0, 50, 25, 12, 60, 0, 1, &rc_thresholds);
+ AddRateControlThresholds(0, 65, 15, 5, 5, 0, 1, &rc_thresholds);
+
+ QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85);
+
+ ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
+ kNoVisualizationParams);
+}
+
+#endif // defined(WEBRTC_ANDROID)
+
+} // namespace test
+} // namespace webrtc
diff --git a/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
similarity index 75%
rename from modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc
rename to modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
index ef6dbde..1a15558 100644
--- a/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc
+++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
@@ -18,7 +18,7 @@
namespace {
// Loop variables.
-const int kBitrates[] = {30, 50, 100, 200, 300, 500, 1000};
+const int kBitrates[] = {500};
const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8};
const bool kHwCodec[] = {false};
@@ -37,21 +37,21 @@
false, // save_decoded_y4m
};
-const int kNumFrames = 300;
+const int kNumFrames = 30;
} // namespace
// Tests for plotting statistics from logs.
-class PlotVideoProcessorIntegrationTest
+class VideoProcessorIntegrationTestParameterized
: public VideoProcessorIntegrationTest,
public ::testing::WithParamInterface<
::testing::tuple<int, VideoCodecType, bool>> {
protected:
- PlotVideoProcessorIntegrationTest()
+ VideoProcessorIntegrationTestParameterized()
: bitrate_(::testing::get<0>(GetParam())),
codec_type_(::testing::get<1>(GetParam())),
hw_codec_(::testing::get<2>(GetParam())) {}
- ~PlotVideoProcessorIntegrationTest() override = default;
+ ~VideoProcessorIntegrationTestParameterized() override = default;
void RunTest(int width,
int height,
@@ -86,30 +86,29 @@
const bool hw_codec_;
};
-INSTANTIATE_TEST_CASE_P(
- CodecSettings,
- PlotVideoProcessorIntegrationTest,
- ::testing::Combine(::testing::ValuesIn(kBitrates),
- ::testing::ValuesIn(kVideoCodecType),
- ::testing::ValuesIn(kHwCodec)));
+INSTANTIATE_TEST_CASE_P(CodecSettings,
+ VideoProcessorIntegrationTestParameterized,
+ ::testing::Combine(::testing::ValuesIn(kBitrates),
+ ::testing::ValuesIn(kVideoCodecType),
+ ::testing::ValuesIn(kHwCodec)));
-TEST_P(PlotVideoProcessorIntegrationTest, Process_128x96_30fps) {
+TEST_P(VideoProcessorIntegrationTestParameterized, Process_128x96_30fps) {
RunTest(128, 96, 30, "foreman_128x96");
}
-TEST_P(PlotVideoProcessorIntegrationTest, Process_160x120_30fps) {
+TEST_P(VideoProcessorIntegrationTestParameterized, Process_160x120_30fps) {
RunTest(160, 120, 30, "foreman_160x120");
}
-TEST_P(PlotVideoProcessorIntegrationTest, Process_176x144_30fps) {
+TEST_P(VideoProcessorIntegrationTestParameterized, Process_176x144_30fps) {
RunTest(176, 144, 30, "foreman_176x144");
}
-TEST_P(PlotVideoProcessorIntegrationTest, Process_320x240_30fps) {
+TEST_P(VideoProcessorIntegrationTestParameterized, Process_320x240_30fps) {
RunTest(320, 240, 30, "foreman_320x240");
}
-TEST_P(PlotVideoProcessorIntegrationTest, Process_352x288_30fps) {
+TEST_P(VideoProcessorIntegrationTestParameterized, Process_352x288_30fps) {
RunTest(352, 288, 30, "foreman_cif");
}
