Add POLQA to low bandwidth audio test BUG=webrtc:7229 Review-Url: https://codereview.webrtc.org/2804083003 Cr-Commit-Position: refs/heads/master@{#17671}
diff --git a/tools-webrtc/audio_quality/linux/PolqaOem64.sha1 b/tools-webrtc/audio_quality/linux/PolqaOem64.sha1 new file mode 100644 index 0000000..00dd87c --- /dev/null +++ b/tools-webrtc/audio_quality/linux/PolqaOem64.sha1
@@ -0,0 +1 @@ +2f705f4fc8f4175f75d0d946ad57787b99126e44 \ No newline at end of file
diff --git a/tools-webrtc/audio_quality/win/PolqaOem64.dll.sha1 b/tools-webrtc/audio_quality/win/PolqaOem64.dll.sha1 new file mode 100644 index 0000000..77c5853 --- /dev/null +++ b/tools-webrtc/audio_quality/win/PolqaOem64.dll.sha1
@@ -0,0 +1 @@ +4b0a44ae698593cb3456aadd86207d168bb72abf \ No newline at end of file
diff --git a/tools-webrtc/audio_quality/win/PolqaOem64.exe.sha1 b/tools-webrtc/audio_quality/win/PolqaOem64.exe.sha1 new file mode 100644 index 0000000..bae133a --- /dev/null +++ b/tools-webrtc/audio_quality/win/PolqaOem64.exe.sha1
@@ -0,0 +1 @@ +af3d2dce28bb52786bdad14270fb9f2decb4c220 \ No newline at end of file
diff --git a/tools-webrtc/audio_quality/win/vcomp120.dll.sha1 b/tools-webrtc/audio_quality/win/vcomp120.dll.sha1 new file mode 100644 index 0000000..b5b3287 --- /dev/null +++ b/tools-webrtc/audio_quality/win/vcomp120.dll.sha1
@@ -0,0 +1 @@ +1f8c667df810fed8fb42a6680a15465ac1e288eb \ No newline at end of file
diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn index 0b4b580..0de4229 100644 --- a/webrtc/audio/BUILD.gn +++ b/webrtc/audio/BUILD.gn
@@ -100,6 +100,7 @@ "../test:fake_audio_device", "../test:test_common", "../test:test_main", + "//third_party/gflags", ] if (is_android) { deps += [ "//testing/android/native_test:native_test_native_code" ] @@ -107,6 +108,7 @@ data = [ "//resources/voice_engine/audio_tiny16.wav", + "//resources/voice_engine/audio_tiny48.wav", ] if (!build_with_chromium && is_clang) {
diff --git a/webrtc/audio/test/low_bandwidth_audio_test.cc b/webrtc/audio/test/low_bandwidth_audio_test.cc index 65f28fa..98cfa70 100644 --- a/webrtc/audio/test/low_bandwidth_audio_test.cc +++ b/webrtc/audio/test/low_bandwidth_audio_test.cc
@@ -10,18 +10,26 @@ #include <algorithm> +#include "gflags/gflags.h" #include "webrtc/audio/test/low_bandwidth_audio_test.h" #include "webrtc/common_audio/wav_file.h" #include "webrtc/test/gtest.h" #include "webrtc/system_wrappers/include/sleep.h" #include "webrtc/test/testsupport/fileutils.h" + +DEFINE_int32(sample_rate_hz, 16000, + "Sample rate (Hz) of the produced audio files."); + namespace { + // Wait half a second between stopping sending and stopping receiving audio. constexpr int kExtraRecordTimeMs = 500; -// The best that can be done with PESQ. -constexpr int kAudioFileBitRate = 16000; +std::string FileSampleRateSuffix() { + return std::to_string(FLAGS_sample_rate_hz / 1000); +} + } // namespace namespace webrtc { @@ -41,14 +49,15 @@ } std::string AudioQualityTest::AudioInputFile() { - return test::ResourcePath("voice_engine/audio_tiny16", "wav"); + return test::ResourcePath("voice_engine/audio_tiny" + FileSampleRateSuffix(), + "wav"); } std::string AudioQualityTest::AudioOutputFile() { const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - return webrtc::test::OutputPath() + - "LowBandwidth_" + test_info->name() + ".wav"; + return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() + + "_" + FileSampleRateSuffix() + ".wav"; } std::unique_ptr<test::FakeAudioDevice::Capturer> @@ -59,7 +68,7 @@ std::unique_ptr<test::FakeAudioDevice::Renderer> AudioQualityTest::CreateRenderer() { return test::FakeAudioDevice::CreateBoundedWavFileWriter( - AudioOutputFile(), kAudioFileBitRate); + AudioOutputFile(), FLAGS_sample_rate_hz); } void AudioQualityTest::OnFakeAudioDevicesCreated(
diff --git a/webrtc/audio/test/low_bandwidth_audio_test.py b/webrtc/audio/test/low_bandwidth_audio_test.py index 8f40681..8815daa 100755 --- a/webrtc/audio/test/low_bandwidth_audio_test.py +++ b/webrtc/audio/test/low_bandwidth_audio_test.py
@@ -15,6 +15,7 @@ """ import argparse +import collections import logging import os import re @@ -59,13 +60,14 @@ tools_dir = os.path.join(SRC_DIR, 'tools-webrtc') toolchain_dir = os.path.join(tools_dir, 'audio_quality') - # Download pesq. + # Download PESQ and POLQA. download_script = os.path.join(tools_dir, 'download_tools.py') command = [sys.executable, download_script, toolchain_dir] subprocess.check_call(_LogCommand(command)) pesq_path = os.path.join(toolchain_dir, _GetPlatform(), 'pesq') - return pesq_path + polqa_path = os.path.join(toolchain_dir, _GetPlatform(), 'PolqaOem64') + return pesq_path, polqa_path def ExtractTestRuns(lines, echo=False): @@ -108,13 +110,74 @@ return out_file_path +def _RunPesq(executable_path, reference_file, degraded_file, + sample_rate_hz=16000): + directory = os.path.dirname(reference_file) + assert os.path.dirname(degraded_file) == directory + + # Analyze audio. + command = [executable_path, '+%d' % sample_rate_hz, + os.path.basename(reference_file), + os.path.basename(degraded_file)] + # Need to provide paths in the current directory due to a bug in PESQ: + # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than + # 'path/to', PESQ crashes. + out = subprocess.check_output(_LogCommand(command), + cwd=directory, stderr=subprocess.STDOUT) + + # Find the scores in stdout of PESQ. + match = re.search( + r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', out) + if match: + raw_mos, _ = match.groups() + + return {'pesq_mos': (raw_mos, 'score')} + else: + logging.error('PESQ: %s', out.splitlines()[-1]) + return {} + + +def _RunPolqa(executable_path, reference_file, degraded_file): + # Analyze audio. + command = [executable_path, '-q', '-LC', 'NB', + '-Ref', reference_file, '-Test', degraded_file] + try: + process = subprocess.Popen(_LogCommand(command), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + if e.errno == os.errno.ENOENT: + logging.warning('POLQA executable missing, skipping test.') + return {} + else: + raise + out, err = process.communicate() + + # Find the scores in stdout of POLQA. + match = re.search(r'\bMOS-LQO:\s+([\d.]+)', out) + + if process.returncode != 0 or not match: + if process.returncode == 2: + logging.warning('%s (2)', err.strip()) + logging.warning('POLQA license error, skipping test.') + else: + logging.error('%s (%d)', err.strip(), process.returncode) + return {} + + mos_lqo, = match.groups() + return {'polqa_mos_lqo': (mos_lqo, 'score')} + + +Analyzer = collections.namedtuple('Analyzer', ['func', 'executable', + 'sample_rate_hz']) + + def main(): # pylint: disable=W0101 logging.basicConfig(level=logging.INFO) args = _ParseArgs() - pesq_path = _DownloadTools() + pesq_path, polqa_path = _DownloadTools() out_dir = os.path.join(args.build_dir, '..') if args.android: @@ -123,51 +186,44 @@ else: test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')] - # Start the test executable that produces audio files. - test_process = subprocess.Popen(_LogCommand(test_command), - stdout=subprocess.PIPE) + analyzers = [Analyzer(_RunPesq, pesq_path, 16000)] + # Check if POLQA can run at all, or skip the 48 kHz tests entirely. + example_path = os.path.join(SRC_DIR, 'resources', + 'voice_engine', 'audio_tiny48.wav') + if _RunPolqa(polqa_path, example_path, example_path): + analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000)) - try: - lines = iter(test_process.stdout.readline, '') - for result in ExtractTestRuns(lines, echo=True): - (android_device, test_name, reference_file, degraded_file) = result + for analyzer in analyzers: + # Start the test executable that produces audio files. + test_process = subprocess.Popen( + _LogCommand(test_command + ['--sample_rate_hz=%d' % + analyzer.sample_rate_hz]), + stdout=subprocess.PIPE) + try: + lines = iter(test_process.stdout.readline, '') + for result in ExtractTestRuns(lines, echo=True): + (android_device, test_name, reference_file, degraded_file) = result - adb_prefix = (args.adb_path,) - if android_device: - adb_prefix += ('-s', android_device) + adb_prefix = (args.adb_path,) + if android_device: + adb_prefix += ('-s', android_device) - reference_file = _GetFile(reference_file, out_dir, - android=args.android, adb_prefix=adb_prefix) - degraded_file = _GetFile(degraded_file, out_dir, move=True, - android=args.android, adb_prefix=adb_prefix) + reference_file = _GetFile(reference_file, out_dir, + android=args.android, adb_prefix=adb_prefix) + degraded_file = _GetFile(degraded_file, out_dir, move=True, + android=args.android, adb_prefix=adb_prefix) - # Analyze audio. - pesq_command = [pesq_path, '+16000', - os.path.basename(reference_file), - os.path.basename(degraded_file)] - # Need to provide paths in the current directory due to a bug in PESQ: - # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than - # 'path/to', PESQ crashes. - pesq_output = subprocess.check_output(_LogCommand(pesq_command), - cwd=out_dir) + analyzer_results = analyzer.func(analyzer.executable, + reference_file, degraded_file) + for metric, (value, units) in analyzer_results.items(): + # Output a result for the perf dashboard. + print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units) - # Find the scores in stdout of pesq. - match = re.search( - r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', - pesq_output) - if match: - raw_mos, _ = match.groups() - - # Output a result for the perf dashboard. - print 'RESULT pesq_mos: %s= %s score' % (test_name, raw_mos) - else: - logging.error('PESQ: %s', pesq_output.splitlines()[-1]) - - if args.remove: - os.remove(reference_file) - os.remove(degraded_file) - finally: - test_process.terminate() + if args.remove: + os.remove(reference_file) + os.remove(degraded_file) + finally: + test_process.terminate() return test_process.wait()