Switch low bw audio test to histograms.
Also requires a recipe change so the results processor switches to
histogram mode when this CL is landed.
Bug: chromium:1029452
Change-Id: Ic09deefc3f4f9d7a82ffeafeb5209fcfc361aece
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171683
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30884}
diff --git a/audio/BUILD.gn b/audio/BUILD.gn
index 142b140..c902922 100644
--- a/audio/BUILD.gn
+++ b/audio/BUILD.gn
@@ -206,7 +206,6 @@
if (is_android) {
deps += [ "//testing/android/native_test:native_test_native_code" ]
}
-
data = [
"../resources/voice_engine/audio_tiny16.wav",
"../resources/voice_engine/audio_tiny48.wav",
@@ -223,6 +222,8 @@
"../resources/voice_engine/audio_tiny16.wav",
"../resources/voice_engine/audio_tiny48.wav",
]
+ data_deps =
+ [ "//third_party/catapult/tracing/tracing/proto:histogram_proto" ]
if (is_win) {
data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ]
} else {
diff --git a/audio/test/low_bandwidth_audio_test.py b/audio/test/low_bandwidth_audio_test.py
index 44ad1a1..0744889 100755
--- a/audio/test/low_bandwidth_audio_test.py
+++ b/audio/test/low_bandwidth_audio_test.py
@@ -16,7 +16,6 @@
import argparse
import collections
-import json
import logging
import os
import re
@@ -58,7 +57,7 @@
parser.add_argument('--num-retries', default='0',
help='Number of times to retry the test on Android.')
parser.add_argument('--isolated_script_test_perf_output', default=None,
- help='Path to store perf results in chartjson format.')
+ help='Path to store perf results in histogram proto format.')
parser.add_argument('--extra-test-args', default=[], action='append',
help='Extra args to path to the test binary.')
@@ -170,7 +169,7 @@
if match:
raw_mos, _ = match.groups()
- return {'pesq_mos': (raw_mos, 'score')}
+ return {'pesq_mos': (raw_mos, 'unitless')}
else:
logging.error('PESQ: %s', out.splitlines()[-1])
return {}
@@ -196,41 +195,66 @@
return {}
mos_lqo, = match.groups()
- return {'polqa_mos_lqo': (mos_lqo, 'score')}
+ return {'polqa_mos_lqo': (mos_lqo, 'unitless')}
-def _AddChart(charts, metric, test_name, value, units):
- chart = charts.setdefault(metric, {})
- chart[test_name] = {
- "type": "scalar",
- "value": value,
- "units": units,
- }
+def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file):
+ from tracing.value import histogram_set
-
-def _AddRunPerfResults(charts, run_perf_results_file):
+ cc_histograms = histogram_set.HistogramSet()
with open(run_perf_results_file, 'rb') as f:
- per_run_perf_results = json.load(f)
- if 'charts' not in per_run_perf_results:
- return
- for metric, cases in per_run_perf_results['charts'].items():
- chart = charts.setdefault(metric, {})
- for case_name, case_value in cases.items():
- if case_name in chart:
- logging.error('Overriding results for %s/%s', metric, case_name)
- chart[case_name] = case_value
+ contents = f.read()
+ if not contents:
+ return
+
+ cc_histograms.ImportProto(contents)
+
+ histograms.Merge(cc_histograms)
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
'sample_rate_hz'])
+def _ConfigurePythonPath(args):
+ script_dir = os.path.dirname(os.path.realpath(__file__))
+ checkout_root = os.path.abspath(
+ os.path.join(script_dir, os.pardir, os.pardir))
+
+ sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'catapult',
+ 'tracing'))
+ sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'protobuf',
+ 'python'))
+
+ # The low_bandwidth_audio_perf_test gn rule will build the protobuf stub for
+ # python, so put it in the path for this script before we attempt to import
+ # it.
+ histogram_proto_path = os.path.join(
+ args.build_dir, 'pyproto', 'tracing', 'tracing', 'proto')
+ sys.path.insert(0, histogram_proto_path)
+
+ # Fail early in case the proto hasn't been built.
+ from tracing.proto import histogram_proto
+ if not histogram_proto.HAS_PROTO:
+ raise ImportError('Could not find histogram_pb2. You need to build the '
+ 'low_bandwidth_audio_perf_test target before invoking '
+ 'this script. Expected to find '
+ 'histogram_pb2.py in %s.' % histogram_proto_path)
+
+
def main():
# pylint: disable=W0101
logging.basicConfig(level=logging.INFO)
args = _ParseArgs()
+ _ConfigurePythonPath(args)
+
+ # Import catapult modules here after configuring the pythonpath.
+ from tracing.value import histogram_set
+ from tracing.value.diagnostics import reserved_infos
+ from tracing.value.diagnostics import generic_set
+
pesq_path, polqa_path = _GetPathToTools()
if pesq_path is None:
return 1
@@ -250,14 +274,14 @@
if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
- charts = {}
-
+ histograms = histogram_set.HistogramSet()
for analyzer in analyzers:
# Start the test executable that produces audio files.
test_process = subprocess.Popen(
_LogCommand(test_command + [
'--sample_rate_hz=%d' % analyzer.sample_rate_hz,
- '--test_case_prefix=%s' % analyzer.name
+ '--test_case_prefix=%s' % analyzer.name,
+ '--write_histogram_proto_json'
] + args.extra_test_args),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
perf_results_file = None
@@ -279,9 +303,12 @@
analyzer_results = analyzer.func(analyzer.executable,
reference_file, degraded_file)
for metric, (value, units) in analyzer_results.items():
- # Output a result for the perf dashboard.
+ hist = histograms.CreateHistogram(metric, units, [value])
+ user_story = generic_set.GenericSet([test_name])
+ hist.diagnostics[reserved_infos.STORIES.name] = user_story
+
+ # Output human readable results.
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
- _AddChart(charts, metric, test_name, value, units)
if args.remove:
os.remove(reference_file)
@@ -291,13 +318,13 @@
if perf_results_file:
perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
android=args.android, adb_prefix=adb_prefix)
- _AddRunPerfResults(charts, perf_results_file)
+ _MergeInPerfResultsFromCcTests(histograms, perf_results_file)
if args.remove:
os.remove(perf_results_file)
if args.isolated_script_test_perf_output:
- with open(args.isolated_script_test_perf_output, 'w') as f:
- json.dump({"format_version": "1.0", "charts": charts}, f)
+ with open(args.isolated_script_test_perf_output, 'wb') as f:
+ f.write(histograms.AsProto().SerializeToString())
return test_process.wait()
diff --git a/audio/test/pc_low_bandwidth_audio_test.cc b/audio/test/pc_low_bandwidth_audio_test.cc
index 37c8086..aafb65f 100644
--- a/audio/test/pc_low_bandwidth_audio_test.cc
+++ b/audio/test/pc_low_bandwidth_audio_test.cc
@@ -105,7 +105,7 @@
std::string PerfResultsOutputFile() {
return webrtc::test::OutputPath() + "PCLowBandwidth_perf_" +
- FileSampleRateSuffix() + ".json";
+ FileSampleRateSuffix() + ".pb";
}
void LogTestResults() {
