blob: c275ecdff68051143b0838f6db969fcdff2848a8 [file] [log] [blame]
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <map>
#include <memory>
#include <string>
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/metrics/metrics_logger.h"
#include "api/test/metrics/stdout_metrics_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/pclf/media_configuration.h"
#include "api/test/pclf/media_quality_test_params.h"
#include "api/test/pclf/peer_configurer.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/units/time_delta.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/pc/e2e/metric_metadata_keys.h"
#include "test/pc/e2e/peer_connection_quality_test.h"
#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
using ::testing::UnorderedElementsAre;
using ::webrtc::test::DefaultMetricsLogger;
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Metric;
using ::webrtc::test::MetricsExporter;
using ::webrtc::test::StdoutMetricsExporter;
using ::webrtc::test::Unit;
using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
// Adds a peer with some audio and video (the client should not care about
// details about audio and video configs).
void AddDefaultAudioVideoPeer(
absl::string_view peer_name,
absl::string_view audio_stream_label,
absl::string_view video_stream_label,
const PeerNetworkDependencies& network_dependencies,
PeerConnectionE2EQualityTestFixture& fixture) {
AudioConfig audio{std::string(audio_stream_label)};
audio.sync_group = std::string(peer_name);
VideoConfig video(std::string(video_stream_label), 320, 180, 15);
video.sync_group = std::string(peer_name);
auto peer = std::make_unique<PeerConfigurer>(network_dependencies);
peer->SetName(peer_name);
peer->SetAudioConfig(std::move(audio));
peer->AddVideoConfig(std::move(video));
peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
fixture.AddPeer(std::move(peer));
}
// Metric fields to assert on
struct MetricValidationInfo {
std::string test_case;
std::string name;
Unit unit;
ImprovementDirection improvement_direction;
std::map<std::string, std::string> metadata;
};
bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
a.improvement_direction == b.improvement_direction &&
a.metadata == b.metadata;
}
std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
os << "{ test_case=" << m.test_case << "; name=" << m.name
<< "; unit=" << test::ToString(m.unit)
<< "; improvement_direction=" << test::ToString(m.improvement_direction)
<< "; metadata={ ";
for (const auto& [key, value] : m.metadata) {
os << "{ key=" << key << "; value=" << value << " }";
}
os << " }}";
return os;
}
std::vector<MetricValidationInfo> ToValidationInfo(
const std::vector<Metric>& metrics) {
std::vector<MetricValidationInfo> out;
for (const Metric& m : metrics) {
out.push_back(
MetricValidationInfo{.test_case = m.test_case,
.name = m.name,
.unit = m.unit,
.improvement_direction = m.improvement_direction,
.metadata = m.metric_metadata});
}
return out;
}
TEST(PeerConnectionE2EQualityTestMetricNamesTest,
ExportedMetricsHasCorrectNamesAndAnnotation) {
std::unique_ptr<NetworkEmulationManager> network_emulation =
CreateNetworkEmulationManager(TimeMode::kSimulated);
DefaultMetricsLogger metrics_logger(
network_emulation->time_controller()->GetClock());
PeerConnectionE2EQualityTest fixture(
"test_case", *network_emulation->time_controller(),
/*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
&metrics_logger);
EmulatedEndpoint* alice_endpoint =
network_emulation->CreateEndpoint(EmulatedEndpointConfig());
EmulatedEndpoint* bob_endpoint =
network_emulation->CreateEndpoint(EmulatedEndpointConfig());
network_emulation->CreateRoute(
alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
bob_endpoint);
network_emulation->CreateRoute(
bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
alice_endpoint);
EmulatedNetworkManagerInterface* alice_network =
network_emulation->CreateEmulatedNetworkManagerInterface(
{alice_endpoint});
EmulatedNetworkManagerInterface* bob_network =
network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
alice_network->network_dependencies(), fixture);
AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
bob_network->network_dependencies(), fixture);
fixture.AddQualityMetricsReporter(
std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
std::map<std::string, std::vector<EmulatedEndpoint*>>(
{{"alice", alice_network->endpoints()},
{"bob", bob_network->endpoints()}}),
network_emulation.get(), &metrics_logger));
// Run for at least 7 seconds, so AV-sync metrics will be collected.
fixture.Run(RunParams(TimeDelta::Seconds(7)));
std::vector<MetricValidationInfo> metrics =
ToValidationInfo(metrics_logger.GetCollectedMetrics());
EXPECT_THAT(
metrics,
UnorderedElementsAre(
// Metrics from PeerConnectionE2EQualityTest
MetricValidationInfo{
.test_case = "test_case",
.name = "alice_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case",
.name = "bob_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from DefaultAudioQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
// Metrics from DefaultVideoQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "qp",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "qp",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case",
.name = "cpu_usage_%",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
// Metrics from StatsBasedNetworkQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from VideoQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from CrossMediaMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice_alice_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_alice_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {
{MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}}));
}
} // namespace
} // namespace webrtc_pc_e2e
} // namespace webrtc