blob: 94a46fbae55467b02ae07acacb78691142240b91 [file] [log] [blame]
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
#include "absl/strings/string_view.h"
#include "api/array_view.h"
#include "api/scoped_refptr.h"
#include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h"
#include "api/test/network_emulation/network_emulation_interfaces.h"
#include "api/test/network_emulation_manager.h"
#include "api/units/data_rate.h"
#include "api/units/timestamp.h"
#include "rtc_base/event.h"
#include "rtc_base/ip_address.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
constexpr int kStatsWaitTimeoutMs = 1000;
// Field trial which controls whether to report standard-compliant bytes
// sent/received per stream. If enabled, padding and headers are not included
// in bytes sent or received.
constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
std::unique_ptr<EmulatedNetworkStats> PopulateStats(
std::vector<EmulatedEndpoint*> endpoints,
NetworkEmulationManager* network_emulation) {
rtc::Event stats_loaded;
std::unique_ptr<EmulatedNetworkStats> stats;
network_emulation->GetStats(endpoints,
[&](std::unique_ptr<EmulatedNetworkStats> s) {
stats = std::move(s);
stats_loaded.Set();
});
bool stats_received = stats_loaded.Wait(kStatsWaitTimeoutMs);
RTC_CHECK(stats_received);
return stats;
}
std::map<rtc::IPAddress, std::string> PopulateIpToPeer(
const std::map<std::string, std::vector<EmulatedEndpoint*>>&
peer_endpoints) {
std::map<rtc::IPAddress, std::string> out;
for (const auto& entry : peer_endpoints) {
for (const EmulatedEndpoint* const endpoint : entry.second) {
RTC_CHECK(out.find(endpoint->GetPeerLocalAddress()) == out.end())
<< "Two peers can't share the same endpoint";
out.emplace(endpoint->GetPeerLocalAddress(), entry.first);
}
}
return out;
}
} // namespace
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
NetworkLayerStatsCollector(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation)
: peer_endpoints_(std::move(peer_endpoints)),
ip_to_peer_(PopulateIpToPeer(peer_endpoints_)),
network_emulation_(network_emulation) {}
void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
Start() {
MutexLock lock(&mutex_);
// Check that network stats are clean before test execution.
for (const auto& entry : peer_endpoints_) {
std::unique_ptr<EmulatedNetworkStats> stats =
PopulateStats(entry.second, network_emulation_);
RTC_CHECK_EQ(stats->PacketsSent(), 0);
RTC_CHECK_EQ(stats->PacketsReceived(), 0);
}
}
void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
AddPeer(absl::string_view peer_name,
std::vector<EmulatedEndpoint*> endpoints) {
MutexLock lock(&mutex_);
// When new peer is added not in the constructor, don't check if it has empty
// stats, because their endpoint could be used for traffic before.
peer_endpoints_.emplace(peer_name, std::move(endpoints));
for (const EmulatedEndpoint* const endpoint : endpoints) {
RTC_CHECK(ip_to_peer_.find(endpoint->GetPeerLocalAddress()) ==
ip_to_peer_.end())
<< "Two peers can't share the same endpoint";
ip_to_peer_.emplace(endpoint->GetPeerLocalAddress(), peer_name);
}
}
std::map<std::string,
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStats>
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
GetStats() {
MutexLock lock(&mutex_);
std::map<std::string, NetworkLayerStats> peer_to_stats;
std::map<std::string, std::vector<std::string>> sender_to_receivers;
for (const auto& entry : peer_endpoints_) {
NetworkLayerStats stats;
stats.stats = PopulateStats(entry.second, network_emulation_);
const std::string& peer_name = entry.first;
for (const auto& income_stats_entry :
stats.stats->IncomingStatsPerSource()) {
const rtc::IPAddress& source_ip = income_stats_entry.first;
auto it = ip_to_peer_.find(source_ip);
if (it == ip_to_peer_.end()) {
// Source IP is unknown for this collector, so will be skipped.
continue;
}
sender_to_receivers[it->second].push_back(peer_name);
}
peer_to_stats.emplace(peer_name, std::move(stats));
}
for (auto& entry : peer_to_stats) {
const std::vector<std::string>& receivers =
sender_to_receivers[entry.first];
entry.second.receivers =
std::set<std::string>(receivers.begin(), receivers.end());
}
return peer_to_stats;
}
void StatsBasedNetworkQualityMetricsReporter::AddPeer(
absl::string_view peer_name,
std::vector<EmulatedEndpoint*> endpoints) {
collector_.AddPeer(peer_name, std::move(endpoints));
}
void StatsBasedNetworkQualityMetricsReporter::Start(
absl::string_view test_case_name,
const TrackIdStreamInfoMap* reporter_helper) {
test_case_name_ = std::string(test_case_name);
collector_.Start();
start_time_ = clock_->CurrentTime();
}
void StatsBasedNetworkQualityMetricsReporter::OnStatsReports(
absl::string_view pc_label,
const rtc::scoped_refptr<const RTCStatsReport>& report) {
PCStats cur_stats;
auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
for (const auto& stat : inbound_stats) {
cur_stats.payload_received +=
DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) +
stat->header_bytes_received.ValueOrDefault(0ul));
}
auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>();
for (const auto& stat : outbound_stats) {
cur_stats.payload_sent +=
DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) +
stat->header_bytes_sent.ValueOrDefault(0ul));
}
auto candidate_pairs_stats = report->GetStatsOfType<RTCTransportStats>();
for (const auto& stat : candidate_pairs_stats) {
cur_stats.total_received +=
DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul));
cur_stats.total_sent +=
DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul));
cur_stats.packets_received += stat->packets_received.ValueOrDefault(0ul);
cur_stats.packets_sent += stat->packets_sent.ValueOrDefault(0ul);
}
MutexLock lock(&mutex_);
pc_stats_[std::string(pc_label)] = cur_stats;
}
void StatsBasedNetworkQualityMetricsReporter::StopAndReportResults() {
Timestamp end_time = clock_->CurrentTime();
if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) {
RTC_LOG(LS_ERROR)
<< "Non-standard GetStats; \"payload\" counts include RTP headers";
}
std::map<std::string, NetworkLayerStats> stats = collector_.GetStats();
for (const auto& entry : stats) {
LogNetworkLayerStats(entry.first, entry.second);
}
MutexLock lock(&mutex_);
for (const auto& pair : pc_stats_) {
auto it = stats.find(pair.first);
RTC_CHECK(it != stats.end())
<< "Peer name used for PeerConnection stats collection and peer name "
"used for endpoints naming doesn't match. No endpoints found for "
"peer "
<< pair.first;
const NetworkLayerStats& network_layer_stats = it->second;
int64_t total_packets_received = 0;
bool found = false;
for (const auto& dest_peer : network_layer_stats.receivers) {
auto pc_stats_it = pc_stats_.find(dest_peer);
if (pc_stats_it == pc_stats_.end()) {
continue;
}
found = true;
total_packets_received += pc_stats_it->second.packets_received;
}
int64_t packet_loss = -1;
if (found) {
packet_loss = pair.second.packets_sent - total_packets_received;
}
ReportStats(pair.first, pair.second, network_layer_stats, packet_loss,
end_time);
}
}
void StatsBasedNetworkQualityMetricsReporter::ReportStats(
const std::string& pc_label,
const PCStats& pc_stats,
const NetworkLayerStats& network_layer_stats,
int64_t packet_loss,
const Timestamp& end_time) {
ReportResult("bytes_discarded_no_receiver", pc_label,
network_layer_stats.stats->BytesDropped().bytes(),
"sizeInBytes");
ReportResult("packets_discarded_no_receiver", pc_label,
network_layer_stats.stats->PacketsDropped(), "unitless");
ReportResult("payload_bytes_received", pc_label,
pc_stats.payload_received.bytes(), "sizeInBytes");
ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(),
"sizeInBytes");
ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(),
"sizeInBytes");
ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless");
ReportResult("average_send_rate", pc_label,
(pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(),
"sizeInBytes");
ReportResult("packets_received", pc_label, pc_stats.packets_received,
"unitless");
ReportResult(
"average_receive_rate", pc_label,
(pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless");
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
absl::string_view network_label) const {
rtc::StringBuilder builder;
builder << test_case_name_ << "/" << network_label.data();
return builder.str();
}
void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
const std::string& peer_name,
const NetworkLayerStats& stats) const {
DataRate average_send_rate = stats.stats->PacketsSent() >= 2
? stats.stats->AverageSendRate()
: DataRate::Zero();
DataRate average_receive_rate = stats.stats->PacketsReceived() >= 2
? stats.stats->AverageReceiveRate()
: DataRate::Zero();
rtc::StringBuilder log;
log << "Raw network layer statistic for [" << peer_name << "]:\n"
<< "Local IPs:\n";
std::vector<rtc::IPAddress> local_ips = stats.stats->LocalAddresses();
for (size_t i = 0; i < local_ips.size(); ++i) {
log << " " << local_ips[i].ToString() << "\n";
}
if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) {
ReportResult("sent_packets_size", peer_name,
stats.stats->SentPacketsSizeCounter(), "sizeInBytes");
}
if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) {
ReportResult("received_packets_size", peer_name,
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
}
if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) {
ReportResult("dropped_packets_size", peer_name,
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
}
if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) {
ReportResult("sent_packets_queue_wait_time_us", peer_name,
stats.stats->SentPacketsQueueWaitTimeUs(), "unitless");
}
log << "Send statistic:\n"
<< " packets: " << stats.stats->PacketsSent()
<< " bytes: " << stats.stats->BytesSent().bytes()
<< " avg_rate (bytes/sec): " << average_send_rate.bytes_per_sec()
<< " avg_rate (bps): " << average_send_rate.bps() << "\n"
<< "Send statistic per destination:\n";
for (const auto& entry : stats.stats->OutgoingStatsPerDestination()) {
DataRate source_average_send_rate = entry.second->PacketsSent() >= 2
? entry.second->AverageSendRate()
: DataRate::Zero();
log << "(" << entry.first.ToString() << "):\n"
<< " packets: " << entry.second->PacketsSent()
<< " bytes: " << entry.second->BytesSent().bytes()
<< " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_send_rate.bps() << "\n";
if (!entry.second->SentPacketsSizeCounter().IsEmpty()) {
ReportResult("sent_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes");
}
}
log << "Receive statistic:\n"
<< " packets: " << stats.stats->PacketsReceived()
<< " bytes: " << stats.stats->BytesReceived().bytes()
<< " avg_rate (bytes/sec): " << average_receive_rate.bytes_per_sec()
<< " avg_rate (bps): " << average_receive_rate.bps() << "\n"
<< "Receive statistic per source:\n";
for (const auto& entry : stats.stats->IncomingStatsPerSource()) {
DataRate source_average_receive_rate =
entry.second->PacketsReceived() >= 2
? entry.second->AverageReceiveRate()
: DataRate::Zero();
log << "(" << entry.first.ToString() << "):\n"
<< " packets: " << entry.second->PacketsReceived()
<< " bytes: " << entry.second->BytesReceived().bytes()
<< " avg_rate (bytes/sec): "
<< source_average_receive_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_receive_rate.bps() << "\n";
if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) {
ReportResult("received_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
}
if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) {
ReportResult("dropped_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
}
}
RTC_LOG(LS_INFO) << log.str();
}
} // namespace webrtc_pc_e2e
} // namespace webrtc