Ignore BWE expectations in ScreamTest ScreamTests are despite running in simulated time not deterministic enough on all platforms and have caused flakiness on try bots. Therefore, ignore BWE expectations for now unless the flag --enable-scream-bwe-expectations is set. Bug: webrtc:447037083 Change-Id: I5a240806603e37952f08f7e900877baa6a6ff3e8 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/459440 Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org> Commit-Queue: Per Kjellander <perkj@webrtc.org> Cr-Commit-Position: refs/heads/main@{#47236}
diff --git a/test/peer_scenario/bwe_integration_tests/BUILD.gn b/test/peer_scenario/bwe_integration_tests/BUILD.gn index 729c2b0..763487a 100644 --- a/test/peer_scenario/bwe_integration_tests/BUILD.gn +++ b/test/peer_scenario/bwe_integration_tests/BUILD.gn
@@ -70,6 +70,7 @@ "../../../rtc_base:network_constants", "../../network:emulated_network", "../../scenario", + "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/strings", ] }
diff --git a/test/peer_scenario/bwe_integration_tests/scream_test.cc b/test/peer_scenario/bwe_integration_tests/scream_test.cc index ef5f84c..538b91a 100644 --- a/test/peer_scenario/bwe_integration_tests/scream_test.cc +++ b/test/peer_scenario/bwe_integration_tests/scream_test.cc
@@ -16,6 +16,7 @@ #include <utility> #include <vector> +#include "absl/flags/flag.h" #include "api/audio_options.h" #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" @@ -31,6 +32,7 @@ #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "rtc_base/logging.h" #include "test/create_frame_generator_capturer.h" #include "test/gmock.h" #include "test/gtest.h" @@ -39,6 +41,21 @@ #include "test/peer_scenario/peer_scenario.h" #include "test/peer_scenario/peer_scenario_client.h" +// Tests for SCReAM. +// Per default, the tests currently does not set expectations on BWE. The +// reason is due to that the tests are not deterministic enough on all +// platforms. +// TODO: bugs.webrtc.org/447037083 - Attempt to make tests stable enough for +// the none deterministic behaviour to not matter. +// To enable BWE expectations, set the flag --enable-scream-bwe-expectations. +// To write RTC eventlogs to file, set the flag --peer_logs=true and +// --peer_logs_root=<path>. + +ABSL_FLAG(bool, + enable_scream_bwe_expectations, + false, + "Enable BWE expectations in Scream tests."); + namespace webrtc { namespace { @@ -76,8 +93,15 @@ *result_listener << "the available send bitrate is " << available_bwe.kbps() << "kbps, which is not between " << low.kbps() << "kbps and " << high.kbps() << " kbps"; - - return false; + if (absl::GetFlag(FLAGS_enable_scream_bwe_expectations)) { + // TODO: bugs.webrtc.org/447037083 - Attempt to make tests stable enough for + // the none deterministic behaviour to not matter. + return false; + } + RTC_LOG(LS_ERROR) << "!!! the available send bitrate is " + << available_bwe.kbps() << "kbps, which is not between " + << low.kbps() << "kbps and " << high.kbps() << " kbps"; + return true; } MATCHER_P2(CurrentRoundTripTimeIsBetween, low, high, "") { @@ -398,9 +422,6 @@ {"WebRTC-Bwe-ScreamV2", "mode:only_after_ce"}}; SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - EXPECT_THAT(result.caller_stats.back(), - AvailableSendBitrateIsBetween(DataRate::KilobitsPerSec(350), - DataRate::KilobitsPerSec(660))); // All packets are sent as ECT1. EXPECT_EQ(GetPacketsSent(result.caller_stats.back()), @@ -408,6 +429,9 @@ // Not all packets has been received yet. EXPECT_GE(GetPacketsSentWithEct1(result.caller_stats.back()), 0.9 * (GetPacketsReceived(result.callee_stats.back()))); + EXPECT_THAT(result.caller_stats.back(), + AvailableSendBitrateIsBetween(DataRate::KilobitsPerSec(350), + DataRate::KilobitsPerSec(660))); } // Test that we can switch from Goog CC sending ECT1 to send ECT 0 and adapt. @@ -499,7 +523,6 @@ DataRate::KilobitsPerSec(2000), TimeDelta::Millis(25)); SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( DataRate::KilobitsPerSec(1300), DataRate::KilobitsPerSec(2300)))); @@ -680,7 +703,6 @@ s, DataRate::KilobitsPerSec(5000), TimeDelta::Millis(100), TimeDelta::Millis(200)); SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( DataRate::KilobitsPerSec(1200), DataRate::KilobitsPerSec(5000)))); @@ -697,7 +719,6 @@ CreateNetworkPath(s, /*use_dual_pi= */ false, DataRate::KilobitsPerSec(50000), TimeDelta::Millis(10)); SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - // After 400ms. EXPECT_THAT(result.caller_stats[3], AvailableSendBitrateIsBetween(DataRate::KilobitsPerSec(1200), @@ -730,14 +751,14 @@ network_builder.packet_queue_length(3), /*use_dual_pi= */ false); SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - - EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( - DataRate::KilobitsPerSec(200), - DataRate::KilobitsPerSec(1100)))); EXPECT_THAT( GetPacketsLost(result.callee_stats.back()) / static_cast<double>(GetPacketsSent(result.caller_stats.back())), Lt(0.05)); + + EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( + DataRate::KilobitsPerSec(200), + DataRate::KilobitsPerSec(1100)))); } TEST(ScreamTest, @@ -755,6 +776,7 @@ ASSERT_GE(GetPacketsLost(result.callee_stats.back()), 0.05 * GetPacketsSent(result.caller_stats.back())); + // Ignore estimate during rampup. EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( DataRate::KilobitsPerSec(100), @@ -779,6 +801,7 @@ // feedback is eventually received even if feedback packets are lost. EXPECT_GT(GetPacketsSent(result.caller_stats.back()), GetPacketsSent(result.caller_stats[5])); + EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( DataRate::KilobitsPerSec(300), DataRate::KilobitsPerSec(1300)))); @@ -804,6 +827,7 @@ // packets to fast if queued too long, BWE drop to a very low value. EXPECT_GT(GetPacketsSent(result.caller_stats.back()), GetPacketsSent(result.caller_stats[5])); + EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( DataRate::KilobitsPerSec(10), DataRate::KilobitsPerSec(1200)))); @@ -826,14 +850,14 @@ CreateNetworkPath(network_builder, /*use_dual_pi= */ false); SendMediaTestResult result = SendMediaInOneDirection(std::move(params), s); - - EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( - DataRate::KilobitsPerSec(150), - DataRate::KilobitsPerSec(900)))); EXPECT_THAT( GetPacketsLost(result.callee_stats.back()) / static_cast<double>(GetPacketsSent(result.caller_stats.back())), AllOf(Lt(0.08), Gt(0.01))); + + EXPECT_THAT(result.caller().subspan(1), Each(AvailableSendBitrateIsBetween( + DataRate::KilobitsPerSec(150), + DataRate::KilobitsPerSec(900)))); } TEST(ScreamTest, MaybeTest(LinkCapacity5MbitWithCrossTrafficNoEcn)) {