Roll gtest-parallel.
Brings in fixes that save log output to disk instead of piping them
through Python. Should fix problem where output from tests stall for
more than 10 seconds.
Also enabling JsepPeerConnectionP2PTestClient on all platforms again.
BUG=webrtc:5231
R=kjellander@webrtc.org
Review URL: https://codereview.webrtc.org/1509463002 .
Cr-Commit-Position: refs/heads/master@{#10917}
diff --git a/talk/app/webrtc/peerconnection_unittest.cc b/talk/app/webrtc/peerconnection_unittest.cc
index 9ecf080..1759965 100644
--- a/talk/app/webrtc/peerconnection_unittest.cc
+++ b/talk/app/webrtc/peerconnection_unittest.cc
@@ -908,19 +908,11 @@
rtc::scoped_ptr<MockDataChannelObserver> data_observer_;
};
-// Flaky on Mac Debug bots. See webrtc:5231
-#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
-#define MAYBE_JsepPeerConnectionP2PTestClient \
- DISABLED_JsepPeerConnectionP2PTestClient
-#else
-#define MAYBE_JsepPeerConnectionP2PTestClient JsepPeerConnectionP2PTestClient
-#endif
-
// TODO(deadbeef): Rename this to P2PTestConductor once the Linux memcheck and
// Windows DrMemory Full bots' blacklists are updated.
-class MAYBE_JsepPeerConnectionP2PTestClient : public testing::Test {
+class JsepPeerConnectionP2PTestClient : public testing::Test {
public:
- MAYBE_JsepPeerConnectionP2PTestClient()
+ JsepPeerConnectionP2PTestClient()
: pss_(new rtc::PhysicalSocketServer),
ss_(new rtc::VirtualSocketServer(pss_.get())),
ss_scope_(ss_.get()) {}
@@ -975,7 +967,7 @@
receiving_client_->VerifyLocalIceUfragAndPassword();
}
- ~MAYBE_JsepPeerConnectionP2PTestClient() {
+ ~JsepPeerConnectionP2PTestClient() {
if (initiating_client_) {
initiating_client_->set_signaling_message_receiver(nullptr);
}
@@ -1161,7 +1153,7 @@
// This test sets up a Jsep call between two parties and test Dtmf.
// TODO(holmer): Disabled due to sometimes crashing on buildbots.
// See issue webrtc/2378.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTestDtmf) {
+TEST_F(JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTestDtmf) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
VerifyDtmf();
@@ -1169,7 +1161,7 @@
// This test sets up a Jsep call between two parties and test that we can get a
// video aspect ratio of 16:9.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTest16To9) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTest16To9) {
ASSERT_TRUE(CreateTestClients());
FakeConstraints constraint;
double requested_ratio = 640.0/360;
@@ -1194,7 +1186,7 @@
// received video has a resolution of 1280*720.
// TODO(mallinath): Enable when
// http://code.google.com/p/webrtc/issues/detail?id=981 is fixed.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTest1280By720) {
+TEST_F(JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTest1280By720) {
ASSERT_TRUE(CreateTestClients());
FakeConstraints constraint;
constraint.SetMandatoryMinWidth(1280);
@@ -1206,13 +1198,13 @@
// This test sets up a call between two endpoints that are configured to use
// DTLS key agreement. As a result, DTLS is negotiated and used for transport.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestDtls) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestDtls) {
SetupAndVerifyDtlsCall();
}
// This test sets up a audio call initially and then upgrades to audio/video,
// using DTLS.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsRenegotiate) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsRenegotiate) {
MAYBE_SKIP_TEST(rtc::SSLStreamAdapter::HaveDtlsSrtp);
FakeConstraints setup_constraints;
setup_constraints.AddMandatory(MediaConstraintsInterface::kEnableDtlsSrtp,
@@ -1226,7 +1218,7 @@
// This test sets up a call transfer to a new caller with a different DTLS
// fingerprint.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsTransferCallee) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsTransferCallee) {
MAYBE_SKIP_TEST(rtc::SSLStreamAdapter::HaveDtlsSrtp);
SetupAndVerifyDtlsCall();
@@ -1244,7 +1236,7 @@
// This test sets up a call transfer to a new callee with a different DTLS
// fingerprint.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsTransferCaller) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestDtlsTransferCaller) {
MAYBE_SKIP_TEST(rtc::SSLStreamAdapter::HaveDtlsSrtp);
SetupAndVerifyDtlsCall();
@@ -1263,7 +1255,7 @@
// This test sets up a call between two endpoints that are configured to use
// DTLS key agreement. The offerer don't support SDES. As a result, DTLS is
// negotiated and used for transport.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestOfferDtlsButNotSdes) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestOfferDtlsButNotSdes) {
MAYBE_SKIP_TEST(rtc::SSLStreamAdapter::HaveDtlsSrtp);
FakeConstraints setup_constraints;
setup_constraints.AddMandatory(MediaConstraintsInterface::kEnableDtlsSrtp,
@@ -1276,7 +1268,7 @@
// This test sets up a Jsep call between two parties, and the callee only
// accept to receive video.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerVideo) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerVideo) {
ASSERT_TRUE(CreateTestClients());
receiving_client()->SetReceiveAudioVideo(false, true);
LocalP2PTest();
@@ -1284,7 +1276,7 @@
// This test sets up a Jsep call between two parties, and the callee only
// accept to receive audio.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerAudio) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerAudio) {
ASSERT_TRUE(CreateTestClients());
receiving_client()->SetReceiveAudioVideo(true, false);
LocalP2PTest();
@@ -1292,7 +1284,7 @@
// This test sets up a Jsep call between two parties, and the callee reject both
// audio and video.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerNone) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestAnswerNone) {
ASSERT_TRUE(CreateTestClients());
receiving_client()->SetReceiveAudioVideo(false, false);
LocalP2PTest();
@@ -1303,7 +1295,7 @@
// being rejected. Once the re-negotiation is done, the video flow should stop
// and the audio flow should continue.
// Disabled due to b/14955157.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient,
+TEST_F(JsepPeerConnectionP2PTestClient,
DISABLED_UpdateOfferWithRejectedContent) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
@@ -1313,7 +1305,7 @@
// This test sets up a Jsep call between two parties. The MSID is removed from
// the SDP strings from the caller.
// Disabled due to b/14955157.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient,
+TEST_F(JsepPeerConnectionP2PTestClient,
DISABLED_LocalP2PTestWithoutMsid) {
ASSERT_TRUE(CreateTestClients());
receiving_client()->RemoveMsidFromReceivedSdp(true);
@@ -1329,7 +1321,7 @@
// sends two steams.
// TODO(perkj): Disabled due to
// https://code.google.com/p/webrtc/issues/detail?id=1454
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTestTwoStreams) {
+TEST_F(JsepPeerConnectionP2PTestClient, DISABLED_LocalP2PTestTwoStreams) {
ASSERT_TRUE(CreateTestClients());
// Set optional video constraint to max 320pixels to decrease CPU usage.
FakeConstraints constraint;
@@ -1343,7 +1335,7 @@
}
// Test that we can receive the audio output level from a remote audio track.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetAudioOutputLevelStats) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetAudioOutputLevelStats) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
@@ -1362,7 +1354,7 @@
}
// Test that an audio input level is reported.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetAudioInputLevelStats) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetAudioInputLevelStats) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
@@ -1373,7 +1365,7 @@
}
// Test that we can get incoming byte counts from both audio and video tracks.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetBytesReceivedStats) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetBytesReceivedStats) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
@@ -1395,7 +1387,7 @@
}
// Test that we can get outgoing byte counts from both audio and video tracks.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetBytesSentStats) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetBytesSentStats) {
ASSERT_TRUE(CreateTestClients());
LocalP2PTest();
@@ -1417,7 +1409,7 @@
}
// Test that DTLS 1.0 is used if both sides only support DTLS 1.0.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetDtls12None) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetDtls12None) {
PeerConnectionFactory::Options init_options;
init_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_10;
PeerConnectionFactory::Options recv_options;
@@ -1448,7 +1440,7 @@
}
// Test that DTLS 1.2 is used if both ends support it.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetDtls12Both) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetDtls12Both) {
PeerConnectionFactory::Options init_options;
init_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_12;
PeerConnectionFactory::Options recv_options;
@@ -1480,7 +1472,7 @@
// Test that DTLS 1.0 is used if the initator supports DTLS 1.2 and the
// received supports 1.0.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetDtls12Init) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetDtls12Init) {
PeerConnectionFactory::Options init_options;
init_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_12;
PeerConnectionFactory::Options recv_options;
@@ -1512,7 +1504,7 @@
// Test that DTLS 1.0 is used if the initator supports DTLS 1.0 and the
// received supports 1.2.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, GetDtls12Recv) {
+TEST_F(JsepPeerConnectionP2PTestClient, GetDtls12Recv) {
PeerConnectionFactory::Options init_options;
init_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_10;
PeerConnectionFactory::Options recv_options;
@@ -1544,7 +1536,7 @@
// This test sets up a call between two parties with audio, video and an RTP
// data channel.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestRtpDataChannel) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestRtpDataChannel) {
FakeConstraints setup_constraints;
setup_constraints.SetAllowRtpDataChannels();
ASSERT_TRUE(CreateTestClients(&setup_constraints, &setup_constraints));
@@ -1576,7 +1568,7 @@
// This test sets up a call between two parties with audio, video and an SCTP
// data channel.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestSctpDataChannel) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestSctpDataChannel) {
ASSERT_TRUE(CreateTestClients());
initializing_client()->CreateDataChannel();
LocalP2PTest();
@@ -1610,7 +1602,7 @@
// transport has detected that a channel is writable and thus data can be
// received before the data channel state changes to open. That is hard to test
// but the same buffering is used in that case.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, RegisterDataChannelObserver) {
+TEST_F(JsepPeerConnectionP2PTestClient, RegisterDataChannelObserver) {
FakeConstraints setup_constraints;
setup_constraints.SetAllowRtpDataChannels();
ASSERT_TRUE(CreateTestClients(&setup_constraints, &setup_constraints));
@@ -1640,7 +1632,7 @@
// This test sets up a call between two parties with audio, video and but only
// the initiating client support data.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient,
+TEST_F(JsepPeerConnectionP2PTestClient,
LocalP2PTestReceiverDoesntSupportData) {
FakeConstraints setup_constraints_1;
setup_constraints_1.SetAllowRtpDataChannels();
@@ -1660,7 +1652,7 @@
// This test sets up a call between two parties with audio, video. When audio
// and video is setup and flowing and data channel is negotiated.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient,
+TEST_F(JsepPeerConnectionP2PTestClient,
AddDataChannelAfterRenegotiation) {
FakeConstraints setup_constraints;
setup_constraints.SetAllowRtpDataChannels();
@@ -1680,7 +1672,7 @@
// This test sets up a Jsep call with SCTP DataChannel and verifies the
// negotiation is completed without error.
#ifdef HAVE_SCTP
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, CreateOfferWithSctpDataChannel) {
+TEST_F(JsepPeerConnectionP2PTestClient, CreateOfferWithSctpDataChannel) {
MAYBE_SKIP_TEST(rtc::SSLStreamAdapter::HaveDtlsSrtp);
FakeConstraints constraints;
constraints.SetMandatory(
@@ -1694,7 +1686,7 @@
// This test sets up a call between two parties with audio, and video.
// During the call, the initializing side restart ice and the test verifies that
// new ice candidates are generated and audio and video still can flow.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, IceRestart) {
+TEST_F(JsepPeerConnectionP2PTestClient, IceRestart) {
ASSERT_TRUE(CreateTestClients());
// Negotiate and wait for ice completion and make sure audio and video plays.
@@ -1744,7 +1736,7 @@
// This test sets up a call between two parties with audio, and video.
// It then renegotiates setting the video m-line to "port 0", then later
// renegotiates again, enabling video.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, LocalP2PTestVideoDisableEnable) {
+TEST_F(JsepPeerConnectionP2PTestClient, LocalP2PTestVideoDisableEnable) {
ASSERT_TRUE(CreateTestClients());
// Do initial negotiation. Will result in video and audio sendonly m-lines.
@@ -1768,7 +1760,7 @@
// VideoDecoderFactory.
// TODO(holmer): Disabled due to sometimes crashing on buildbots.
// See issue webrtc/2378.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient,
+TEST_F(JsepPeerConnectionP2PTestClient,
DISABLED_LocalP2PTestWithVideoDecoderFactory) {
ASSERT_TRUE(CreateTestClients());
EnableVideoDecoderFactory();
@@ -1778,7 +1770,7 @@
// This tests that if we negotiate after calling CreateSender but before we
// have a track, then set a track later, frames from the newly-set track are
// received end-to-end.
-TEST_F(MAYBE_JsepPeerConnectionP2PTestClient, EarlyWarmupTest) {
+TEST_F(JsepPeerConnectionP2PTestClient, EarlyWarmupTest) {
ASSERT_TRUE(CreateTestClients());
auto audio_sender = initializing_client()->pc()->CreateSender("audio");
auto video_sender = initializing_client()->pc()->CreateSender("video");
diff --git a/third_party/gtest-parallel/README.webrtc b/third_party/gtest-parallel/README.webrtc
index acea04f..eafdf66 100644
--- a/third_party/gtest-parallel/README.webrtc
+++ b/third_party/gtest-parallel/README.webrtc
@@ -1,5 +1,5 @@
URL: https://github.com/google/gtest-parallel
-Version: c0f8ded77566c657ccc7f745fd9cb070750cccf8
+Version: 34ae4d7ed77fe2ac22d5b444a8468c9d847ed9d2
License: Apache 2.0
License File: LICENSE
diff --git a/third_party/gtest-parallel/gtest-parallel b/third_party/gtest-parallel/gtest-parallel
index 0be59e4..64fbb1c 100755
--- a/third_party/gtest-parallel/gtest-parallel
+++ b/third_party/gtest-parallel/gtest-parallel
@@ -13,12 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import cPickle
+import errno
import gzip
import multiprocessing
import optparse
import os
import subprocess
import sys
+import tempfile
import threading
import time
import zlib
@@ -62,6 +64,10 @@
stdout_lock = threading.Lock()
class FilterFormat:
+ if sys.stdout.isatty():
+ # stdout needs to be unbuffered since the output is interactive.
+ sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
out = Outputter(sys.stdout)
total_tests = 0
finished_tests = 0
@@ -80,7 +86,6 @@
if command == "TEST":
(binary, test) = arg.split(' ', 1)
self.tests[job_id] = (binary, test.strip())
- self.outputs[job_id] = []
elif command == "EXIT":
(exit_code, time_ms) = [int(x) for x in arg.split(' ', 1)]
self.finished_tests += 1
@@ -88,8 +93,9 @@
self.print_test_status(test, time_ms)
if exit_code != 0:
self.failures.append(self.tests[job_id])
- for line in self.outputs[job_id]:
- self.out.permanent_line(line)
+ with open(self.outputs[job_id]) as f:
+ for line in f.readlines():
+ self.out.permanent_line(line.rstrip())
self.out.permanent_line(
"[%d/%d] %s returned/aborted with exit code %d (%d ms)"
% (self.finished_tests, self.total_tests, test, exit_code, time_ms))
@@ -97,17 +103,15 @@
self.total_tests = int(arg.split(' ', 1)[1])
self.out.transient_line("[0/%d] Running tests..." % self.total_tests)
- def add_stdout(self, job_id, output):
- self.outputs[job_id].append(output)
+ def logfile(self, job_id, name):
+ self.outputs[job_id] = name
def log(self, line):
stdout_lock.acquire()
(prefix, output) = line.split(' ', 1)
- if prefix[-1] == ':':
- self.handle_meta(int(prefix[:-1]), output)
- else:
- self.add_stdout(int(prefix[:-1]), output)
+ assert prefix[-1] == ':'
+ self.handle_meta(int(prefix[:-1]), output)
stdout_lock.release()
def end(self):
@@ -123,6 +127,10 @@
sys.stdout.write(line + "\n")
sys.stdout.flush()
stdout_lock.release()
+ def logfile(self, job_id, name):
+ with open(self.outputs[job_id]) as f:
+ for line in f.readlines():
+ self.log(str(job_id) + '> ' + line.rstrip())
def end(self):
pass
@@ -184,6 +192,9 @@
parser = optparse.OptionParser(
usage = 'usage: %prog [options] binary [binary ...] -- [additional args]')
+parser.add_option('-d', '--output_dir', type='string',
+ default=os.path.join(tempfile.gettempdir(), "gtest-parallel"),
+ help='output directory for test logs')
parser.add_option('-r', '--repeat', type='int', default=1,
help='repeat tests')
parser.add_option('-w', '--workers', type='int',
@@ -260,25 +271,33 @@
exit_code = 0
+# Create directory for test log output.
+try:
+ os.makedirs(options.output_dir)
+except OSError as e:
+ # Ignore errors if this directory already exists.
+ if e.errno != errno.EEXIST or not os.path.isdir(options.output_dir):
+ raise e
+# Remove files from old test runs.
+for logfile in os.listdir(options.output_dir):
+ os.remove(os.path.join(options.output_dir, logfile))
+
# Run the specified job. Return the elapsed time in milliseconds if
# the job succeeds, or a very large number (larger than any reasonable
# elapsed time) if the job fails. (This ensures that failing tests
# will run first the next time.)
def run_job((command, job_id, test)):
begin = time.time()
- sub = subprocess.Popen(command + ['--gtest_filter=' + test] +
- ['--gtest_color=' + options.gtest_color],
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT)
- while True:
- line = sub.stdout.readline()
- if line == '':
- break
- logger.log(str(job_id) + '> ' + line.rstrip())
+ with tempfile.NamedTemporaryFile(dir=options.output_dir, delete=False) as log:
+ sub = subprocess.Popen(command + ['--gtest_filter=' + test] +
+ ['--gtest_color=' + options.gtest_color],
+ stdout=log.file,
+ stderr=log.file)
+ code = sub.wait()
+ runtime_ms = int(1000 * (time.time() - begin))
+ logger.logfile(job_id, log.name)
- code = sub.wait()
- runtime_ms = int(1000 * (time.time() - begin))
logger.log("%s: EXIT %s %d" % (job_id, code, runtime_ms))
if code == 0:
return runtime_ms
