ThreadSanitizer (TSAN) and AddressSanitizer (ASAN) bots
BUG=None
TEST=Tested on local master with the live slaves.
Review URL: https://webrtc-codereview.appspot.com/571011
git-svn-id: http://webrtc.googlecode.com/svn/trunk@2258 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/tools/.gitignore b/tools/.gitignore
index 252d08b..348517b 100644
--- a/tools/.gitignore
+++ b/tools/.gitignore
@@ -38,14 +38,16 @@
/continuous_build/build_internal/masters/master.webrtc/android
/continuous_build/build_internal/masters/master.webrtc/android-ndk
/continuous_build/build_internal/masters/master.webrtc/chromeos
+/continuous_build/build_internal/masters/master.webrtc/linux-asan
/continuous_build/build_internal/masters/master.webrtc/linux-clang
/continuous_build/build_internal/masters/master.webrtc/linux-large-tests
+/continuous_build/build_internal/masters/master.webrtc/linux-memcheck
/continuous_build/build_internal/masters/master.webrtc/linux-slave-1
/continuous_build/build_internal/masters/master.webrtc/linux-slave-2
/continuous_build/build_internal/masters/master.webrtc/linux-slave-4
/continuous_build/build_internal/masters/master.webrtc/linux-slave-5
/continuous_build/build_internal/masters/master.webrtc/linux-slave-gcc-4.6
-/continuous_build/build_internal/masters/master.webrtc/linux-valgrind
+/continuous_build/build_internal/masters/master.webrtc/linux-tsan
/continuous_build/build_internal/masters/master.webrtc/LinuxChrome
/continuous_build/build_internal/masters/master.webrtc/LinuxChromeBloat
/continuous_build/build_internal/masters/master.webrtc/mac-large-tests/
diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg
index 870c1c9..2c4ace1 100755
--- a/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg
+++ b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg
@@ -47,7 +47,9 @@
'Linux64DBG',
'Linux64Release',
'LinuxClang',
- 'LinuxValgrind',
+ 'LinuxMemcheck',
+ 'LinuxTsan',
+ 'LinuxAsan',
'Linux64DBG-GCC4.6',
'LinuxLargeTests',
'MacLargeTests',
@@ -114,10 +116,22 @@
'voe_auto_test': (True, True, True),
}
-VALGRIND_DISABLED_TESTS = [
+MEMCHECK_DISABLED_TESTS = [
'audio_coding_module_test', # Issue 270
'test_fec', # Too slow for Valgrind memcheck
]
+TSAN_DISABLED_TESTS = [
+ 'audio_coding_module_test', # Issue 283
+ 'audioproc_unittest', # Issue 299
+ 'system_wrappers_unittests', # Issue 300
+ 'udp_transport_unittests', # Issue 536
+ 'video_processing_unittests', # Issue 303
+ 'test_fec', # Too slow for TSAN
+]
+ASAN_DISABLED_TESTS = [
+ 'audio_coding_module_test', # Issue 281
+ 'neteq_unittests', # Issue 282
+]
linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Linux')
mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Mac')
@@ -144,6 +158,10 @@
('trunk/third_party/valgrind',
'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
]
+ASAN_DEPS_LIST = [
+ ('trunk/third_party/asan',
+ 'http://src.chromium.org/chrome/trunk/deps/third_party/asan'),
+]
# Linux
linux_factory_64_dbg = utils.WebRTCLinuxFactory(
@@ -188,14 +206,33 @@
linux_clang.EnableBuild(clang=True)
linux_clang.EnableTests(linux_normal_tests)
-linux_valgrind = utils.WebRTCLinuxFactory(
- utils.BuildStatusOracle('linux_valgrind'), run_with_memcheck=True,
+linux_memcheck_factory = utils.WebRTCLinuxFactory(
+ utils.BuildStatusOracle('linux_memcheck_factory'), run_with_memcheck=True,
custom_deps_list=VALGRIND_DEPS_LIST)
-linux_valgrind.EnableBuild(release=True, compile_for_memory_tooling=True)
-# Filter out disabled Valgrind tests:
-valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS,
+linux_memcheck_factory.EnableBuild(release=True,
+ compile_for_memory_tooling=True)
+# Filter out disabled memcheck tests:
+memcheck_tests = filter(lambda test: test not in MEMCHECK_DISABLED_TESTS,
linux_normal_tests)
-linux_valgrind.EnableTests(valgrind_tests)
+linux_memcheck_factory.EnableTests(memcheck_tests)
+
+linux_tsan_factory = utils.WebRTCLinuxFactory(
+ utils.BuildStatusOracle('linux_tsan_factory'), run_with_tsan=True,
+ custom_deps_list=VALGRIND_DEPS_LIST)
+linux_tsan_factory.EnableBuild(release=True, compile_for_memory_tooling=True)
+# Filter out disabled tsan tests:
+tsan_tests = filter(lambda test: test not in TSAN_DISABLED_TESTS,
+ linux_normal_tests)
+linux_tsan_factory.EnableTests(tsan_tests)
+
+linux_asan_factory = utils.WebRTCLinuxFactory(
+ utils.BuildStatusOracle('linux_asan_factory'), run_with_asan=True,
+ custom_deps_list=ASAN_DEPS_LIST)
+linux_asan_factory.EnableBuild(release=True, clang=True)
+# Filter out disabled asan tests:
+asan_tests = filter(lambda test: test not in ASAN_DISABLED_TESTS,
+ linux_normal_tests)
+linux_asan_factory.EnableTests(asan_tests)
android_factory = utils.WebRTCAndroidFactory(
utils.BuildStatusOracle('android_factory'))
@@ -316,11 +353,23 @@
'builddir': 'linux-clang',
'factory': linux_clang,
}
-linux_builder_valgrind = {
- 'name': 'LinuxValgrind',
+linux_builder_memcheck = {
+ 'name': 'LinuxMemcheck',
'slavename': 'webrtc-cb-linux-slave-6',
- 'builddir': 'linux-valgrind',
- 'factory': linux_valgrind,
+ 'builddir': 'linux-memcheck',
+ 'factory': linux_memcheck_factory,
+ }
+linux_builder_tsan = {
+ 'name': 'LinuxTsan',
+ 'slavename': 'webrtc-cb-linux-slave-15',
+ 'builddir': 'linux-tsan',
+ 'factory': linux_tsan_factory,
+ }
+linux_builder_asan = {
+ 'name': 'LinuxAsan',
+ 'slavename': 'webrtc-cb-linux-slave-16',
+ 'builddir': 'linux-asan',
+ 'factory': linux_asan_factory,
}
android_builder_1 = {
'name': 'Android',
@@ -350,7 +399,9 @@
linux_builder_64_release,
linux_builder_64_debug,
linux_builder_clang,
- linux_builder_valgrind,
+ linux_builder_memcheck,
+ linux_builder_tsan,
+ linux_builder_asan,
linux_builder_gcc_4_6,
win_builder_large_tests,
mac_builder_large_tests,
diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg b/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg
index f11be7e..8839d71 100644
--- a/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg
+++ b/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg
@@ -55,7 +55,7 @@
'os': 'linux',
'version': 'lucid',
'bits': '64',
- 'builder': 'LinuxValgrind',
+ 'builder': 'LinuxMemcheck',
'hostname': 'webrtc-cb-linux-slave-6',
},
{
@@ -95,6 +95,22 @@
'os': 'linux',
'version': 'lucid',
'bits': '64',
+ 'builder': 'LinuxTsan',
+ 'hostname': 'webrtc-cb-linux-slave-15',
+ },
+ {
+ 'master': 'WebRTC',
+ 'os': 'linux',
+ 'version': 'lucid',
+ 'bits': '64',
+ 'builder': 'LinuxAsan',
+ 'hostname': 'webrtc-cb-linux-slave-16',
+ },
+ {
+ 'master': 'WebRTC',
+ 'os': 'linux',
+ 'version': 'lucid',
+ 'bits': '64',
'builder': 'LinuxLargeTests',
'hostname': 'webrtc-build-bot-se',
},
diff --git a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py
index f59c002..73f264c 100755
--- a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py
+++ b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py
@@ -33,6 +33,8 @@
WEBRTC_BUILD_DIR = 'build'
MEMCHECK_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t']
+TSAN_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '--tool', 'tsan', '-t']
+ASAN_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '--tool', 'asan', '-t']
DEFAULT_COVERAGE_DIR = '/var/www/coverage'
DEFAULT_MASTER_WORK_DIR = '.'
@@ -42,8 +44,8 @@
# but converted to a list since we set defines instead of using an environment
# variable.
#
-# On valgrind bots, override the optimizer settings so we don't inline too
-# much and make the stacks harder to figure out. Use the same settings
+# On memcheck and tsan bots, override the optimizer settings so we don't inline
+# too much and make the stacks harder to figure out. Use the same settings
# on all buildbot masters to make it easier to move bots.
MEMORY_TOOLS_GYP_DEFINES = [
# GCC flags
@@ -131,7 +133,7 @@
def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
halt_build_on_failure=True, warn_on_failure=False,
- timeout=1200, use_pty=True):
+ timeout=1200, use_pty=True, env={}):
"""Adds a step which will run as a shell command on the slave.
NOTE: you are recommended to use this method to add new shell commands
@@ -163,6 +165,8 @@
stderr output with red in the web interface. Some shell
commands seem to fail when Pseudo-terminal is enabled on
Linux.
+ env: dict of string->string that describes the environment the command
+ shall be excuted with on the build slave.
"""
flunk_on_failure = not warn_on_failure
@@ -183,7 +187,8 @@
haltOnFailure=halt_build_on_failure,
name='_'.join(descriptor),
timeout=timeout,
- usePTY=use_pty))
+ usePTY=use_pty,
+ env=env))
def AddSmartCleanStep(self):
"""Adds a smart clean step.
@@ -519,13 +524,16 @@
"""
def __init__(self, build_status_oracle, is_try_slave=False,
- run_with_memcheck=False, custom_deps_list=None):
+ run_with_memcheck=False, run_with_tsan=False,
+ run_with_asan=False, custom_deps_list=None):
WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
is_try_slave=is_try_slave,
custom_deps_list=custom_deps_list)
self.build_enabled = False
self.coverage_enabled = False
self.run_with_memcheck = run_with_memcheck
+ self.run_with_tsan = run_with_tsan
+ self.run_with_asan = run_with_asan
self.compile_for_memory_tooling = False
def EnableCoverage(self, coverage_url, coverage_dir=DEFAULT_COVERAGE_DIR):
@@ -555,15 +563,23 @@
self.release = release
self.AddSmartCleanStep()
+ self.AddGclientSyncStep()
# Valgrind bots need special GYP defines to enable memory profiling
- # friendly compilation. They already has a custom .gclient configuration
- # file created so they don't need one being generated like the other bots.
+ # friendly compilation.
if self.compile_for_memory_tooling:
for gyp_define in MEMORY_TOOLS_GYP_DEFINES:
self.gyp_params.append('-D' + gyp_define)
- self.AddGclientSyncStep()
+ if self.run_with_asan:
+ # ASAN requires Clang compilation and we enforce Release mode since
+ # that's what Chromium recommends.
+ assert clang
+ assert release
+ self.gyp_params.append('-Dasan=1')
+ self.gyp_params.append('-Dlinux_use_tcmalloc=0')
+ self.gyp_params.append('-Drelease_extra_cflags="-g -O1 '
+ '-fno-inline-functions -fno-inline"')
if chrome_os:
self.gyp_params.append('-Dchromeos=1')
@@ -591,6 +607,10 @@
cmd = ['out/%s/%s' % (test_folder, test)]
if self.run_with_memcheck:
cmd = MEMCHECK_CMD + cmd
+ if self.run_with_tsan:
+ cmd = TSAN_CMD + cmd
+ if self.run_with_asan:
+ cmd = ASAN_CMD + cmd
self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False)
def AddXvfbTestRunStep(self, test_name, test_binary, test_arguments=''):
@@ -604,7 +624,18 @@
cmd = ['make', target, '-j100']
if make_extra:
cmd.append(make_extra)
- self.AddCommonStep(cmd=cmd, descriptor=descriptor)
+
+ env = {}
+ if self.run_with_asan:
+ # Override the Clang compiler with the prebuilt ASAN executables.
+ # It seems like CC and CXX must contain full paths and since there's no
+ # way to evaluate subcommands on the slaves, this hardcoding is the only
+ # way for now.
+ asan_bin = ('/b/build/slave/linux-asan/build/trunk/third_party/asan/'
+ 'asan_clang_Linux/bin')
+ env = {'CC': '%s/clang' % asan_bin,
+ 'CXX': '%s/clang++ ' % asan_bin}
+ self.AddCommonStep(cmd=cmd, descriptor=descriptor, env=env)
def AddStepsToEstablishCoverageBaseline(self):
self.AddCommonStep(['lcov', '--directory', '.', '--capture', '-b',
