Remove third_party from DEPS file to prepare to check it into webrtc.
Remove third_party from DEPS and modify autoroller script to check
chromium third_party directly into webrtc repo.
Change-Id: Ib0b77fc414116babc193b2289a5e9c3256daf566
No-Presubmit: True
Bug: webrtc:8366
Reviewed-on: https://webrtc-review.googlesource.com/73801
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@google.com>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23204}diff --git a/.gitignore b/.gitignore
index bf93736..ecd89ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,7 +45,7 @@
/mojo
/out
/testing
-/third_party
+/third_party_chromium
/tools
/tools_webrtc/android/profiling/flamegraph
/tools_webrtc/android/profiling/simpleperf
diff --git a/DEPS b/DEPS
index dbc38b7..8810ec2 100644
--- a/DEPS
+++ b/DEPS
@@ -2,6 +2,9 @@
vars = {
'chromium_git': 'https://chromium.googlesource.com',
+ # Used by the WebRTC DEPS autoroller to update third_party/. If you need to run autoroller localy,
+ # you can set it via custom_vars section in the .gclient file.
+ 'roll_chromium_into_webrtc': False,
# By default, we should check out everything needed to run on the main
# chromium waterfalls. More info at: crbug.com/570091.
'checkout_configuration': 'default',
@@ -37,6 +40,10 @@
# the commit queue can handle CLs rolling HarfBuzz
# and whatever else without interference from each other.
'harfbuzz_revision': '957e7756634a4fdf1654041e20e883cf964ecac9',
+ # Three lines of non-changing comments so that
+ # the commit queue can handle CLs rolling Chromium third_party
+ # and whatever else without interference from each other.
+ 'chromium_third_party_revision': '4e16929f465a47942875a80da0140bfaa59e99fb',
}
deps = {
# TODO(kjellander): Move this to be Android-only once the libevent dependency
@@ -59,8 +66,11 @@
},
'src/testing':
Var('chromium_git') + '/chromium/src/testing' + '@' + '519bd6bd8883f17137857e86cc73491d39415057',
- 'src/third_party':
- Var('chromium_git') + '/chromium/src/third_party' + '@' + '332e6754f377ca77a48eeeb139ee8d1c2f3ca739',
+ # This entry is used for chromium third_party rolling into webrtc third_party only.
+ 'src/third_party_chromium': {
+ 'url': Var('chromium_git') + '/chromium/src/third_party' + '@' + Var('chromium_third_party_revision'),
+ 'condition': 'roll_chromium_into_webrtc',
+ },
'src/third_party/android_ndk': {
'url': Var('chromium_git') + '/android_ndk.git' + '@' + '5cd86312e794bdf542a3685c6f10cbb96072990b',
'condition': 'checkout_android',
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
index 5a40612..abb8582 100755
--- a/PRESUBMIT.py
+++ b/PRESUBMIT.py
@@ -40,6 +40,7 @@
'test',
'tools_webrtc',
'voice_engine',
+ 'third_party',
]
# These filters will always be removed, even if the caller specifies a filter
@@ -180,12 +181,15 @@
return []
-def CheckNoIOStreamInHeaders(input_api, output_api):
+def CheckNoIOStreamInHeaders(input_api, output_api,
+ source_file_filter):
"""Checks to make sure no .h files include <iostream>."""
files = []
pattern = input_api.re.compile(r'^#include\s*<iostream>',
input_api.re.MULTILINE)
- for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
+ file_filter = lambda x: (input_api.FilterSourceFile(x)
+ and source_file_filter(x))
+ for f in input_api.AffectedSourceFiles(file_filter):
if not f.LocalPath().endswith('.h'):
continue
contents = input_api.ReadFile(f)
@@ -201,12 +205,15 @@
return []
-def CheckNoPragmaOnce(input_api, output_api):
+def CheckNoPragmaOnce(input_api, output_api,
+ source_file_filter):
"""Make sure that banned functions are not used."""
files = []
pattern = input_api.re.compile(r'^#pragma\s+once',
input_api.re.MULTILINE)
- for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
+ file_filter = lambda x: (input_api.FilterSourceFile(x)
+ and source_file_filter(x))
+ for f in input_api.AffectedSourceFiles(file_filter):
if not f.LocalPath().endswith('.h'):
continue
contents = input_api.ReadFile(f)
@@ -221,13 +228,15 @@
return []
-def CheckNoFRIEND_TEST(input_api, output_api): # pylint: disable=invalid-name
+def CheckNoFRIEND_TEST(input_api, output_api, # pylint: disable=invalid-name
+ source_file_filter):
"""Make sure that gtest's FRIEND_TEST() macro is not used, the
FRIEND_TEST_ALL_PREFIXES() macro from testsupport/gtest_prod_util.h should be
used instead since that allows for FLAKY_, FAILS_ and DISABLED_ prefixes."""
problems = []
- file_filter = lambda f: f.LocalPath().endswith(('.cc', '.h'))
+ file_filter = lambda f: (f.LocalPath().endswith(('.cc', '.h'))
+ and source_file_filter(f))
for f in input_api.AffectedFiles(file_filter=file_filter):
for line_num, line in f.ChangedContents():
if 'FRIEND_TEST(' in line:
@@ -249,7 +258,7 @@
def CheckApprovedFilesLintClean(input_api, output_api,
- source_file_filter=None):
+ source_file_filter=None):
"""Checks that all new or non-blacklisted .cc and .h files pass cpplint.py.
This check is based on CheckChangeLintsClean in
depot_tools/presubmit_canned_checks.py but has less filters and only checks
@@ -408,7 +417,8 @@
def CheckNoStreamUsageIsAdded(input_api, output_api,
- error_formatter=_ReportErrorFileAndLineNumber):
+ error_formatter=_ReportErrorFileAndLineNumber,
+ source_file_filter):
"""Make sure that no more dependencies on stringstream are added."""
error_msg = ('Usage of <sstream>, <istream> and <ostream> in WebRTC is '
'deprecated.\n'
@@ -433,7 +443,9 @@
usage_re = input_api.re.compile(r'std::(w|i|o|io|wi|wo|wio)(string)*stream')
no_presubmit_re = input_api.re.compile(
r'// no-presubmit-check TODO\(webrtc:8982\)')
- for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
+ file_filter = lambda x: (input_api.FilterSourceFile(x)
+ and source_file_filter(x))
+ for f in input_api.AffectedSourceFiles(file_filter):
if f.LocalPath() == 'PRESUBMIT.py':
continue
for line_num, line in f.ChangedContents():
@@ -482,13 +494,14 @@
line_number)))
return result
-def CheckGnChanges(input_api, output_api):
- source_file_filter = lambda x: input_api.FilterSourceFile(
+def CheckGnChanges(input_api, output_api, source_file_filter):
+ file_filter = lambda x: (input_api.FilterSourceFile(
x, white_list=(r'.+\.(gn|gni)$',),
black_list=(r'.*/presubmit_checks_lib/testdata/.*',))
+ and source_file_filter(x))
gn_files = []
- for f in input_api.AffectedSourceFiles(source_file_filter):
+ for f in input_api.AffectedSourceFiles(file_filter):
gn_files.append(f)
result = []
@@ -517,7 +530,7 @@
long_text='\n\n'.join(errors))]
return []
-def CheckUnwantedDependencies(input_api, output_api):
+def CheckUnwantedDependencies(input_api, output_api, source_file_filter):
"""Runs checkdeps on #include statements added in this
change. Breaking - rules is an error, breaking ! rules is a
warning.
@@ -539,7 +552,7 @@
from rules import Rule
added_includes = []
- for f in input_api.AffectedFiles():
+ for f in input_api.AffectedFiles(file_filter=source_file_filter):
if not CppChecker.IsCppFile(f.LocalPath()):
continue
@@ -620,11 +633,12 @@
' * https://bugs.webrtc.org - reference it using Bug: webrtc:XXXX\n'
' * https://crbug.com - reference it using Bug: chromium:XXXXXX')]
-def CheckJSONParseErrors(input_api, output_api):
+def CheckJSONParseErrors(input_api, output_api, source_file_filter):
"""Check that JSON files do not contain syntax errors."""
def FilterFile(affected_file):
- return input_api.os_path.splitext(affected_file.LocalPath())[1] == '.json'
+ return (input_api.os_path.splitext(affected_file.LocalPath())[1] == '.json'
+ and source_file_filter(affected_file))
def GetJSONParseError(input_api, filename):
try:
@@ -670,12 +684,15 @@
return input_api.RunTests(tests, parallel=True)
-def CheckUsageOfGoogleProtobufNamespace(input_api, output_api):
+def CheckUsageOfGoogleProtobufNamespace(input_api, output_api,
+ source_file_filter):
"""Checks that the namespace google::protobuf has not been used."""
files = []
pattern = input_api.re.compile(r'google::protobuf')
proto_utils_path = os.path.join('rtc_base', 'protobuf_utils.h')
- for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
+ file_filter = lambda x: (input_api.FilterSourceFile(x)
+ and source_file_filter(x))
+ for f in input_api.AffectedSourceFiles(file_filter):
if f.LocalPath() in [proto_utils_path, 'PRESUBMIT.py']:
continue
contents = input_api.ReadFile(f)
@@ -752,8 +769,11 @@
objc_filter_list = (r'.+\.m$', r'.+\.mm$', r'.+objc\/.+\.h$')
# Skip long-lines check for DEPS and GN files.
build_file_filter_list = (r'.+\.gn$', r'.+\.gni$', 'DEPS')
+ # Also we will skip most checks for third_party directory.
+ third_party_filter_list = (r'^third_party[\\\/].+',)
eighty_char_sources = lambda x: input_api.FilterSourceFile(x,
- black_list=build_file_filter_list + objc_filter_list)
+ black_list=build_file_filter_list + objc_filter_list +
+ third_party_filter_list)
hundred_char_sources = lambda x: input_api.FilterSourceFile(x,
white_list=objc_filter_list)
results.extend(input_api.canned_checks.CheckLongLines(
@@ -762,26 +782,38 @@
input_api, output_api, maxlen=100,
source_file_filter=hundred_char_sources))
+ non_third_party_sources = lambda x: input_api.FilterSourceFile(x,
+ black_list=third_party_filter_list)
results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
- input_api, output_api))
+ input_api, output_api, source_file_filter=non_third_party_sources))
results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
- input_api, output_api))
+ input_api, output_api, source_file_filter=non_third_party_sources))
results.extend(input_api.canned_checks.CheckAuthorizedAuthor(
input_api, output_api))
results.extend(input_api.canned_checks.CheckChangeTodoHasOwner(
- input_api, output_api))
+ input_api, output_api, source_file_filter=non_third_party_sources))
results.extend(CheckNativeApiHeaderChanges(input_api, output_api))
- results.extend(CheckNoIOStreamInHeaders(input_api, output_api))
- results.extend(CheckNoPragmaOnce(input_api, output_api))
- results.extend(CheckNoFRIEND_TEST(input_api, output_api))
- results.extend(CheckGnChanges(input_api, output_api))
- results.extend(CheckUnwantedDependencies(input_api, output_api))
- results.extend(CheckJSONParseErrors(input_api, output_api))
+ results.extend(CheckNoIOStreamInHeaders(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckNoPragmaOnce(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckNoFRIEND_TEST(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckGnChanges(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckUnwantedDependencies(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckJSONParseErrors(
+ input_api, output_api, source_file_filter=non_third_party_sources))
results.extend(RunPythonTests(input_api, output_api))
- results.extend(CheckUsageOfGoogleProtobufNamespace(input_api, output_api))
- results.extend(CheckOrphanHeaders(input_api, output_api))
- results.extend(CheckNewlineAtTheEndOfProtoFiles(input_api, output_api))
- results.extend(CheckNoStreamUsageIsAdded(input_api, output_api))
+ results.extend(CheckUsageOfGoogleProtobufNamespace(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckOrphanHeaders(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckNewlineAtTheEndOfProtoFiles(
+ input_api, output_api, source_file_filter=non_third_party_sources))
+ results.extend(CheckNoStreamUsageIsAdded(
+ input_api, output_api, source_file_filter=non_third_party_sources))
return results
@@ -811,7 +843,7 @@
return results
-def CheckOrphanHeaders(input_api, output_api):
+def CheckOrphanHeaders(input_api, output_api, source_file_filter):
# We need to wait until we have an input_api object and use this
# roundabout construct to import prebubmit_checks_lib because this file is
# eval-ed and thus doesn't have __file__.
@@ -825,9 +857,9 @@
from check_orphan_headers import GetBuildGnPathFromFilePath
from check_orphan_headers import IsHeaderInBuildGn
- source_file_filter = lambda x: input_api.FilterSourceFile(
- x, black_list=orphan_blacklist)
- for f in input_api.AffectedSourceFiles(source_file_filter):
+ file_filter = lambda x: input_api.FilterSourceFile(
+ x, black_list=orphan_blacklist) and source_file_filter(x)
+ for f in input_api.AffectedSourceFiles(file_filter):
if f.LocalPath().endswith('.h'):
file_path = os.path.abspath(f.LocalPath())
root_dir = os.getcwd()
@@ -840,13 +872,14 @@
return results
-def CheckNewlineAtTheEndOfProtoFiles(input_api, output_api):
+def CheckNewlineAtTheEndOfProtoFiles(input_api, output_api,
+ source_file_filter):
"""Checks that all .proto files are terminated with a newline."""
error_msg = 'File {} must end with exactly one newline.'
results = []
- source_file_filter = lambda x: input_api.FilterSourceFile(
- x, white_list=(r'.+\.proto$',))
- for f in input_api.AffectedSourceFiles(source_file_filter):
+ file_filter = lambda x: input_api.FilterSourceFile(
+ x, white_list=(r'.+\.proto$',)) and source_file_filter(x)
+ for f in input_api.AffectedSourceFiles(file_filter):
file_path = f.LocalPath()
with open(file_path) as f:
lines = f.readlines()
diff --git a/THIRD_PARTY_DEPS b/THIRD_PARTY_DEPS
new file mode 100644
index 0000000..d09c978
--- /dev/null
+++ b/THIRD_PARTY_DEPS
@@ -0,0 +1,97 @@
+DEPS = [
+ # Common
+ 'binutils',
+ 'boringssl',
+ 'ced',
+ 'freetype',
+ 'googletest',
+ 'harfbuzz-ng',
+ 'instrumented_libraries',
+ 'jsoncpp',
+ 'libFuzzer',
+ 'libpng',
+ 'libvpx',
+ 'mockito',
+ 'openh264',
+ 'opus',
+ 'protobuf',
+ 'requests',
+ 'rnnoise',
+ 'usrsctp',
+ 'yasm',
+ 'zlib',
+ 'colorama',
+
+ # These common deps will be synced by gclient:
+ #'depot_tools',
+ #'ffmpeg',
+ #'icu',
+ #'libjpeg_turbo',
+ #'libsrtp',
+ #'libyuv',
+ #'llvm-build',
+ #'lss',
+ #'openmax_dl',
+ #'catapult',
+ #'gtest-parallel',
+
+ # Windows specific deps will be synced by gclient:
+ #'syzygy',
+ #'winsdk_samples',
+
+ # Android specific deps
+ # compile time deps
+ 'accessibility_test_framework',
+ 'android_platform',
+ 'android_support_test_runner',
+ 'apk-patch-size-estimator',
+ 'ashmem',
+ 'auto',
+ 'bazel',
+ 'bouncycastle',
+ 'breakpad',
+ 'byte_buddy',
+ 'closure_compiler',
+ 'errorprone',
+ 'espresso',
+ 'eu-strip',
+ 'gson',
+ 'guava',
+ 'hamcrest',
+ 'icu4j',
+ 'ijar',
+ 'intellij',
+ 'javax_inject',
+ 'jinja2',
+ 'jsr-305',
+ 'junit',
+ 'libxml',
+ 'markupsafe',
+ 'modp_b64',
+ 'objenesis',
+ 'ow2_asm',
+ 'robolectric',
+ 'sqlite4java',
+ 'tcmalloc',
+ 'ub-uiautomator',
+ 'xstream',
+ # test time deps
+ 'proguard',
+
+ 'android_system_sdk',
+ # These Android specific deps will be synced by gclient:
+ #'android_ndk',
+ #'android_tools',
+ #'findbugs',
+
+ # Mac and iOS specific deps
+ 'ocmock',
+
+ # List of files to sync
+ 'BUILD.gn',
+ 'DEPS',
+ 'libjpeg.gni',
+ 'PRESUBMIT.py',
+ 'README.chromium',
+ 'README.chromium.template',
+]
diff --git a/third_party/.gitignore b/third_party/.gitignore
new file mode 100644
index 0000000..566e8a3
--- /dev/null
+++ b/third_party/.gitignore
@@ -0,0 +1,64 @@
+# mirror in DEPS. Without it, a lot is wiped and re-downloaded for each sync.
+/findbugs/
+/gtest-parallel/
+/winsdk_samples/
+/accessibility_test_framework/lib/
+/android_ndk/
+/android_protobuf/src
+/android_support_test_runner/lib/
+/android_tools/
+/android_tools_internal/
+/apk-patch-size-estimator/lib/
+/auto/src
+/bazel/desugar/Desugar.jar
+/boringssl/src
+/bouncycastle/lib/
+/byte_buddy/lib/
+/catapult
+/ced/src
+/colorama/src
+/depot_tools
+/errorprone/lib
+/espresso/lib/
+/ffmpeg
+/freetype/src
+/gnu_binutils/
+/googletest/src
+/gson/lib/
+/guava/lib/
+/hamcrest/lib/
+/harfbuzz-ng/src
+/icu
+/icu4j/lib/
+/intellij/lib/
+/instrumented_libraries/scripts/*.tgz
+/instrumented_libraries/scripts/out/*
+/javax_inject/lib/
+/jsoncpp/source
+/jsr-305/src
+/junit/src
+/libFuzzer/src
+/libprotobuf-mutator/src
+/libjpeg_turbo
+/libsrtp
+/libvpx/source/libvpx
+/libyuv
+/llvm-build
+/llvm-build-tools
+/lss
+/mockito/src
+/objenesis/lib/
+/openmax_dl/
+/openh264/src
+/ow2_asm/lib/
+/requests/src
+/robolectric/lib/
+/robolectric/robolectric
+/sqlite4java/lib/
+/ub-uiautomator/lib
+/usrsctp/usrsctplib
+/xstream/lib/
+/yasm/binaries
+/yasm/generate_files.xml
+/yasm/source/patched-yasm
+/yasm/yasm.xml
diff --git a/third_party/OWNERS b/third_party/OWNERS
new file mode 100644
index 0000000..908fabb
--- /dev/null
+++ b/third_party/OWNERS
@@ -0,0 +1,7 @@
+phoglund@webrtc.org
+titovartem@webrtc.org
+buildbot@webrtc.org
+
+
+
+per-file .gitignore=*
diff --git a/third_party/binutils/.gitignore b/third_party/binutils/.gitignore
new file mode 100644
index 0000000..5605b2f
--- /dev/null
+++ b/third_party/binutils/.gitignore
@@ -0,0 +1,8 @@
+binutils-*
+*-chroot-*
+output-*
+Linux_ia32/*stamp*
+Linux_ia32/*tar.bz2
+Linux_x64/*stamp*
+Linux_x64/*tar.bz2
+*/Release
diff --git a/third_party/instrumented_libraries/binaries/.gitignore b/third_party/instrumented_libraries/binaries/.gitignore
new file mode 100644
index 0000000..d177d9a
--- /dev/null
+++ b/third_party/instrumented_libraries/binaries/.gitignore
@@ -0,0 +1,2 @@
+# Ignore downloaded binaries.
+*.tgz
diff --git a/third_party/opus/src/.gitignore b/third_party/opus/src/.gitignore
new file mode 100644
index 0000000..33127c9
--- /dev/null
+++ b/third_party/opus/src/.gitignore
@@ -0,0 +1,83 @@
+Doxyfile
+Makefile
+Makefile.in
+TAGS
+aclocal.m4
+autom4te.cache
+*.kdevelop.pcs
+*.kdevses
+compile
+config.guess
+config.h
+config.h.in
+config.log
+config.status
+config.sub
+configure
+depcomp
+INSTALL
+install-sh
+.deps
+.libs
+.dirstamp
+*.a
+*.exe
+*.la
+*-gnu.S
+testcelt
+libtool
+ltmain.sh
+missing
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
+opus_compare
+opus_demo
+repacketizer_demo
+stamp-h1
+test-driver
+*.sw*
+*.o
+*.lo
+*.pc
+*.tar.gz
+*~
+tests/*test
+tests/test_opus_api
+tests/test_opus_decode
+tests/test_opus_encode
+tests/test_opus_padding
+celt/arm/armopts.s
+celt/dump_modes/dump_modes
+celt/tests/test_unit_cwrs32
+celt/tests/test_unit_dft
+celt/tests/test_unit_entropy
+celt/tests/test_unit_laplace
+celt/tests/test_unit_mathops
+celt/tests/test_unit_mdct
+celt/tests/test_unit_rotation
+celt/tests/test_unit_types
+doc/doxygen_sqlite3.db
+doc/doxygen-build.stamp
+doc/html
+doc/latex
+doc/man
+package_version
+version.h
+celt/Debug
+celt/Release
+celt/x64
+silk/Debug
+silk/Release
+silk/x64
+silk/fixed/Debug
+silk/fixed/Release
+silk/fixed/x64
+silk/float/Debug
+silk/float/Release
+silk/float/x64
+src/Debug
+src/Release
+src/x64
diff --git a/third_party/opus/src/win32/.gitignore b/third_party/opus/src/win32/.gitignore
new file mode 100644
index 0000000..c17feab
--- /dev/null
+++ b/third_party/opus/src/win32/.gitignore
@@ -0,0 +1,26 @@
+# Visual Studio ignores
+[Dd]ebug/
+[Dd]ebugDLL/
+[Dd]ebugDLL_fixed/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleaseDLL/
+[Rr]eleaseDLL_fixed/
+[Rr]eleases/
+*.manifest
+*.lastbuildstate
+*.lib
+*.log
+*.idb
+*.ipdb
+*.ilk
+*.iobj
+*.obj
+*.opensdf
+*.pdb
+*.sdf
+*.suo
+*.tlog
+*.vcxproj.user
+*.vc.db
+*.vc.opendb
diff --git a/third_party/protobuf/.gitignore b/third_party/protobuf/.gitignore
new file mode 100644
index 0000000..51d092c7
--- /dev/null
+++ b/third_party/protobuf/.gitignore
@@ -0,0 +1,171 @@
+# autogen.sh-generated files
+Makefile.in
+src/Makefile.in
+config.guess
+config.h.in
+config.sub
+configure
+depcomp
+install-sh
+ltmain.sh
+missing
+
+aclocal.m4
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
+autom4te.cache
+
+# downloaded files
+gmock
+
+# in-tree configure-generated files
+Makefile
+src/Makefile
+/config.h
+config.log
+config.status
+
+libtool
+protobuf-lite.pc
+protobuf.pc
+.deps
+stamp-h1
+
+# in-tree build products
+*.o
+*.lo
+*.la
+src/.libs
+*.so
+
+.dirstamp
+
+any_test.pb.*
+map*unittest.pb.*
+unittest*.pb.*
+cpp_test*.pb.*
+src/google/protobuf/util/**/*.pb.cc
+src/google/protobuf/util/**/*.pb.h
+
+*.pyc
+*.egg-info
+*_pb2.py
+python/*.egg
+python/.eggs/
+python/.tox
+python/build/
+python/google/protobuf/compiler/
+python/google/protobuf/util/
+
+src/js_embed
+src/protoc
+src/unittest_proto_middleman
+
+# vim generated
+*.swp
+
+# Generated test scaffolding
+src/no_warning_test.cc
+src/no-warning-test
+src/protobuf*-test
+src/test_plugin
+src/testzip.*
+src/zcg*zip
+ar-lib
+
+test-driver
+compile
+
+src/**/*.log
+src/**/*.trs
+
+# JavaBuild output.
+java/core/target
+java/util/target
+javanano/target
+java/.idea
+java/**/*.iml
+
+# Windows native output.
+cmake/build
+build_msvc
+
+# NuGet packages: we want the repository configuration, but not the
+# packages themselves.
+/csharp/src/packages/*/
+
+# OS X's Finder creates these for state about opened windows/etc.
+**/.DS_Store
+
+# Cocoapods artifacts
+# Podfile.lock and the workspace file are tracked, to ease deleting them. That's
+# needed to trigger "pod install" to rerun the preinstall commands.
+Pods/
+
+# Comformance test output
+conformance/.libs/
+conformance/com/
+conformance/conformance-cpp
+conformance/conformance-csharp
+conformance/conformance-java
+conformance/conformance-objc
+conformance/conformance-test-runner
+conformance/conformance.pb.cc
+conformance/conformance.pb.h
+conformance/Conformance.pbobjc.h
+conformance/Conformance.pbobjc.m
+conformance/conformance_pb.js
+conformance/conformance_pb.rb
+conformance/failing_tests.txt
+conformance/google/
+conformance/google-protobuf/
+conformance/javac_middleman
+conformance/lite/
+conformance/nonexistent_tests.txt
+conformance/protoc_middleman
+conformance/succeeding_tests.txt
+conformance/Conformance/
+conformance/GPBMetadata/
+conformance/Google/
+conformance/Protobuf_test_messages/
+conformance/conformance-php
+conformance/conformance-php-c
+
+# php test output
+composer.lock
+php/tests/generated/
+php/tests/old_protoc
+php/tests/protobuf/
+php/ext/google/protobuf/.libs/
+php/ext/google/protobuf/Makefile.fragments
+php/ext/google/protobuf/Makefile.global
+php/ext/google/protobuf/Makefile.objects
+php/ext/google/protobuf/acinclude.m4
+php/ext/google/protobuf/build/
+php/ext/google/protobuf/config.h
+php/ext/google/protobuf/config.nice
+php/ext/google/protobuf/configure.in
+php/ext/google/protobuf/mkinstalldirs
+php/ext/google/protobuf/run-tests.php
+vendor/
+
+# JavaScript artifacts
+js/commonjs_out/
+js/compatibility_tests/v3.0.0/commonjs_out*
+js/compatibility_tests/v3.0.0/protoc
+js/compatibility_tests/v3.0.0/testproto_libs1.js
+js/compatibility_tests/v3.0.0/testproto_libs1_new.js
+js/compatibility_tests/v3.0.0/testproto_libs2.js
+js/compatibility_tests/v3.0.0/testproto_libs2_new.js
+js/deps.js
+js/google-protobuf.js
+js/google/
+js/node_modules/
+js/testproto_libs1.js
+js/testproto_libs2.js
+
+# Ignore the bazel symlinks
+/bazel-*
diff --git a/third_party/protobuf/csharp/.gitignore b/third_party/protobuf/csharp/.gitignore
new file mode 100644
index 0000000..8ba8849
--- /dev/null
+++ b/third_party/protobuf/csharp/.gitignore
@@ -0,0 +1,31 @@
+# Output
+bin
+obj
+project.lock.json
+TestResult.xml
+
+# Possibly legacy now?
+mono/bin
+mono/tmp
+mono/protoc
+build_output
+build_temp
+build/msbuild*.log
+lib/Microsoft.Silverlight.Testing
+lib/NUnit
+
+#
+# Untracked files
+#
+.vs
+*.user
+*.suo
+*.nupkg
+_ReSharper.*
+*.sln.cache
+mono/TestResult.xml
+mono/.libs
+mono/*.exe
+mono/*.dll
+lib/protoc.exe
+*.ncrunch*
diff --git a/third_party/protobuf/objectivec/.gitignore b/third_party/protobuf/objectivec/.gitignore
new file mode 100644
index 0000000..f786ffc
--- /dev/null
+++ b/third_party/protobuf/objectivec/.gitignore
@@ -0,0 +1,23 @@
+## Build generated
+build/
+DerivedData/
+
+## Various settings
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata/
+
+## Other
+*.moved-aside
+*.xccheckout
+*.xcscmblueprint
+
+## Obj-C/Swift specific
+*.hmap
+*.ipa
diff --git a/third_party/protobuf/ruby/.gitignore b/third_party/protobuf/ruby/.gitignore
new file mode 100644
index 0000000..bd8745d
--- /dev/null
+++ b/third_party/protobuf/ruby/.gitignore
@@ -0,0 +1,8 @@
+*.bundle
+tags
+.idea/
+lib/google/protobuf_java.jar
+protobuf-jruby.iml
+target/
+pkg/
+tmp/
diff --git a/tools_webrtc/autoroller/roll_deps.py b/tools_webrtc/autoroller/roll_deps.py
index 469668b..4dce62e 100755
--- a/tools_webrtc/autoroller/roll_deps.py
+++ b/tools_webrtc/autoroller/roll_deps.py
@@ -19,22 +19,24 @@
import sys
import urllib2
-
# Skip these dependencies (list without solution name prefix).
DONT_AUTOROLL_THESE = [
'src/examples/androidtests/third_party/gradle',
+ 'src/third_party_chromium',
]
# Run these CQ trybots in addition to the default ones in infra/config/cq.cfg.
EXTRA_TRYBOTS = (
- 'master.internal.tryserver.corp.webrtc:linux_internal'
+ 'master.internal.tryserver.corp.webrtc:linux_internal'
)
WEBRTC_URL = 'https://webrtc.googlesource.com/src'
CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
+CHROMIUM_THIRD_PARTY_URL = '%s/third_party' % CHROMIUM_SRC_URL
CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
+CHROMIUM_3P_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/third_party/+log/%s'
COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
@@ -47,6 +49,7 @@
sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
import find_depot_tools
+
find_depot_tools.add_depot_tools_to_path()
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
@@ -60,6 +63,13 @@
ChangedCipdPackage = collections.namedtuple(
'ChangedCipdPackage', 'path package current_version new_version')
+ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate',
+ ('current_chromium_rev '
+ 'new_chromium_rev '
+ 'current_third_party_rev '
+ 'new_third_party_rev'))
+
+
class RollError(Exception):
pass
@@ -74,7 +84,7 @@
'Var': VarLookup(local_scope),
'deps_os': {},
}
- exec(deps_content, global_scope, local_scope)
+ exec (deps_content, global_scope, local_scope)
return local_scope
@@ -100,7 +110,7 @@
def _RunCommand(command, working_dir=None, ignore_exit_code=False,
- extra_env=None):
+ extra_env=None, input_data=None):
"""Runs a command and returns the output from that command.
If the command fails (exit code != 0), the function will exit the process.
@@ -115,12 +125,12 @@
assert all(type(value) == str for value in extra_env.values())
logging.debug('extra env: %s', extra_env)
env.update(extra_env)
- p = subprocess.Popen(command, stdout=subprocess.PIPE,
+ p = subprocess.Popen(command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env,
cwd=working_dir, universal_newlines=True)
- std_output = p.stdout.read()
- err_output = p.stderr.read()
- p.wait()
+ std_output, err_output = p.communicate(input_data)
p.stdout.close()
p.stderr.close()
if not ignore_exit_code and p.returncode != 0:
@@ -209,6 +219,7 @@
def BuildDepsentryDict(deps_dict):
"""Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
result = {}
+
def AddDepsEntries(deps_subdict):
for path, dep in deps_subdict.iteritems():
if path in result:
@@ -279,8 +290,8 @@
# Use the revision from Chromium's DEPS file.
new_rev = cr_deps_entry.revision
assert webrtc_deps_entry.url == cr_deps_entry.url, (
- 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
- (path, webrtc_deps_entry.url, cr_deps_entry.url))
+ 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
+ (path, webrtc_deps_entry.url, cr_deps_entry.url))
else:
# Use the HEAD of the deps repo.
stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
@@ -308,23 +319,28 @@
current_rev = GetClangRev(current_lines)
new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
- new_cr_rev).splitlines()
+ new_cr_rev).splitlines()
new_rev = GetClangRev(new_clang_update_py)
return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
-def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
- new_commit_pos, changed_deps_list, clang_change):
- current_cr_rev = current_cr_rev[0:10]
- new_cr_rev = new_cr_rev[0:10]
+def GenerateCommitMessage(rev_update, current_commit_pos,
+ new_commit_pos, changed_deps_list, clang_change):
+ current_cr_rev = rev_update.current_chromium_rev[0:10]
+ new_cr_rev = rev_update.new_chromium_rev[0:10]
rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
+ rev_3p_interval = '%s..%s' % (rev_update.current_third_party_rev[0:10],
+ rev_update.new_third_party_rev[0:10])
git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
- git_number_interval)]
- commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
- commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
- rev_interval))
+ git_number_interval),
+ 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
+ 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
+ rev_interval),
+ 'Roll chromium third_party %s' % rev_3p_interval,
+ 'Change log: %s\n' % (
+ CHROMIUM_3P_LOG_TEMPLATE % rev_3p_interval)]
tbr_authors = ''
if changed_deps_list:
commit_msg.append('Changed dependencies:')
@@ -365,14 +381,16 @@
return '\n'.join(commit_msg)
-def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
- changed_deps):
+def UpdateDepsFile(deps_filename, rev_update, changed_deps):
"""Update the DEPS file with the new revision."""
# Update the chromium_revision variable.
with open(deps_filename, 'rb') as deps_file:
deps_content = deps_file.read()
- deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
+ deps_content = deps_content.replace(rev_update.current_chromium_rev,
+ rev_update.new_chromium_rev)
+ deps_content = deps_content.replace(rev_update.current_third_party_rev,
+ rev_update.new_third_party_rev)
with open(deps_filename, 'wb') as deps_file:
deps_file.write(deps_content)
@@ -395,6 +413,36 @@
working_dir=CHECKOUT_SRC_DIR)
+def _LoadThirdPartyDepsAndFiles(filename):
+ third_party_deps = {}
+ with open(filename, 'rb') as f:
+ deps_content = f.read()
+ global_scope = {}
+ exec (deps_content, global_scope, third_party_deps)
+ return third_party_deps.get('DEPS', [])
+
+
+def UpdateThirdPartyDeps(new_rev, dest_dir, source_dir,
+ third_party_deps_file):
+ """Syncing deps, specified in third_party_deps_file with repo in source_dir.
+
+ Will exit if sync failed for some reasons.
+ Params:
+ new_rev - revision of third_party to update to
+ dest_dir - webrtc directory, that will be used as root for third_party deps
+ source_dir - checked out chromium third_party repo
+ third_party_deps_file - file with list of third_party deps to copy
+ """
+
+ deps_to_checkout = _LoadThirdPartyDepsAndFiles(third_party_deps_file)
+ # Update existing chromium third_party checkout to new rev.
+ _RunCommand(['git', 'fetch', 'origin', new_rev], working_dir=source_dir)
+ # Checkout chromium repo into dest dir basing on source checkout.
+ _RunCommand(
+ ['git', '--git-dir', '%s/.git' % source_dir, 'checkout',
+ new_rev] + deps_to_checkout, working_dir=dest_dir)
+
+
def _IsTreeClean():
stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
if len(stdout) == 0:
@@ -437,6 +485,7 @@
logging.info('Committing changes locally.')
if not dry_run:
_RunCommand(['git', 'add', '--update', '.'])
+ _RunCommand(['git', 'add', '-A', 'third_party'])
_RunCommand(['git', 'commit', '-m', commit_msg])
@@ -466,6 +515,30 @@
_RunCommand(cmd, extra_env={'EDITOR': 'true', 'SKIP_GCE_AUTH_FOR_GIT': '1'})
+def GetRollRevisionRanges(opts, webrtc_deps):
+ current_cr_rev = webrtc_deps['vars']['chromium_revision']
+ current_third_party_rev = webrtc_deps['vars']['chromium_third_party_revision']
+ new_cr_rev = opts.revision
+ if not new_cr_rev:
+ stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
+ head_rev = stdout.strip().split('\t')[0]
+ logging.info('No revision specified. Using HEAD: %s', head_rev)
+ new_cr_rev = head_rev
+
+ new_third_party_rev = opts.third_party_revision
+ if not new_third_party_rev:
+ stdout, _ = _RunCommand(
+ ['git', 'ls-remote', CHROMIUM_THIRD_PARTY_URL, 'HEAD'])
+ new_third_party_rev = stdout.strip().split('\t')[0]
+ logging.info(
+ 'No third_party revision specified. Using HEAD: %s',
+ new_third_party_rev)
+
+ return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev,
+ current_third_party_rev,
+ new_third_party_rev)
+
+
def main():
p = argparse.ArgumentParser()
p.add_argument('--clean', action='store_true', default=False,
@@ -473,6 +546,9 @@
p.add_argument('-r', '--revision',
help=('Chromium Git revision to roll to. Defaults to the '
'Chromium HEAD revision if omitted.'))
+ p.add_argument('--third-party-revision',
+ help=('Chromium third_party Git revision to roll to. Default '
+ 'to the Chromium third_party HEAD revision if omitted.'))
p.add_argument('-u', '--rietveld-email',
help=('E-mail address to use for creating the CL at Rietveld'
'If omitted a previously cached one will be used or an '
@@ -510,30 +586,38 @@
if not opts.ignore_unclean_workdir:
_EnsureUpdatedMasterBranch(opts.dry_run)
- new_cr_rev = opts.revision
- if not new_cr_rev:
- stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
- head_rev = stdout.strip().split('\t')[0]
- logging.info('No revision specified. Using HEAD: %s', head_rev)
- new_cr_rev = head_rev
-
deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
webrtc_deps = ParseLocalDepsFile(deps_filename)
- current_cr_rev = webrtc_deps['vars']['chromium_revision']
+ cr_3p_repo = os.path.join(CHECKOUT_SRC_DIR, 'third_party_chromium')
+ if not os.path.exists(cr_3p_repo):
+ raise RollError('missing third_party_chromium/. '
+ 'Please add this to your gclient: \n'
+ '"custom_vars": {\n'
+ ' "roll_chromium_into_webrtc": True,\n'
+ '},\n'
+ 'Then run "gclient sync" again.')
- current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
- new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
+ rev_update = GetRollRevisionRanges(opts, webrtc_deps)
- new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
+ current_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.current_chromium_rev))
+ new_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.new_chromium_rev))
+
+ new_cr_deps = ParseRemoteCrDepsFile(rev_update.new_chromium_rev)
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
- clang_change = CalculateChangedClang(new_cr_rev)
- commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
+ clang_change = CalculateChangedClang(rev_update.new_chromium_rev)
+ commit_msg = GenerateCommitMessage(rev_update,
current_commit_pos, new_commit_pos,
changed_deps, clang_change)
logging.debug('Commit message:\n%s', commit_msg)
_CreateRollBranch(opts.dry_run)
- UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
+ UpdateThirdPartyDeps(rev_update.new_third_party_rev,
+ os.path.join(CHECKOUT_SRC_DIR, 'third_party'),
+ cr_3p_repo,
+ os.path.join(CHECKOUT_SRC_DIR, 'THIRD_PARTY_DEPS'))
+ UpdateDepsFile(deps_filename, rev_update, changed_deps)
if _IsTreeClean():
logging.info("No DEPS changes detected, skipping CL creation.")
else:
diff --git a/tools_webrtc/autoroller/unittests/roll_deps_test.py b/tools_webrtc/autoroller/unittests/roll_deps_test.py
index b6ac1be..1753508 100755
--- a/tools_webrtc/autoroller/unittests/roll_deps_test.py
+++ b/tools_webrtc/autoroller/unittests/roll_deps_test.py
@@ -20,7 +20,8 @@
sys.path.append(PARENT_DIR)
import roll_deps
from roll_deps import CalculateChangedDeps, ChooseCQMode, \
- GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
+ GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile, \
+ ChromiumRevisionUpdate
TEST_DATA_VARS = {
@@ -90,7 +91,10 @@
new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
current_rev = TEST_DATA_VARS['chromium_revision']
- UpdateDepsFile(self._webrtc_depsfile, current_rev, new_rev, [])
+ UpdateDepsFile(self._webrtc_depsfile,
+ ChromiumRevisionUpdate(
+ current_rev, new_rev, current_rev, new_rev),
+ [])
with open(self._webrtc_depsfile) as deps_file:
deps_contents = deps_file.read()
self.assertTrue(new_rev in deps_contents,
