blob: 6466f6c2f401944f8c76e987f833a8c5bab8caf6 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Given the output of -t commands from a ninja build for a gyp and GN generated
build, report on differences between the command lines.
When invoked from the command line, this script assumes that the GN and GYP
targets have been generated in the specified folders. It is meant to be used as
$ python tools/ gyp_dir gn_dir target
When the GN and GYP target names differ, it should be called invoked as follows:
$ python tools/ gyp_dir gn_dir gyp_target gn_target
When all targets want to be compared, it should be called without a target name,
$ python tools/ gyp_dir gn_dir
This script can also be used interactively. Then ConfigureBuild can optionally
be used to generate ninja files with GYP and GN.
Here's an example setup. Note that the current working directory must be the
project root:
$ PYTHONPATH=tools python
>>> import sys
>>> import pprint
>>> sys.displayhook = pprint.pprint
>>> import gyp_flag_compare as fc
>>> fc.ConfigureBuild(['gyp_define=1', 'define=2'], ['gn_arg=1', 'arg=2'])
>>> modules_unittests = fc.Comparison('modules_unittests')
The above starts interactive Python, sets up the output to be pretty-printed
(useful for making lists, dicts, and sets readable), configures the build with
GN arguments and GYP defines, and then generates a comparison for that build
configuration for the "modules_unittests" target.
After that, the |modules_unittests| object can be used to investigate
differences in the build.
To configure an official build, use this configuration. Disabling NaCl produces
a more meaningful comparison, as certain files need to get compiled twice
for the IRT build, which uses different flags:
>>> fc.ConfigureBuild(
['disable_nacl=1', 'buildtype=Official', 'branding=Chrome'],
['enable_nacl=false', 'is_official_build=true',
import os
import shlex
import subprocess
import sys
# Must be in src/.
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_DEFAULT_GN_DIR = 'out/gn'
_DEFAULT_GYP_DIR = 'out/Release'
def FilterChromium(filename):
"""Replaces 'chromium/src/' by '' in the filename."""
return filename.replace('chromium/src/', '')
def ConfigureBuild(gyp_args=None, gn_args=None, gn_dir=_DEFAULT_GN_DIR):
"""Generates gn and gyp targets with the given arguments."""
gyp_args = gyp_args or []
gn_args = gn_args or []
print >> sys.stderr, 'Regenerating GN in %s...' % gn_dir
# Currently only Release, non-component.
Run('gn gen %s --args="is_debug=false is_component_build=false %s"' % \
(gn_dir, ' '.join(gn_args)))
os.environ.pop('GYP_DEFINES', None)
# Remove environment variables required by gn but conflicting with GYP.
# Relevant if Windows toolchain isn't provided by depot_tools.
os.environ.pop('GYP_MSVS_OVERRIDE_PATH', None)
os.environ.pop('WINDOWSSDKDIR', None)
gyp_defines = ''
if len(gyp_args) > 0:
gyp_defines = '-D' + ' -D'.join(gyp_args)
print >> sys.stderr, 'Regenerating GYP in %s...' % _DEFAULT_GYP_DIR
Run('python webrtc/build/ -Gconfig=Release %s' % gyp_defines)
def Counts(dict_of_list):
"""Given a dictionary whose value are lists, returns a dictionary whose values
are the length of the list. This can be used to summarize a dictionary.
return {k: len(v) for k, v in dict_of_list.iteritems()}
def CountsByDirname(dict_of_list):
"""Given a list of files, returns a dict of dirname to counts in that dir."""
r = {}
for path in dict_of_list:
dirname = os.path.dirname(path)
r.setdefault(dirname, 0)
r[dirname] += 1
return r
class Comparison(object):
"""A comparison of the currently-configured build for a target."""
def __init__(self, gyp_target="", gn_target=None, gyp_dir=_DEFAULT_GYP_DIR,
"""Creates a comparison of a GN and GYP target. If the target names differ
between the two build systems, then two names may be passed.
if gn_target is None:
gn_target = gyp_target
self._gyp_target = gyp_target
self._gn_target = gn_target
self._gyp_dir = gyp_dir
self._gn_dir = gn_dir
self._skipped = []
self._total_diffs = 0
self._missing_gyp_flags = {}
self._missing_gn_flags = {}
self._missing_gyp_files = {}
self._missing_gn_files = {}
def gyp_files(self):
"""Returns the set of files that are in the GYP target."""
return set(self._gyp_flags.keys())
def gn_files(self):
"""Returns the set of files that are in the GN target."""
return set(self._gn_flags.keys())
def skipped(self):
"""Returns the list of compiler commands that were not processed during the
return self._skipped
def total_differences(self):
"""Returns the total number of differences detected."""
return self._total_diffs
def missing_in_gyp(self):
"""Differences that are only in GYP build but not in GN, indexed by the
return self._missing_gyp_flags
def missing_in_gn(self):
"""Differences that are only in the GN build but not in GYP, indexed by
the difference."""
return self._missing_gn_flags
def missing_in_gyp_by_file(self):
"""Differences that are only in the GYP build but not in GN, indexed by
return self._missing_gyp_files
def missing_in_gn_by_file(self):
"""Differences that are only in the GYP build but not in GN, indexed by
return self._missing_gn_files
def _CompareFiles(self):
"""Performs the actual target comparison."""
if sys.platform == 'win32':
# On Windows flags are stored in .rsp files which are created by building.
print >> sys.stderr, 'Building in %s...' % self._gn_dir
Run('ninja -C %s -d keeprsp %s' % (self._gn_dir, self._gn_target))
print >> sys.stderr, 'Building in %s...' % self._gyp_dir
Run('ninja -C %s -d keeprsp %s' % (self._gyp_dir, self._gn_target))
gn = Run('ninja -C %s -t commands %s' % (self._gn_dir, self._gn_target))
gyp = Run('ninja -C %s -t commands %s' % (self._gyp_dir, self._gyp_target))
self._gn_flags = self._GetFlags(gn.splitlines(),
os.path.join(os.getcwd(), self._gn_dir))
self._gyp_flags = self._GetFlags(gyp.splitlines(),
os.path.join(os.getcwd(), self._gyp_dir))
self._gn_flags = dict((FilterChromium(filename), value)
for filename, value in self._gn_flags.iteritems())
self._gyp_flags = dict((FilterChromium(filename), value)
for filename, value in self._gyp_flags.iteritems())
all_files = sorted(self.gn_files & self.gyp_files)
for filename in all_files:
gyp_flags = self._gyp_flags[filename]
gn_flags = self._gn_flags[filename]
self._CompareLists(filename, gyp_flags, gn_flags, 'dash_f')
self._CompareLists(filename, gyp_flags, gn_flags, 'defines',
# These defines are not used by WebRTC
self._CompareLists(filename, gyp_flags, gn_flags, 'include_dirs')
self._CompareLists(filename, gyp_flags, gn_flags, 'warnings',
# More conservative warnings in GN we consider to be OK.
'/wd4091', # 'keyword' : ignored on left of 'type' when no variable
# is declared.
'/wd4456', # Declaration hides previous local declaration.
'/wd4457', # Declaration hides function parameter.
'/wd4458', # Declaration hides class member.
'/wd4459', # Declaration hides global declaration.
'/wd4702', # Unreachable code.
'/wd4800', # Forcing value to bool 'true' or 'false'.
'/wd4838', # Conversion from 'type' to 'type' requires a narrowing
# conversion.
] if sys.platform == 'win32' else None,
] if not sys.platform == 'win32' else None)
self._CompareLists(filename, gyp_flags, gn_flags, 'other',
dont_care_gyp=['-g'], dont_care_gn=['-g2'])
def _CompareLists(self, filename, gyp, gn, name,
dont_care_gyp=None, dont_care_gn=None):
"""Return a report of any differences between gyp and gn lists, ignoring
anything in |dont_care_{gyp|gn}| respectively."""
if gyp[name] == gn[name]:
if not dont_care_gyp:
dont_care_gyp = []
if not dont_care_gn:
dont_care_gn = []
gyp_set = set(gyp[name])
gn_set = set(gn[name])
missing_in_gyp = gyp_set - gn_set
missing_in_gn = gn_set - gyp_set
missing_in_gyp -= set(dont_care_gyp)
missing_in_gn -= set(dont_care_gn)
for m in missing_in_gyp:
self._missing_gyp_flags.setdefault(name, {}) \
.setdefault(m, []).append(filename)
self._total_diffs += 1
self._missing_gyp_files.setdefault(filename, {}) \
.setdefault(name, set()).update(missing_in_gyp)
for m in missing_in_gn:
self._missing_gn_flags.setdefault(name, {}) \
.setdefault(m, []).append(filename)
self._total_diffs += 1
self._missing_gn_files.setdefault(filename, {}) \
.setdefault(name, set()).update(missing_in_gn)
def _GetFlags(self, lines, build_dir):
"""Turn a list of command lines into a semi-structured dict."""
is_win = sys.platform == 'win32'
flags_by_output = {}
for line in lines:
line = FilterChromium(line)
line = line.replace(os.getcwd(), '../../')
line = line.replace('//', '/')
command_line = shlex.split(line.strip(), posix=not is_win)[1:]
output_name = _FindAndRemoveArgWithValue(command_line, '-o')
dep_name = _FindAndRemoveArgWithValue(command_line, '-MF')
command_line = _MergeSpacedArgs(command_line, '-Xclang')
cc_file = [x for x in command_line if x.endswith('.cc') or
x.endswith('.c') or
x.endswith('.cpp') or
x.endswith('.mm') or
if len(cc_file) != 1:
assert len(cc_file) == 1
if is_win:
rsp_file = [x for x in command_line if x.endswith('.rsp')]
assert len(rsp_file) <= 1
if rsp_file:
rsp_file = os.path.join(build_dir, rsp_file[0][1:])
with open(rsp_file, "r") as open_rsp_file:
command_line = shlex.split(open_rsp_file, posix=False)
defines = [x for x in command_line if x.startswith('-D')]
include_dirs = [x for x in command_line if x.startswith('-I')]
dash_f = [x for x in command_line if x.startswith('-f')]
warnings = \
[x for x in command_line if x.startswith('/wd' if is_win else '-W')]
others = [x for x in command_line if x not in defines and \
x not in include_dirs and \
x not in dash_f and \
x not in warnings and \
x not in cc_file]
for index, value in enumerate(include_dirs):
if value == '-Igen':
path = value[2:]
if not os.path.isabs(path):
path = os.path.join(build_dir, path)
include_dirs[index] = '-I' + os.path.normpath(path)
# GYP supports paths above the source root like <(DEPTH)/../foo while such
# paths are unsupported by gn. But gn allows to use system-absolute paths
# instead (paths that start with single '/'). Normalize all paths.
cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))]
# Filter for having a relative path in one and
# absolute path in the other.
others_filtered = []
for x in others:
if x.startswith('-Xclang ') and \
(x.endswith('') or \
'-Xclang ' +
os.path.join(os.getcwd(), os.path.normpath(
os.path.join('out/gn_flags', x.split(' ', 1)[1]))))
elif x.startswith('-B'):
'-B' +
os.path.join(os.getcwd(), os.path.normpath(
os.path.join('out/gn_flags', x[2:]))))
others = others_filtered
flags_by_output[cc_file[0]] = {
'output': output_name,
'depname': dep_name,
'defines': sorted(defines),
'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong.
'dash_f': sorted(dash_f),
'warnings': sorted(warnings),
'other': sorted(others),
return flags_by_output
def _FindAndRemoveArgWithValue(command_line, argname):
"""Given a command line as a list, remove and return the value of an option
that takes a value as a separate entry.
Modifies |command_line| in place.
if argname not in command_line:
return ''
location = command_line.index(argname)
value = command_line[location + 1]
command_line[location:location + 2] = []
return value
def _MergeSpacedArgs(command_line, argname):
"""Combine all arguments |argname| with their values, separated by a space."""
i = 0
result = []
while i < len(command_line):
arg = command_line[i]
if arg == argname:
result.append(arg + ' ' + command_line[i + 1])
i += 1
i += 1
return result
def Run(command_line):
"""Run |command_line| as a subprocess and return stdout. Raises on error."""
print >> sys.stderr, command_line
return subprocess.check_output(command_line, shell=True)
def main():
if len(sys.argv) < 3:
print 'usage: %s gyp_dir gn_dir' % __file__
print ' or: %s gyp_dir gn_dir target' % __file__
print ' or: %s gyp_dir gn_dir gyp_target gn_target' % __file__
return 1
gyp_dir = sys.argv[1]
gn_dir = sys.argv[2]
gyp_target = gn_target = ""
if len(sys.argv) > 3:
gyp_target = sys.argv[3]
if len(sys.argv) > 4:
gn_target = sys.argv[4]
print 'GYP output directory is %s' % gyp_dir
print 'GN output directory is %s' % gn_dir
comparison = Comparison(gyp_target, gn_target, gyp_dir, gn_dir)
differing_files = set(comparison.missing_in_gn_by_file.keys()) & \
files_with_given_differences = {}
for filename in differing_files:
output = ''
missing_in_gyp = comparison.missing_in_gyp_by_file.get(filename, {})
missing_in_gn = comparison.missing_in_gn_by_file.get(filename, {})
difference_types = sorted(set(missing_in_gyp.keys() + missing_in_gn.keys()))
for difference_type in difference_types:
if (len(missing_in_gyp[difference_type]) == 0 and
len(missing_in_gn[difference_type]) == 0):
output += ' %s differ:\n' % difference_type
if (difference_type in missing_in_gyp and
output += ' In gyp, but not in GN:\n %s' % '\n '.join(
sorted(missing_in_gyp[difference_type])) + '\n'
if (difference_type in missing_in_gn and
output += ' In GN, but not in gyp:\n %s' % '\n '.join(
sorted(missing_in_gn[difference_type])) + '\n'
if output:
files_with_given_differences.setdefault(output, []).append(filename)
for diff, files in files_with_given_differences.iteritems():
print '\n'.join(sorted(files))
print diff
print 'Total differences:', comparison.total_differences
# TODO(scottmg): Return failure on difference once we're closer to identical.
return 0
if __name__ == '__main__':