#!/usr/bin/env python
# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS.  All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.

"""MB - the Meta-Build wrapper around GN.

MB is a wrapper script for GN that can be used to generate build files
for sets of canned configurations and analyze them.
"""

from __future__ import print_function

import argparse
import ast
import errno
import json
import os
import pipes
import pprint
import re
import shutil
import sys
import subprocess
import tempfile
import traceback
try:
  from urllib2 import urlopen # for Python2
except ImportError:
  from urllib.request import urlopen # for Python3

from collections import OrderedDict

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
sys.path = [os.path.join(SRC_DIR, 'build')] + sys.path

import gn_helpers


def main(args):
    mbw = MetaBuildWrapper()
    return mbw.Main(args)


class MetaBuildWrapper(object):
    def __init__(self):
        self.src_dir = SRC_DIR
        self.default_config = os.path.join(SCRIPT_DIR, 'mb_config.pyl')
        self.default_isolate_map = os.path.join(
            SCRIPT_DIR, 'gn_isolate_map.pyl')
        self.executable = sys.executable
        self.platform = sys.platform
        self.sep = os.sep
        self.args = argparse.Namespace()
        self.configs = {}
        self.builder_groups = {}
        self.mixins = {}
        self.isolate_exe = 'isolate.exe' if self.platform.startswith(
            'win') else 'isolate'

    def Main(self, args):
        self.ParseArgs(args)
        try:
            ret = self.args.func()
            if ret:
                self.DumpInputFiles()
            return ret
        except KeyboardInterrupt:
            self.Print('interrupted, exiting')
            return 130
        except Exception:
            self.DumpInputFiles()
            s = traceback.format_exc()
            for l in s.splitlines():
                self.Print(l)
            return 1

    def ParseArgs(self, argv):
        def AddCommonOptions(subp):
            subp.add_argument('-b', '--builder',
                              help='builder name to look up config from')
            subp.add_argument('-m', '--builder-group',
                              help='builder group name to look up config from')
            subp.add_argument('-c', '--config',
                              help='configuration to analyze')
            subp.add_argument('--phase',
                              help='optional phase name (used when builders '
                                   'do multiple compiles with different '
                                   'arguments in a single build)')
            subp.add_argument('-f', '--config-file', metavar='PATH',
                              default=self.default_config,
                              help='path to config file '
                                   '(default is %(default)s)')
            subp.add_argument('-i', '--isolate-map-file', metavar='PATH',
                              default=self.default_isolate_map,
                              help='path to isolate map file '
                                   '(default is %(default)s)')
            subp.add_argument('-r', '--realm', default='webrtc:try',
                              help='optional LUCI realm to use (for example '
                                   'when triggering tasks on Swarming)')
            subp.add_argument('-g', '--goma-dir',
                              help='path to goma directory')
            subp.add_argument('--android-version-code',
                              help='Sets GN arg android_default_version_code')
            subp.add_argument('--android-version-name',
                              help='Sets GN arg android_default_version_name')
            subp.add_argument('-n', '--dryrun', action='store_true',
                              help='Do a dry run (i.e., do nothing, just '
                                   'print the commands that will run)')
            subp.add_argument('-v', '--verbose', action='store_true',
                              help='verbose logging')

        parser = argparse.ArgumentParser(prog='mb')
        subps = parser.add_subparsers()

        subp = subps.add_parser('analyze',
                                help='analyze whether changes to a set of '
                                     'files will cause a set of binaries '
                                     'to be rebuilt.')
        AddCommonOptions(subp)
        subp.add_argument('path', nargs=1,
                          help='path build was generated into.')
        subp.add_argument('input_path', nargs=1,
                          help='path to a file containing the input '
                               'arguments as a JSON object.')
        subp.add_argument('output_path', nargs=1,
                          help='path to a file containing the output '
                               'arguments as a JSON object.')
        subp.add_argument('--json-output',
                          help='Write errors to json.output')
        subp.set_defaults(func=self.CmdAnalyze)

        subp = subps.add_parser('export',
                                help='print out the expanded configuration for'
                                     'each builder as a JSON object')
        subp.add_argument('-f', '--config-file', metavar='PATH',
                          default=self.default_config,
                          help='path to config file (default is %(default)s)')
        subp.add_argument('-g', '--goma-dir',
                          help='path to goma directory')
        subp.set_defaults(func=self.CmdExport)

        subp = subps.add_parser('gen',
                                help='generate a new set of build files')
        AddCommonOptions(subp)
        subp.add_argument('--swarming-targets-file',
                          help='save runtime dependencies for targets listed '
                               'in file.')
        subp.add_argument('--json-output',
                          help='Write errors to json.output')
        subp.add_argument('path', nargs=1,
                          help='path to generate build into')
        subp.set_defaults(func=self.CmdGen)

        subp = subps.add_parser('isolate',
                                help='generate the .isolate files for a given'
                                     'binary')
        AddCommonOptions(subp)
        subp.add_argument('path', nargs=1,
                          help='path build was generated into')
        subp.add_argument('target', nargs=1,
                          help='ninja target to generate the isolate for')
        subp.set_defaults(func=self.CmdIsolate)

        subp = subps.add_parser('lookup',
                                help='look up the command for a given config '
                                     'or builder')
        AddCommonOptions(subp)
        subp.add_argument('--quiet', default=False, action='store_true',
                          help='Print out just the arguments, do '
                               'not emulate the output of the gen subcommand.')
        subp.set_defaults(func=self.CmdLookup)

        subp = subps.add_parser(
            'run',
            help='build and run the isolated version of a '
                 'binary',
            formatter_class=argparse.RawDescriptionHelpFormatter)
        subp.description = (
            'Build, isolate, and run the given binary with the command line\n'
            'listed in the isolate. You may pass extra arguments after the\n'
            'target; use "--" if the extra arguments need to include switches.'
            '\n\n'
            'Examples:\n'
            '\n'
            '  % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n'
            '    //out/Default content_browsertests\n'
            '\n'
            '  % tools/mb/mb.py run out/Default content_browsertests\n'
            '\n'
            '  % tools/mb/mb.py run out/Default content_browsertests -- \\\n'
            '    --test-launcher-retry-limit=0'
            '\n'
        )
        AddCommonOptions(subp)
        subp.add_argument('-j', '--jobs', dest='jobs', type=int,
                          help='Number of jobs to pass to ninja')
        subp.add_argument('--no-build', dest='build', default=True,
                          action='store_false',
                          help='Do not build, just isolate and run')
        subp.add_argument('path', nargs=1,
                          help=('path to generate build into (or use).'
                                ' This can be either a regular path or a '
                                'GN-style source-relative path like '
                                '//out/Default.'))
        subp.add_argument('-s', '--swarmed', action='store_true',
                          help='Run under swarming')
        subp.add_argument('-d', '--dimension', default=[], action='append',
                          nargs=2, dest='dimensions', metavar='FOO bar',
                          help='dimension to filter on')
        subp.add_argument('target', nargs=1,
                          help='ninja target to build and run')
        subp.add_argument('extra_args', nargs='*',
                          help=('extra args to pass to the isolate to run. '
                                'Use "--" as the first arg if you need to '
                                'pass switches'))
        subp.set_defaults(func=self.CmdRun)

        subp = subps.add_parser('validate',
                                help='validate the config file')
        subp.add_argument('-f', '--config-file', metavar='PATH',
                          default=self.default_config,
                          help='path to config file (default is %(default)s)')
        subp.set_defaults(func=self.CmdValidate)

        subp = subps.add_parser('help',
                                help='Get help on a subcommand.')
        subp.add_argument(nargs='?', action='store', dest='subcommand',
                          help='The command to get help for.')
        subp.set_defaults(func=self.CmdHelp)

        self.args = parser.parse_args(argv)

    def DumpInputFiles(self):

        def DumpContentsOfFilePassedTo(arg_name, path):
            if path and self.Exists(path):
                self.Print("\n# To recreate the file passed to %s:" % arg_name)
                self.Print("%% cat > %s <<EOF" % path)
                contents = self.ReadFile(path)
                self.Print(contents)
                self.Print("EOF\n%\n")

        if getattr(self.args, 'input_path', None):
            DumpContentsOfFilePassedTo(
                'argv[0] (input_path)', self.args.input_path[0])
        if getattr(self.args, 'swarming_targets_file', None):
            DumpContentsOfFilePassedTo(
                '--swarming-targets-file', self.args.swarming_targets_file)

    def CmdAnalyze(self):
        vals = self.Lookup()
        return self.RunGNAnalyze(vals)

    def CmdExport(self):
        self.ReadConfigFile()
        obj = {}
        for builder_group, builders in self.builder_groups.items():
            obj[builder_group] = {}
            for builder in builders:
                config = self.builder_groups[builder_group][builder]
                if not config:
                    continue

                if isinstance(config, dict):
                    args = {k: self.FlattenConfig(v)['gn_args']
                            for k, v in config.items()}
                elif config.startswith('//'):
                    args = config
                else:
                    args = self.FlattenConfig(config)['gn_args']
                    if 'error' in args:
                        continue

                obj[builder_group][builder] = args

        # Dump object and trim trailing whitespace.
        s = '\n'.join(l.rstrip() for l in
                      json.dumps(obj, sort_keys=True, indent=2).splitlines())
        self.Print(s)
        return 0

    def CmdGen(self):
        vals = self.Lookup()
        return self.RunGNGen(vals)

    def CmdHelp(self):
        if self.args.subcommand:
            self.ParseArgs([self.args.subcommand, '--help'])
        else:
            self.ParseArgs(['--help'])

    def CmdIsolate(self):
        vals = self.GetConfig()
        if not vals:
            return 1
        return self.RunGNIsolate(vals)

    def CmdLookup(self):
        vals = self.Lookup()
        gn_args = self.GNArgs(vals)
        if self.args.quiet:
            self.Print(gn_args, end='')
        else:
            cmd = self.GNCmd('gen', '_path_')
            self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
            env = None

            self.PrintCmd(cmd, env)
        return 0

    def CmdRun(self):
        vals = self.GetConfig()
        if not vals:
            return 1

        build_dir = self.args.path[0]
        target = self.args.target[0]

        if self.args.build:
            ret = self.Build(target)
            if ret:
                return ret
        ret = self.RunGNIsolate(vals)
        if ret:
            return ret

        if self.args.swarmed:
            cmd, _ = self.GetSwarmingCommand(self.args.target[0], vals)
            return self._RunUnderSwarming(build_dir, target, cmd)
        return self._RunLocallyIsolated(build_dir, target)

    def _RunUnderSwarming(self, build_dir, target, isolate_cmd):
        cas_instance = 'chromium-swarm'
        swarming_server = 'chromium-swarm.appspot.com'
        # TODO(dpranke): Look up the information for the target in
        # the //testing/buildbot.json file, if possible, so that we
        # can determine the isolate target, command line, and additional
        # swarming parameters, if possible.
        #
        # TODO(dpranke): Also, add support for sharding and merging results.
        dimensions = []
        for k, v in self.args.dimensions:
            dimensions += ['-d', '%s=%s' % (k, v)]

        archive_json_path = self.ToSrcRelPath(
            '%s/%s.archive.json' % (build_dir, target))
        cmd = [
            self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe),
            'archive',
            '-i',
            self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
            '-cas-instance',
            cas_instance,
            '-dump-json',
            archive_json_path,
        ]

        # Talking to the isolateserver may fail because we're not logged in.
        # We trap the command explicitly and rewrite the error output so that
        # the error message is actually correct for a Chromium check out.
        self.PrintCmd(cmd, env=None)
        ret, out, err = self.Run(cmd, force_verbose=False)
        if ret:
            self.Print('  -> returned %d' % ret)
            if out:
                self.Print(out, end='')
            if err:
                self.Print(err, end='', file=sys.stderr)

            return ret

        try:
            archive_hashes = json.loads(self.ReadFile(archive_json_path))
        except Exception:
            self.Print(
                'Failed to read JSON file "%s"' %
                archive_json_path, file=sys.stderr)
            return 1
        try:
            cas_digest = archive_hashes[target]
        except Exception:
            self.Print(
                'Cannot find hash for "%s" in "%s", file content: %s' %
                (target, archive_json_path, archive_hashes),
                file=sys.stderr)
            return 1

        try:
            json_dir = self.TempDir()
            json_file = self.PathJoin(json_dir, 'task.json')

            cmd = [
                self.PathJoin('tools', 'luci-go', 'swarming'),
                'trigger',
                '-realm',
                self.args.realm,
                '-digest',
                cas_digest,
                '-server',
                swarming_server,
                '-tag=purpose:user-debug-mb',
                '-relative-cwd',
                self.ToSrcRelPath(build_dir),
                '-dump-json',
                json_file,
            ] + dimensions + ['--'] + list(isolate_cmd)

            if self.args.extra_args:
                cmd += ['--'] + self.args.extra_args
            self.Print('')
            ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
            if ret:
                return ret
            task_json = self.ReadFile(json_file)
            task_id = json.loads(task_json)["tasks"][0]['task_id']
        finally:
            if json_dir:
                self.RemoveDirectory(json_dir)

        cmd = [
            self.PathJoin('tools', 'luci-go', 'swarming'),
            'collect',
            '-server',
            swarming_server,
            '-task-output-stdout=console',
            task_id,
        ]
        ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
        return ret

    def _RunLocallyIsolated(self, build_dir, target):
        cmd = [
            self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe),
            'run',
            '-i',
            self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
        ]
        if self.args.extra_args:
            cmd += ['--'] + self.args.extra_args
        ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
        return ret

    def CmdValidate(self, print_ok=True):
        errs = []

        # Read the file to make sure it parses.
        self.ReadConfigFile()

        # Build a list of all of the configs referenced by builders.
        all_configs = {}
        for builder_group in self.builder_groups:
            for config in self.builder_groups[builder_group].values():
                if isinstance(config, dict):
                    for c in config.values():
                        all_configs[c] = builder_group
                else:
                    all_configs[config] = builder_group

        # Check that every referenced args file or config actually exists.
        for config, loc in all_configs.items():
            if config.startswith('//'):
                if not self.Exists(self.ToAbsPath(config)):
                    errs.append('Unknown args file "%s" referenced from "%s".' %
                                (config, loc))
            elif not config in self.configs:
                errs.append('Unknown config "%s" referenced from "%s".' %
                            (config, loc))

        # Check that every actual config is actually referenced.
        for config in self.configs:
            if not config in all_configs:
                errs.append('Unused config "%s".' % config)

        # Figure out the whole list of mixins, and check that every mixin
        # listed by a config or another mixin actually exists.
        referenced_mixins = set()
        for config, mixins in self.configs.items():
            for mixin in mixins:
                if not mixin in self.mixins:
                    errs.append('Unknown mixin "%s" referenced by config "%s".'
                                % (mixin, config))
                referenced_mixins.add(mixin)

        for mixin in self.mixins:
            for sub_mixin in self.mixins[mixin].get('mixins', []):
                if not sub_mixin in self.mixins:
                    errs.append('Unknown mixin "%s" referenced by mixin "%s".'
                                % (sub_mixin, mixin))
                referenced_mixins.add(sub_mixin)

        # Check that every mixin defined is actually referenced somewhere.
        for mixin in self.mixins:
            if not mixin in referenced_mixins:
                errs.append('Unreferenced mixin "%s".' % mixin)

        if errs:
            raise MBErr(('mb config file %s has problems:' %
                         self.args.config_file) + '\n  ' + '\n  '.join(errs))

        if print_ok:
            self.Print('mb config file %s looks ok.' % self.args.config_file)
        return 0

    def GetConfig(self):
        build_dir = self.args.path[0]

        vals = self.DefaultVals()
        if self.args.builder or self.args.builder_group or self.args.config:
            vals = self.Lookup()
            # Re-run gn gen in order to ensure the config is consistent with
            # the build dir.
            self.RunGNGen(vals)
            return vals

        toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
                                       'toolchain.ninja')
        if not self.Exists(toolchain_path):
            self.Print('Must either specify a path to an existing GN build '
                       'dir or pass in a -m/-b pair or a -c flag to specify '
                       'the configuration')
            return {}

        vals['gn_args'] = self.GNArgsFromDir(build_dir)
        return vals

    def GNArgsFromDir(self, build_dir):
        args_contents = ""
        gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')
        if self.Exists(gn_args_path):
            args_contents = self.ReadFile(gn_args_path)
        gn_args = []
        for l in args_contents.splitlines():
            fields = l.split(' ')
            name = fields[0]
            val = ' '.join(fields[2:])
            gn_args.append('%s=%s' % (name, val))

        return ' '.join(gn_args)

    def Lookup(self):
        self.ReadConfigFile()
        config = self.ConfigFromArgs()
        if config.startswith('//'):
            if not self.Exists(self.ToAbsPath(config)):
                raise MBErr('args file "%s" not found' % config)
            vals = self.DefaultVals()
            vals['args_file'] = config
        else:
            if not config in self.configs:
                raise MBErr('Config "%s" not found in %s' %
                            (config, self.args.config_file))
            vals = self.FlattenConfig(config)
        return vals

    def ReadConfigFile(self):
        if not self.Exists(self.args.config_file):
            raise MBErr('config file not found at %s' % self.args.config_file)

        try:
            contents = ast.literal_eval(self.ReadFile(self.args.config_file))
        except SyntaxError as e:
            raise MBErr('Failed to parse config file "%s": %s' %
                        (self.args.config_file, e))

        self.configs = contents['configs']
        self.builder_groups = contents['builder_groups']
        self.mixins = contents['mixins']

    def ReadIsolateMap(self):
        isolate_map = self.args.isolate_map_file
        if not self.Exists(isolate_map):
            raise MBErr('isolate map file not found at %s' % isolate_map)
        try:
            return ast.literal_eval(self.ReadFile(isolate_map))
        except SyntaxError as e:
            raise MBErr(
                'Failed to parse isolate map file "%s": %s' % (isolate_map, e))

    def ConfigFromArgs(self):
        if self.args.config:
            if self.args.builder_group or self.args.builder:
                raise MBErr('Can not specific both -c/--config and '
                            '-m/--builder-group or -b/--builder')

            return self.args.config

        if not self.args.builder_group or not self.args.builder:
            raise MBErr('Must specify either -c/--config or '
                        '(-m/--builder-group and -b/--builder)')

        if not self.args.builder_group in self.builder_groups:
            raise MBErr('Master name "%s" not found in "%s"' %
                        (self.args.builder_group, self.args.config_file))

        if (not self.args.builder in
                self.builder_groups[self.args.builder_group]):
            raise MBErr(
                'Builder name "%s"  not found under builder_groups[%s] in "%s"'
                % (self.args.builder, self.args.builder_group,
                   self.args.config_file))

        config = (
            self.builder_groups[self.args.builder_group][self.args.builder]
        )
        if isinstance(config, dict):
            if self.args.phase is None:
                raise MBErr('Must specify a build --phase for %s on %s' %
                            (self.args.builder, self.args.builder_group))
            phase = str(self.args.phase)
            if phase not in config:
                raise MBErr('Phase %s doesn\'t exist for %s on %s' %
                            (phase, self.args.builder,
                             self.args.builder_group))
            return config[phase]

        if self.args.phase is not None:
            raise MBErr('Must not specify a build --phase for %s on %s' %
                        (self.args.builder, self.args.builder_group))
        return config

    def FlattenConfig(self, config):
        mixins = self.configs[config]
        vals = self.DefaultVals()

        visited = []
        self.FlattenMixins(mixins, vals, visited)
        return vals

    def DefaultVals(self):
        return {
            'args_file': '',
            'cros_passthrough': False,
            'gn_args': '',
        }

    def FlattenMixins(self, mixins, vals, visited):
        for m in mixins:
            if m not in self.mixins:
                raise MBErr('Unknown mixin "%s"' % m)

            visited.append(m)

            mixin_vals = self.mixins[m]

            if 'cros_passthrough' in mixin_vals:
                vals['cros_passthrough'] = mixin_vals['cros_passthrough']
            if 'gn_args' in mixin_vals:
                if vals['gn_args']:
                    vals['gn_args'] += ' ' + mixin_vals['gn_args']
                else:
                    vals['gn_args'] = mixin_vals['gn_args']

            if 'mixins' in mixin_vals:
                self.FlattenMixins(mixin_vals['mixins'], vals, visited)
        return vals

    def RunGNGen(self, vals):
        build_dir = self.args.path[0]

        cmd = self.GNCmd('gen', build_dir, '--check')
        gn_args = self.GNArgs(vals)

        # Since GN hasn't run yet, the build directory may not even exist.
        self.MaybeMakeDirectory(self.ToAbsPath(build_dir))

        gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
        self.WriteFile(gn_args_path, gn_args, force_verbose=True)

        swarming_targets = []
        if getattr(self.args, 'swarming_targets_file', None):
            # We need GN to generate the list of runtime dependencies for
            # the compile targets listed (one per line) in the file so
            # we can run them via swarming. We use gn_isolate_map.pyl to
            # convert the compile targets to the matching GN labels.
            path = self.args.swarming_targets_file
            if not self.Exists(path):
                self.WriteFailureAndRaise('"%s" does not exist' % path,
                                          output_path=None)
            contents = self.ReadFile(path)
            swarming_targets = set(contents.splitlines())

            isolate_map = self.ReadIsolateMap()
            err, labels = self.MapTargetsToLabels(
                isolate_map, swarming_targets)
            if err:
                raise MBErr(err)

            gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
            self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n')
            cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)

        ret, output, _ = self.Run(cmd)
        if ret:
            if self.args.json_output:
                # write errors to json.output
                self.WriteJSON({'output': output}, self.args.json_output)
            # If `gn gen` failed, we should exit early rather than trying to
            # generate isolates. Run() will have already logged any error
            # output.
            self.Print('GN gen failed: %d' % ret)
            return ret

        android = 'target_os="android"' in vals['gn_args']
        for target in swarming_targets:
            if android:
                # Android targets may be either android_apk or executable. The
                # former will result in runtime_deps associated with the stamp
                # file, while the latter will result in runtime_deps associated
                # with the executable.
                label = isolate_map[target]['label']
                runtime_deps_targets = [
                    target + '.runtime_deps',
                    'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
            elif isolate_map[target]['type'] == 'gpu_browser_test':
                if self.platform == 'win32':
                    runtime_deps_targets = ['browser_tests.exe.runtime_deps']
                else:
                    runtime_deps_targets = ['browser_tests.runtime_deps']
            elif isolate_map[target]['type'] == 'script':
                label = isolate_map[target]['label'].split(':')[1]
                runtime_deps_targets = [
                    '%s.runtime_deps' % label]
                if self.platform == 'win32':
                    runtime_deps_targets += [label + '.exe.runtime_deps']
                else:
                    runtime_deps_targets += [label + '.runtime_deps']
            elif self.platform == 'win32':
                runtime_deps_targets = [target + '.exe.runtime_deps']
            else:
                runtime_deps_targets = [target + '.runtime_deps']

            for r in runtime_deps_targets:
                runtime_deps_path = self.ToAbsPath(build_dir, r)
                if self.Exists(runtime_deps_path):
                    break
            else:
                raise MBErr('did not generate any of %s' %
                            ', '.join(runtime_deps_targets))

            command, extra_files = self.GetSwarmingCommand(target, vals)

            runtime_deps = self.ReadFile(runtime_deps_path).splitlines()

            self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
                                   extra_files)

        return 0

    def RunGNIsolate(self, vals):
        target = self.args.target[0]
        isolate_map = self.ReadIsolateMap()
        err, labels = self.MapTargetsToLabels(isolate_map, [target])
        if err:
            raise MBErr(err)
        label = labels[0]

        build_dir = self.args.path[0]
        command, extra_files = self.GetSwarmingCommand(target, vals)

        cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
        ret, out, _ = self.Call(cmd)
        if ret:
            if out:
                self.Print(out)
            return ret

        runtime_deps = out.splitlines()

        self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
                               extra_files)

        ret, _, _ = self.Run([
            self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe),
            'check',
            '-i',
            self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target))],
            buffer_output=False)

        return ret

    def WriteIsolateFiles(self, build_dir, command, target, runtime_deps,
                          extra_files):
        isolate_path = self.ToAbsPath(build_dir, target + '.isolate')
        self.WriteFile(isolate_path,
                       pprint.pformat({
                           'variables': {
                               'command': command,
                               'files': sorted(runtime_deps + extra_files),
                           }
                       }) + '\n')

        self.WriteJSON(
            {
                'args': [
                    '--isolate',
                    self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
                ],
                'dir': self.src_dir,
                'version': 1,
            },
            isolate_path + 'd.gen.json',
        )

    def MapTargetsToLabels(self, isolate_map, targets):
        labels = []
        err = ''

        def StripTestSuffixes(target):
            for suffix in ('_apk_run', '_apk', '_run'):
                if target.endswith(suffix):
                    return target[:-len(suffix)], suffix
            return None, None

        for target in targets:
            if target == 'all':
                labels.append(target)
            elif target.startswith('//'):
                labels.append(target)
            else:
                if target in isolate_map:
                    stripped_target, suffix = target, ''
                else:
                    stripped_target, suffix = StripTestSuffixes(target)
                if stripped_target in isolate_map:
                    if isolate_map[stripped_target]['type'] == 'unknown':
                        err += ('test target "%s" type is unknown\n' % target)
                    else:
                        labels.append(
                            isolate_map[stripped_target]['label'] + suffix)
                else:
                    err += ('target "%s" not found in '
                            '//testing/buildbot/gn_isolate_map.pyl\n' % target)

        return err, labels

    def GNCmd(self, subcommand, path, *args):
        if self.platform.startswith('linux'):
            subdir, exe = 'linux64', 'gn'
        elif self.platform == 'darwin':
            subdir, exe = 'mac', 'gn'
        else:
            subdir, exe = 'win', 'gn.exe'

        gn_path = self.PathJoin(self.src_dir, 'buildtools', subdir, exe)
        return [gn_path, subcommand, path] + list(args)

    def GNArgs(self, vals):
        if vals['cros_passthrough']:
            if not 'GN_ARGS' in os.environ:
                raise MBErr('MB is expecting GN_ARGS to be in the environment')
            gn_args = os.environ['GN_ARGS']
            if not re.search('target_os.*=.*"chromeos"', gn_args):
                raise MBErr('GN_ARGS is missing target_os = "chromeos": '
                            '(GN_ARGS=%s)' % gn_args)
        else:
            gn_args = vals['gn_args']

        if self.args.goma_dir:
            gn_args += ' goma_dir="%s"' % self.args.goma_dir

        android_version_code = self.args.android_version_code
        if android_version_code:
            gn_args += (' android_default_version_code="%s"' %
                        android_version_code)

        android_version_name = self.args.android_version_name
        if android_version_name:
            gn_args += (' android_default_version_name="%s"' %
                        android_version_name)

        # Canonicalize the arg string into a sorted, newline-separated list
        # of key-value pairs, and de-dup the keys if need be so that only
        # the last instance of each arg is listed.
        gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args))

        args_file = vals.get('args_file', None)
        if args_file:
            gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
        return gn_args

    def GetSwarmingCommand(self, target, vals):
        isolate_map = self.ReadIsolateMap()
        test_type = isolate_map[target]['type']

        is_android = 'target_os="android"' in vals['gn_args']
        is_linux = self.platform.startswith('linux') and not is_android

        if test_type == 'nontest':
            self.WriteFailureAndRaise('We should not be isolating %s.' %
                                      target, output_path=None)
        if test_type not in ('console_test_launcher', 'windowed_test_launcher',
                             'non_parallel_console_test_launcher', 'raw',
                             'additional_compile_target', 'junit_test',
                             'script'):
            self.WriteFailureAndRaise('No command line for '
                                      '%s found (test type %s).'
                                      % (target, test_type), output_path=None)

        cmdline = []
        extra_files = [
            '../../.vpython',
            '../../testing/test_env.py',
        ]
        vpython_exe = 'vpython'

        must_retry = False
        if test_type == 'script':
            cmdline += [vpython_exe,
                        '../../' +
                        self.ToSrcRelPath(isolate_map[target]['script'])]
        elif is_android:
            cmdline += [vpython_exe,
                        '../../build/android/test_wrapper/logdog_wrapper.py',
                        '--target', target,
                        '--logdog-bin-cmd', '../../bin/logdog_butler',
                        '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats',
                        '--store-tombstones']
        else:
            if test_type == 'raw':
                cmdline += [vpython_exe,
                            '../../tools_webrtc/flags_compatibility.py']
                extra_files.append('../../tools_webrtc/flags_compatibility.py')

            if isolate_map[target].get('use_webcam', False):
                cmdline += [vpython_exe,
                            '../../tools_webrtc/ensure_webcam_is_running.py']
                extra_files.append(
                    '../../tools_webrtc/ensure_webcam_is_running.py')

            # is_linux uses use_ozone and x11 by default.
            use_x11 = is_linux

            xvfb = use_x11 and test_type == 'windowed_test_launcher'
            if xvfb:
                cmdline += [vpython_exe, '../../testing/xvfb.py']
                extra_files.append('../../testing/xvfb.py')
            else:
                cmdline += [vpython_exe, '../../testing/test_env.py']

            if test_type != 'raw':
                extra_files += [
                    '../../third_party/gtest-parallel/gtest-parallel',
                    '../../third_party/gtest-parallel/gtest_parallel.py',
                    '../../tools_webrtc/gtest-parallel-wrapper.py',
                ]
                sep = '\\' if self.platform == 'win32' else '/'
                output_dir = '${ISOLATED_OUTDIR}' + sep + 'test_logs'
                test_results = '${ISOLATED_OUTDIR}' + sep + 'gtest_output.json'
                timeout = isolate_map[target].get('timeout', 900)
                cmdline += [
                    '../../tools_webrtc/gtest-parallel-wrapper.py',
                    '--output_dir=%s' % output_dir,
                    '--dump_json_test_results=%s' % test_results,
                    '--gtest_color=no',
                    # We tell gtest-parallel to interrupt the test after 900
                    # seconds, so it can exit cleanly and report results,
                    # instead of being interrupted by swarming and not
                    # reporting anything.
                    '--timeout=%s' % timeout,
                ]
                if test_type == 'non_parallel_console_test_launcher':
                    # Still use the gtest-parallel-wrapper.py script since we
                    # need it to run tests on swarming, but don't execute tests
                    # in parallel.
                    cmdline.append('--workers=1')
                must_retry = True

            asan = 'is_asan=true' in vals['gn_args']
            lsan = 'is_lsan=true' in vals['gn_args']
            msan = 'is_msan=true' in vals['gn_args']
            tsan = 'is_tsan=true' in vals['gn_args']
            sanitizer = asan or lsan or msan or tsan
            if must_retry and not sanitizer:
                # Retry would hide most sanitizers detections.
                cmdline.append('--retry_failed=3')

            executable_prefix = '.\\' if self.platform == 'win32' else './'
            executable_suffix = '.exe' if self.platform == 'win32' else ''
            executable = executable_prefix + target + executable_suffix

            cmdline.append(executable)

            cmdline.extend([
                '--asan=%d' % asan,
                '--lsan=%d' % lsan,
                '--msan=%d' % msan,
                '--tsan=%d' % tsan,
            ])

        cmdline += isolate_map[target].get('args', [])

        return cmdline, extra_files

    def ToAbsPath(self, build_path, *comps):
        return self.PathJoin(self.src_dir,
                             self.ToSrcRelPath(build_path),
                             *comps)

    def ToSrcRelPath(self, path):
        """Returns a relative path from the top of the repo."""
        if path.startswith('//'):
            return path[2:].replace('/', self.sep)
        return self.RelPath(path, self.src_dir)

    def RunGNAnalyze(self, vals):
        # Analyze runs before 'gn gen' now, so we need to run gn gen
        # in order to ensure that we have a build directory.
        ret = self.RunGNGen(vals)
        if ret:
            return ret

        build_path = self.args.path[0]
        input_path = self.args.input_path[0]
        gn_input_path = input_path + '.gn'
        output_path = self.args.output_path[0]
        gn_output_path = output_path + '.gn'

        inp = self.ReadInputJSON(['files', 'test_targets',
                                  'additional_compile_targets'])
        if self.args.verbose:
            self.Print()
            self.Print('analyze input:')
            self.PrintJSON(inp)
            self.Print()

        # This shouldn't normally happen, but could due to unusual race
        # conditions, like a try job that gets scheduled before a patch
        # lands but runs after the patch has landed.
        if not inp['files']:
            self.Print(
                'Warning: No files modified in patch, bailing out early.')
            self.WriteJSON({
                'status': 'No dependency',
                'compile_targets': [],
                'test_targets': [],
            }, output_path)
            return 0

        gn_inp = {}
        gn_inp['files'] = ['//' + f for f in inp['files']
                           if not f.startswith('//')]

        isolate_map = self.ReadIsolateMap()
        err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels(
            isolate_map, inp['additional_compile_targets'])
        if err:
            raise MBErr(err)

        err, gn_inp['test_targets'] = self.MapTargetsToLabels(
            isolate_map, inp['test_targets'])
        if err:
            raise MBErr(err)
        labels_to_targets = {}
        for i, label in enumerate(gn_inp['test_targets']):
            labels_to_targets[label] = inp['test_targets'][i]

        try:
            self.WriteJSON(gn_inp, gn_input_path)
            cmd = self.GNCmd('analyze', build_path,
                             gn_input_path, gn_output_path)
            ret, output, _ = self.Run(cmd, force_verbose=True)
            if ret:
                if self.args.json_output:
                    # write errors to json.output
                    self.WriteJSON({'output': output}, self.args.json_output)
                return ret

            gn_outp_str = self.ReadFile(gn_output_path)
            try:
                gn_outp = json.loads(gn_outp_str)
            except Exception as e:
                self.Print("Failed to parse the JSON string GN "
                           "returned: %s\n%s" % (repr(gn_outp_str), str(e)))
                raise

            outp = {}
            if 'status' in gn_outp:
                outp['status'] = gn_outp['status']
            if 'error' in gn_outp:
                outp['error'] = gn_outp['error']
            if 'invalid_targets' in gn_outp:
                outp['invalid_targets'] = gn_outp['invalid_targets']
            if 'compile_targets' in gn_outp:
                if 'all' in gn_outp['compile_targets']:
                    outp['compile_targets'] = ['all']
                else:
                    outp['compile_targets'] = [
                        label.replace('//', '')
                        for label in gn_outp['compile_targets']
                    ]
            if 'test_targets' in gn_outp:
                outp['test_targets'] = [
                    labels_to_targets[label]
                    for label in gn_outp['test_targets']
                ]

            if self.args.verbose:
                self.Print()
                self.Print('analyze output:')
                self.PrintJSON(outp)
                self.Print()

            self.WriteJSON(outp, output_path)

        finally:
            if self.Exists(gn_input_path):
                self.RemoveFile(gn_input_path)
            if self.Exists(gn_output_path):
                self.RemoveFile(gn_output_path)

        return 0

    def ReadInputJSON(self, required_keys):
        path = self.args.input_path[0]
        output_path = self.args.output_path[0]
        if not self.Exists(path):
            self.WriteFailureAndRaise(
                '"%s" does not exist' % path, output_path)

        try:
            inp = json.loads(self.ReadFile(path))
        except Exception as e:
            self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s'
                                      % (path, e), output_path)

        for k in required_keys:
            if not k in inp:
                self.WriteFailureAndRaise('input file is missing a "%s" key' %
                                          k, output_path)

        return inp

    def WriteFailureAndRaise(self, msg, output_path):
        if output_path:
            self.WriteJSON({'error': msg}, output_path, force_verbose=True)
        raise MBErr(msg)

    def WriteJSON(self, obj, path, force_verbose=False):
        try:
            self.WriteFile(path,
                           json.dumps(obj, indent=2, sort_keys=True) + '\n',
                           force_verbose=force_verbose)
        except Exception as e:
            raise MBErr('Error %s writing to the output path "%s"' %
                        (e, path))

    def PrintCmd(self, cmd, env):
        if self.platform == 'win32':
            env_prefix = 'set '
            env_quoter = QuoteForSet
            shell_quoter = QuoteForCmd
        else:
            env_prefix = ''
            env_quoter = pipes.quote
            shell_quoter = pipes.quote

        def print_env(var):
            if env and var in env:
                self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var])))

        print_env('LLVM_FORCE_HEAD_REVISION')

        if cmd[0] == self.executable:
            cmd = ['python'] + cmd[1:]
        self.Print(*[shell_quoter(arg) for arg in cmd])

    def PrintJSON(self, obj):
        self.Print(json.dumps(obj, indent=2, sort_keys=True))

    def Build(self, target):
        build_dir = self.ToSrcRelPath(self.args.path[0])
        ninja_cmd = ['ninja', '-C', build_dir]
        if self.args.jobs:
            ninja_cmd.extend(['-j', '%d' % self.args.jobs])
        ninja_cmd.append(target)
        ret, _, _ = self.Run(
            ninja_cmd, force_verbose=False, buffer_output=False)
        return ret

    def Run(self, cmd, env=None, force_verbose=True, buffer_output=True):
        # This function largely exists so it can be overridden for testing.
        if self.args.dryrun or self.args.verbose or force_verbose:
            self.PrintCmd(cmd, env)
        if self.args.dryrun:
            return 0, '', ''

        ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output)
        if self.args.verbose or force_verbose:
            if ret:
                self.Print('  -> returned %d' % ret)
            if out:
                self.Print(out, end='')
            if err:
                self.Print(err, end='', file=sys.stderr)
        return ret, out, err

    def Call(self, cmd, env=None, buffer_output=True):
        if buffer_output:
            p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 env=env)
            out, err = p.communicate()
        else:
            p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir,
                                 env=env)
            p.wait()
            out = err = ''
        return p.returncode, out, err

    def ExpandUser(self, path):
        # This function largely exists so it can be overridden for testing.
        return os.path.expanduser(path)

    def Exists(self, path):
        # This function largely exists so it can be overridden for testing.
        return os.path.exists(path)

    def Fetch(self, url):
        # This function largely exists so it can be overridden for testing.
        f = urlopen(url)
        contents = f.read()
        f.close()
        return contents

    def MaybeMakeDirectory(self, path):
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

    def PathJoin(self, *comps):
        # This function largely exists so it can be overriden for testing.
        return os.path.join(*comps)

    def Print(self, *args, **kwargs):
        # This function largely exists so it can be overridden for testing.
        print(*args, **kwargs)
        if kwargs.get('stream', sys.stdout) == sys.stdout:
            sys.stdout.flush()

    def ReadFile(self, path):
        # This function largely exists so it can be overriden for testing.
        with open(path) as fp:
            return fp.read()

    def RelPath(self, path, start='.'):
        # This function largely exists so it can be overriden for testing.
        return os.path.relpath(path, start)

    def RemoveFile(self, path):
        # This function largely exists so it can be overriden for testing.
        os.remove(path)

    def RemoveDirectory(self, abs_path):
        if self.platform == 'win32':
            # In other places in chromium, we often have to retry this command
            # because we're worried about other processes still holding on to
            # file handles, but when MB is invoked, it will be early enough in
            # the build that their should be no other processes to interfere.
            # We can change this if need be.
            self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
        else:
            shutil.rmtree(abs_path, ignore_errors=True)

    def TempDir(self):
        # This function largely exists so it can be overriden for testing.
        return tempfile.mkdtemp(prefix='mb_')

    def TempFile(self, mode='w'):
        # This function largely exists so it can be overriden for testing.
        return tempfile.NamedTemporaryFile(mode=mode, delete=False)

    def WriteFile(self, path, contents, force_verbose=False):
        # This function largely exists so it can be overriden for testing.
        if self.args.dryrun or self.args.verbose or force_verbose:
            self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
        with open(path, 'w') as fp:
            return fp.write(contents)


class MBErr(Exception):
    pass


# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful
# details of this next section, which handles escaping command lines
# so that they can be copied and pasted into a cmd window.
UNSAFE_FOR_SET = set('^<>&|')
UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%'))
ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"'))


def QuoteForSet(arg):
    if any(a in UNSAFE_FOR_SET for a in arg):
        arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg)
    return arg


def QuoteForCmd(arg):
    # First, escape the arg so that CommandLineToArgvW will parse it properly.
    if arg == '' or ' ' in arg or '"' in arg:
        quote_re = re.compile(r'(\\*)"')
        arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))

    # Then check to see if the arg contains any metacharacters other than
    # double quotes; if it does, quote everything (including the double
    # quotes) for safety.
    if any(a in UNSAFE_FOR_CMD for a in arg):
        arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
    return arg


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))