setup_links.py

#!/usr/bin/env python
# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS.  All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.

"""Setup links to a Chromium checkout for WebRTC.

WebRTC standalone shares a lot of dependencies and build tools with Chromium.
To do this, many of the paths of a Chromium checkout is emulated by creating
symlinks to files and directories. This script handles the setup of symlinks to
achieve this.

It also handles cleanup of the legacy Subversion-based approach that was used
before Chrome switched over their master repo from Subversion to Git.
"""


import ctypes
import errno
import logging
import optparse
import os
import shelve
import shutil
import subprocess
import sys
import textwrap


DIRECTORIES = [
  'build',
  'buildtools',
  'google_apis',  # Needed by build/common.gypi.
  'net',
  'testing',
  'third_party/binutils',
  'third_party/boringssl',
  'third_party/colorama',
  'third_party/drmemory',
  'third_party/expat',
  'third_party/icu',
  'third_party/instrumented_libraries',
  'third_party/jsoncpp',
  'third_party/libjpeg',
  'third_party/libjpeg_turbo',
  'third_party/libsrtp',
  'third_party/libudev',
  'third_party/libvpx',
  'third_party/libyuv',
  'third_party/llvm-build',
  'third_party/nss',
  'third_party/ocmock',
  'third_party/openmax_dl',
  'third_party/opus',
  'third_party/protobuf',
  'third_party/sqlite',
  'third_party/syzygy',
  'third_party/usrsctp',
  'third_party/yasm',
  'third_party/zlib',
  'tools/clang',
  'tools/generate_library_loader',
  'tools/gn',
  'tools/gyp',
  'tools/memory',
  'tools/protoc_wrapper',
  'tools/python',
  'tools/swarming_client',
  'tools/valgrind',
  'tools/vim',
  'tools/win',
]

from sync_chromium import get_target_os_list
target_os = get_target_os_list()
if 'android' in target_os:
  DIRECTORIES += [
    'base',
    'third_party/android_platform',
    'third_party/android_testrunner',
    'third_party/android_tools',
    'third_party/appurify-python',
    'third_party/ashmem',
    'third_party/jsr-305',
    'third_party/junit',
    'third_party/libevent',
    'third_party/libxml',
    'third_party/mockito',
    'third_party/modp_b64',
    'third_party/requests',
    'third_party/robolectric',
    'tools/android',
    'tools/grit',
    'tools/relocation_packer'
  ]
if 'ios' in target_os:
  DIRECTORIES.append('third_party/class-dump')

FILES = {
  'tools/find_depot_tools.py': None,
  'third_party/BUILD.gn': None,
}

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
LINKS_DB = 'links'

# Version management to make future upgrades/downgrades easier to support.
SCHEMA_VERSION = 1


def query_yes_no(question, default=False):
  """Ask a yes/no question via raw_input() and return their answer.

  Modified from http://stackoverflow.com/a/3041990.
  """
  prompt = " [%s/%%s]: "
  prompt = prompt % ('Y' if default is True  else 'y')
  prompt = prompt % ('N' if default is False else 'n')

  if default is None:
    default = 'INVALID'

  while True:
    sys.stdout.write(question + prompt)
    choice = raw_input().lower()
    if choice == '' and default != 'INVALID':
      return default

    if 'yes'.startswith(choice):
      return True
    elif 'no'.startswith(choice):
      return False

    print "Please respond with 'yes' or 'no' (or 'y' or 'n')."


# Actions
class Action(object):
  def __init__(self, dangerous):
    self.dangerous = dangerous

  def announce(self, planning):
    """Log a description of this action.

    Args:
      planning - True iff we're in the planning stage, False if we're in the
                 doit stage.
    """
    pass

  def doit(self, links_db):
    """Execute the action, recording what we did to links_db, if necessary."""
    pass


class Remove(Action):
  def __init__(self, path, dangerous):
    super(Remove, self).__init__(dangerous)
    self._priority = 0
    self._path = path

  def announce(self, planning):
    log = logging.warn
    filesystem_type = 'file'
    if not self.dangerous:
      log = logging.info
      filesystem_type = 'link'
    if planning:
      log('Planning to remove %s: %s', filesystem_type, self._path)
    else:
      log('Removing %s: %s', filesystem_type, self._path)

  def doit(self, _):
    os.remove(self._path)


class Rmtree(Action):
  def __init__(self, path):
    super(Rmtree, self).__init__(dangerous=True)
    self._priority = 0
    self._path = path

  def announce(self, planning):
    if planning:
      logging.warn('Planning to remove directory: %s', self._path)
    else:
      logging.warn('Removing directory: %s', self._path)

  def doit(self, _):
    if sys.platform.startswith('win'):
      # shutil.rmtree() doesn't work on Windows if any of the directories are
      # read-only, which svn repositories are.
      subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
    else:
      shutil.rmtree(self._path)


class Makedirs(Action):
  def __init__(self, path):
    super(Makedirs, self).__init__(dangerous=False)
    self._priority = 1
    self._path = path

  def doit(self, _):
    try:
      os.makedirs(self._path)
    except OSError as e:
      if e.errno != errno.EEXIST:
        raise


class Symlink(Action):
  def __init__(self, source_path, link_path):
    super(Symlink, self).__init__(dangerous=False)
    self._priority = 2
    self._source_path = source_path
    self._link_path = link_path

  def announce(self, planning):
    if planning:
      logging.info(
          'Planning to create link from %s to %s', self._link_path,
          self._source_path)
    else:
      logging.debug(
          'Linking from %s to %s', self._link_path, self._source_path)

  def doit(self, links_db):
    # Files not in the root directory need relative path calculation.
    # On Windows, use absolute paths instead since NTFS doesn't seem to support
    # relative paths for symlinks.
    if sys.platform.startswith('win'):
      source_path = os.path.abspath(self._source_path)
    else:
      if os.path.dirname(self._link_path) != self._link_path:
        source_path = os.path.relpath(self._source_path,
                                      os.path.dirname(self._link_path))

    os.symlink(source_path, os.path.abspath(self._link_path))
    links_db[self._source_path] = self._link_path


class LinkError(IOError):
  """Failed to create a link."""
  pass


# Handles symlink creation on the different platforms.
if sys.platform.startswith('win'):
  def symlink(source_path, link_path):
    flag = 1 if os.path.isdir(source_path) else 0
    if not ctypes.windll.kernel32.CreateSymbolicLinkW(
        unicode(link_path), unicode(source_path), flag):
      raise OSError('Failed to create symlink to %s. Notice that only NTFS '
                    'version 5.0 and up has all the needed APIs for '
                    'creating symlinks.' % source_path)
  os.symlink = symlink


class WebRTCLinkSetup(object):
  def __init__(self, links_db, force=False, dry_run=False, prompt=False):
    self._force = force
    self._dry_run = dry_run
    self._prompt = prompt
    self._links_db = links_db

  def CreateLinks(self, on_bot):
    logging.debug('CreateLinks')
    # First, make a plan of action
    actions = []

    for source_path, link_path in FILES.iteritems():
      actions += self._ActionForPath(
          source_path, link_path, check_fn=os.path.isfile, check_msg='files')
    for source_dir in DIRECTORIES:
      actions += self._ActionForPath(
          source_dir, None, check_fn=os.path.isdir,
          check_msg='directories')

    if not on_bot and self._force:
      # When making the manual switch from legacy SVN checkouts to the new
      # Git-based Chromium DEPS, the .gclient_entries file that contains cached
      # URLs for all DEPS entries must be removed to avoid future sync problems.
      entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
      if os.path.exists(entries_file):
        actions.append(Remove(entries_file, dangerous=True))

    actions.sort()

    if self._dry_run:
      for action in actions:
        action.announce(planning=True)
      logging.info('Not doing anything because dry-run was specified.')
      sys.exit(0)

    if any(a.dangerous for a in actions):
      logging.warn('Dangerous actions:')
      for action in (a for a in actions if a.dangerous):
        action.announce(planning=True)
      print

      if not self._force:
        logging.error(textwrap.dedent("""\
        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                              A C T I O N     R E Q I R E D
        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

        Because chromium/src is transitioning to Git (from SVN), we needed to
        change the way that the WebRTC standalone checkout works. Instead of
        individually syncing subdirectories of Chromium in SVN, we're now
        syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
        into the `chromium/src` directory.

        As such, all Chromium directories which are currently pulled by DEPS are
        now replaced with a symlink into the full Chromium checkout.

        To avoid disrupting developers, we've chosen to not delete your
        directories forcibly, in case you have some work in progress in one of
        them :).

        ACTION REQUIRED:
        Before running `gclient sync|runhooks` again, you must run:
        %s%s --force

        Which will replace all directories which now must be symlinks, after
        prompting with a summary of the work-to-be-done.
        """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
        sys.exit(1)
      elif self._prompt:
        if not query_yes_no('Would you like to perform the above plan?'):
          sys.exit(1)

    for action in actions:
      action.announce(planning=False)
      action.doit(self._links_db)

    if not on_bot and self._force:
      logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
                   'let the remaining hooks (that probably were interrupted) '
                   'execute.')

  def CleanupLinks(self):
    logging.debug('CleanupLinks')
    for source, link_path  in self._links_db.iteritems():
      if source == 'SCHEMA_VERSION':
        continue
      if os.path.islink(link_path) or sys.platform.startswith('win'):
        # os.path.islink() always returns false on Windows
        # See http://bugs.python.org/issue13143.
        logging.debug('Removing link to %s at %s', source, link_path)
        if not self._dry_run:
          if os.path.exists(link_path):
            if sys.platform.startswith('win') and os.path.isdir(link_path):
              subprocess.check_call(['rmdir', '/q', '/s', link_path],
                                    shell=True)
            else:
              os.remove(link_path)
          del self._links_db[source]

  @staticmethod
  def _ActionForPath(source_path, link_path=None, check_fn=None,
                     check_msg=None):
    """Create zero or more Actions to link to a file or directory.

    This will be a symlink on POSIX platforms. On Windows this requires
    that NTFS is version 5.0 or higher (Vista or newer).

    Args:
      source_path: Path relative to the Chromium checkout root.
        For readability, the path may contain slashes, which will
        automatically be converted to the right path delimiter on Windows.
      link_path: The location for the link to create. If omitted it will be the
        same path as source_path.
      check_fn: A function returning true if the type of filesystem object is
        correct for the attempted call. Otherwise an error message with
        check_msg will be printed.
      check_msg: String used to inform the user of an invalid attempt to create
        a file.
    Returns:
      A list of Action objects.
    """
    def fix_separators(path):
      if sys.platform.startswith('win'):
        return path.replace(os.altsep, os.sep)
      else:
        return path

    assert check_fn
    assert check_msg
    link_path = link_path or source_path
    link_path = fix_separators(link_path)

    source_path = fix_separators(source_path)
    source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
    if os.path.exists(source_path) and not check_fn:
      raise LinkError('_LinkChromiumPath can only be used to link to %s: '
                      'Tried to link to: %s' % (check_msg, source_path))

    if not os.path.exists(source_path):
      logging.debug('Silently ignoring missing source: %s. This is to avoid '
                    'errors on platform-specific dependencies.', source_path)
      return []

    actions = []

    if os.path.exists(link_path) or os.path.islink(link_path):
      if os.path.islink(link_path):
        actions.append(Remove(link_path, dangerous=False))
      elif os.path.isfile(link_path):
        actions.append(Remove(link_path, dangerous=True))
      elif os.path.isdir(link_path):
        actions.append(Rmtree(link_path))
      else:
        raise LinkError('Don\'t know how to plan: %s' % link_path)

    # Create parent directories to the target link if needed.
    target_parent_dirs = os.path.dirname(link_path)
    if (target_parent_dirs and
        target_parent_dirs != link_path and
        not os.path.exists(target_parent_dirs)):
      actions.append(Makedirs(target_parent_dirs))

    actions.append(Symlink(source_path, link_path))

    return actions

def _initialize_database(filename):
  links_database = shelve.open(filename)

  # Wipe the database if this version of the script ends up looking at a
  # newer (future) version of the links db, just to be sure.
  version = links_database.get('SCHEMA_VERSION')
  if version and version != SCHEMA_VERSION:
    logging.info('Found database with schema version %s while this script only '
                 'supports %s. Wiping previous database contents.', version,
                 SCHEMA_VERSION)
    links_database.clear()
  links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
  return links_database


def main():
  on_bot = os.environ.get('CHROME_HEADLESS') == '1'

  parser = optparse.OptionParser()
  parser.add_option('-d', '--dry-run', action='store_true', default=False,
                    help='Print what would be done, but don\'t perform any '
                         'operations. This will automatically set logging to '
                         'verbose.')
  parser.add_option('-c', '--clean-only', action='store_true', default=False,
                    help='Only clean previously created links, don\'t create '
                         'new ones. This will automatically set logging to '
                         'verbose.')
  parser.add_option('-f', '--force', action='store_true', default=on_bot,
                    help='Force link creation. CAUTION: This deletes existing '
                         'folders and files in the locations where links are '
                         'about to be created.')
  parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
                    default=(not on_bot),
                    help='Prompt if we\'re planning to do a dangerous action')
  parser.add_option('-v', '--verbose', action='store_const',
                    const=logging.DEBUG, default=logging.INFO,
                    help='Print verbose output for debugging.')
  options, _ = parser.parse_args()

  if options.dry_run or options.force or options.clean_only:
    options.verbose = logging.DEBUG
  logging.basicConfig(format='%(message)s', level=options.verbose)

  # Work from the root directory of the checkout.
  script_dir = os.path.dirname(os.path.abspath(__file__))
  os.chdir(script_dir)

  if sys.platform.startswith('win'):
    def is_admin():
      try:
        return os.getuid() == 0
      except AttributeError:
        return ctypes.windll.shell32.IsUserAnAdmin() != 0
    if not is_admin():
      logging.error('On Windows, you now need to have administrator '
                    'privileges for the shell running %s (or '
                    '`gclient sync|runhooks`).\nPlease start another command '
                    'prompt as Administrator and try again.', sys.argv[0])
      return 1

  if not os.path.exists(CHROMIUM_CHECKOUT):
    logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
                  'sync" before running this script?', CHROMIUM_CHECKOUT)
    return 2

  links_database = _initialize_database(LINKS_DB)
  try:
    symlink_creator = WebRTCLinkSetup(links_database, options.force,
                                      options.dry_run, options.prompt)
    symlink_creator.CleanupLinks()
    if not options.clean_only:
      symlink_creator.CreateLinks(on_bot)
  except LinkError as e:
    print >> sys.stderr, e.message
    return 3
  finally:
    links_database.close()
  return 0


if __name__ == '__main__':
  sys.exit(main())