git_command.py

# -*- coding:utf-8 -*-
#
# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import os
import re
import sys
import subprocess
import tempfile
from signal import SIGTERM

from error import GitError
from git_refs import HEAD
import platform_utils
from repo_trace import REPO_TRACE, IsTrace, Trace
from wrapper import Wrapper

GIT = 'git'
# NB: These do not need to be kept in sync with the repo launcher script.
# These may be much newer as it allows the repo launcher to roll between
# different repo releases while source versions might require a newer git.
#
# The soft version is when we start warning users that the version is old and
# we'll be dropping support for it.  We'll refuse to work with versions older
# than the hard version.
#
# git-1.7 is in (EOL) Ubuntu Precise.  git-1.9 is in Ubuntu Trusty.
MIN_GIT_VERSION_SOFT = (1, 9, 1)
MIN_GIT_VERSION_HARD = (1, 7, 2)
GIT_DIR = 'GIT_DIR'

LAST_GITDIR = None
LAST_CWD = None

_ssh_proxy_path = None
_ssh_sock_path = None
_ssh_clients = []
_ssh_version = None


def _run_ssh_version():
  """run ssh -V to display the version number"""
  return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()


def _parse_ssh_version(ver_str=None):
  """parse a ssh version string into a tuple"""
  if ver_str is None:
    ver_str = _run_ssh_version()
  m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
  if m:
    return tuple(int(x) for x in m.group(1).split('.'))
  else:
    return ()


def ssh_version():
  """return ssh version as a tuple"""
  global _ssh_version
  if _ssh_version is None:
    try:
      _ssh_version = _parse_ssh_version()
    except subprocess.CalledProcessError:
      print('fatal: unable to detect ssh version', file=sys.stderr)
      sys.exit(1)
  return _ssh_version


def ssh_sock(create=True):
  global _ssh_sock_path
  if _ssh_sock_path is None:
    if not create:
      return None
    tmp_dir = '/tmp'
    if not os.path.exists(tmp_dir):
      tmp_dir = tempfile.gettempdir()
    if ssh_version() < (6, 7):
      tokens = '%r@%h:%p'
    else:
      tokens = '%C'  # hash of %l%h%p%r
    _ssh_sock_path = os.path.join(
        tempfile.mkdtemp('', 'ssh-', tmp_dir),
        'master-' + tokens)
  return _ssh_sock_path


def _ssh_proxy():
  global _ssh_proxy_path
  if _ssh_proxy_path is None:
    _ssh_proxy_path = os.path.join(
        os.path.dirname(__file__),
        'git_ssh')
  return _ssh_proxy_path


def _add_ssh_client(p):
  _ssh_clients.append(p)


def _remove_ssh_client(p):
  try:
    _ssh_clients.remove(p)
  except ValueError:
    pass


def terminate_ssh_clients():
  global _ssh_clients
  for p in _ssh_clients:
    try:
      os.kill(p.pid, SIGTERM)
      p.wait()
    except OSError:
      pass
  _ssh_clients = []


_git_version = None


class _GitCall(object):
  def version_tuple(self):
    global _git_version
    if _git_version is None:
      _git_version = Wrapper().ParseGitVersion()
      if _git_version is None:
        print('fatal: unable to detect git version', file=sys.stderr)
        sys.exit(1)
    return _git_version

  def __getattr__(self, name):
    name = name.replace('_', '-')

    def fun(*cmdv):
      command = [name]
      command.extend(cmdv)
      return GitCommand(None, command).Wait() == 0
    return fun


git = _GitCall()


def RepoSourceVersion():
  """Return the version of the repo.git tree."""
  ver = getattr(RepoSourceVersion, 'version', None)

  # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
  # to initialize version info we provide.
  if ver is None:
    env = GitCommand._GetBasicEnv()

    proj = os.path.dirname(os.path.abspath(__file__))
    env[GIT_DIR] = os.path.join(proj, '.git')

    p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
                         env=env)
    if p.wait() == 0:
      ver = p.stdout.read().strip().decode('utf-8')
      if ver.startswith('v'):
        ver = ver[1:]
    else:
      ver = 'unknown'
    setattr(RepoSourceVersion, 'version', ver)

  return ver


class UserAgent(object):
  """Mange User-Agent settings when talking to external services

  We follow the style as documented here:
  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
  """

  _os = None
  _repo_ua = None
  _git_ua = None

  @property
  def os(self):
    """The operating system name."""
    if self._os is None:
      os_name = sys.platform
      if os_name.lower().startswith('linux'):
        os_name = 'Linux'
      elif os_name == 'win32':
        os_name = 'Win32'
      elif os_name == 'cygwin':
        os_name = 'Cygwin'
      elif os_name == 'darwin':
        os_name = 'Darwin'
      self._os = os_name

    return self._os

  @property
  def repo(self):
    """The UA when connecting directly from repo."""
    if self._repo_ua is None:
      py_version = sys.version_info
      self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
          RepoSourceVersion(),
          self.os,
          git.version_tuple().full,
          py_version.major, py_version.minor, py_version.micro)

    return self._repo_ua

  @property
  def git(self):
    """The UA when running git."""
    if self._git_ua is None:
      self._git_ua = 'git/%s (%s) git-repo/%s' % (
          git.version_tuple().full,
          self.os,
          RepoSourceVersion())

    return self._git_ua


user_agent = UserAgent()


def git_require(min_version, fail=False, msg=''):
  git_version = git.version_tuple()
  if min_version <= git_version:
    return True
  if fail:
    need = '.'.join(map(str, min_version))
    if msg:
      msg = ' for ' + msg
    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
    sys.exit(1)
  return False


class GitCommand(object):
  def __init__(self,
               project,
               cmdv,
               bare=False,
               provide_stdin=False,
               capture_stdout=False,
               capture_stderr=False,
               merge_output=False,
               disable_editor=False,
               ssh_proxy=False,
               cwd=None,
               gitdir=None):
    env = self._GetBasicEnv()

    # If we are not capturing std* then need to print it.
    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}

    if disable_editor:
      env['GIT_EDITOR'] = ':'
    if ssh_proxy:
      env['REPO_SSH_SOCK'] = ssh_sock()
      env['GIT_SSH'] = _ssh_proxy()
      env['GIT_SSH_VARIANT'] = 'ssh'
    if 'http_proxy' in env and 'darwin' == sys.platform:
      s = "'http.proxy=%s'" % (env['http_proxy'],)
      p = env.get('GIT_CONFIG_PARAMETERS')
      if p is not None:
        s = p + ' ' + s
      env['GIT_CONFIG_PARAMETERS'] = s
    if 'GIT_ALLOW_PROTOCOL' not in env:
      env['GIT_ALLOW_PROTOCOL'] = (
          'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
    env['GIT_HTTP_USER_AGENT'] = user_agent.git

    if project:
      if not cwd:
        cwd = project.worktree
      if not gitdir:
        gitdir = project.gitdir

    command = [GIT]
    if bare:
      if gitdir:
        env[GIT_DIR] = gitdir
      cwd = None
    command.append(cmdv[0])
    # Need to use the --progress flag for fetch/clone so output will be
    # displayed as by default git only does progress output if stderr is a TTY.
    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
      if '--progress' not in cmdv and '--quiet' not in cmdv:
        command.append('--progress')
    command.extend(cmdv[1:])

    if provide_stdin:
      stdin = subprocess.PIPE
    else:
      stdin = None

    stdout = subprocess.PIPE
    stderr = subprocess.STDOUT if merge_output else subprocess.PIPE

    if IsTrace():
      global LAST_CWD
      global LAST_GITDIR

      dbg = ''

      if cwd and LAST_CWD != cwd:
        if LAST_GITDIR or LAST_CWD:
          dbg += '\n'
        dbg += ': cd %s\n' % cwd
        LAST_CWD = cwd

      if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
        if LAST_GITDIR or LAST_CWD:
          dbg += '\n'
        dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
        LAST_GITDIR = env[GIT_DIR]

      dbg += ': '
      dbg += ' '.join(command)
      if stdin == subprocess.PIPE:
        dbg += ' 0<|'
      if stdout == subprocess.PIPE:
        dbg += ' 1>|'
      if stderr == subprocess.PIPE:
        dbg += ' 2>|'
      elif stderr == subprocess.STDOUT:
        dbg += ' 2>&1'
      Trace('%s', dbg)

    try:
      p = subprocess.Popen(command,
                           cwd=cwd,
                           env=env,
                           stdin=stdin,
                           stdout=stdout,
                           stderr=stderr)
    except Exception as e:
      raise GitError('%s: %s' % (command[1], e))

    if ssh_proxy:
      _add_ssh_client(p)

    self.process = p
    self.stdin = p.stdin

  @staticmethod
  def _GetBasicEnv():
    """Return a basic env for running git under.

    This is guaranteed to be side-effect free.
    """
    env = os.environ.copy()
    for key in (REPO_TRACE,
                GIT_DIR,
                'GIT_ALTERNATE_OBJECT_DIRECTORIES',
                'GIT_OBJECT_DIRECTORY',
                'GIT_WORK_TREE',
                'GIT_GRAFT_FILE',
                'GIT_INDEX_FILE'):
      env.pop(key, None)
    return env

  def Wait(self):
    try:
      p = self.process
      rc = self._CaptureOutput()
    finally:
      _remove_ssh_client(p)
    return rc

  def _CaptureOutput(self):
    p = self.process
    s_in = platform_utils.FileDescriptorStreams.create()
    s_in.add(p.stdout, sys.stdout, 'stdout')
    if p.stderr is not None:
      s_in.add(p.stderr, sys.stderr, 'stderr')
    self.stdout = ''
    self.stderr = ''

    while not s_in.is_done:
      in_ready = s_in.select()
      for s in in_ready:
        buf = s.read()
        if not buf:
          s_in.remove(s)
          continue
        if not hasattr(buf, 'encode'):
          buf = buf.decode()
        if s.std_name == 'stdout':
          self.stdout += buf
        else:
          self.stderr += buf
        if self.tee[s.std_name]:
          s.dest.write(buf)
          s.dest.flush()
    return p.wait()