PRESUBMIT.py

# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""Top-level presubmit script for Skia.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import collections
import csv
import fnmatch
import os
import re
import subprocess
import sys
import traceback


REVERT_CL_SUBJECT_PREFIX = 'Revert '

SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'

# Please add the complete email address here (and not just 'xyz@' or 'xyz').
PUBLIC_API_OWNERS = (
    'reed@chromium.org',
    'reed@google.com',
    'bsalomon@chromium.org',
    'bsalomon@google.com',
    'djsollen@chromium.org',
    'djsollen@google.com',
    'hcm@chromium.org',
    'hcm@google.com',
)

AUTHORS_FILE_NAME = 'AUTHORS'

DOCS_PREVIEW_URL = 'https://skia.org/?cl='
GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='

# Path to CQ bots feature is described in https://bug.skia.org/4364
PATH_PREFIX_TO_EXTRA_TRYBOTS = {
    'src/opts/': ('skia.primary:'
      'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SKNX_NO_SIMD'),
    'include/private/SkAtomics.h': ('skia.primary:'
      'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN,'
      'Test-Ubuntu17-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-TSAN'
    ),

    # Below are examples to show what is possible with this feature.
    # 'src/svg/': 'master1:abc;master2:def',
    # 'src/svg/parser/': 'master3:ghi,jkl;master4:mno',
    # 'src/image/SkImage_Base.h': 'master5:pqr,stu;master1:abc1;master2:def',
}

SERVICE_ACCOUNT_SUFFIX = [
    '@%s.iam.gserviceaccount.com' % project for project in [
        'skia-buildbots.google.com', 'skia-swarming-bots']]


def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
  """Checks that files end with atleast one \n (LF)."""
  eof_files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    contents = input_api.ReadFile(f, 'rb')
    # Check that the file ends in atleast one newline character.
    if len(contents) > 1 and contents[-1:] != '\n':
      eof_files.append(f.LocalPath())

  if eof_files:
    return [output_api.PresubmitPromptWarning(
      'These files should end in a newline character:',
      items=eof_files)]
  return []


def _PythonChecks(input_api, output_api):
  """Run checks on any modified Python files."""
  blacklist = [
      r'infra[\\\/]bots[\\\/]recipes.py',

      # Blacklist DEPS. Those under third_party are already covered by
      # input_api.DEFAULT_BLACK_LIST.
      r'common[\\\/].*',
      r'buildtools[\\\/].*',
      r'.*[\\\/]\.recipe_deps[\\\/].*',
  ]
  blacklist.extend(input_api.DEFAULT_BLACK_LIST)

  pylint_disabled_warnings = (
      'F0401',  # Unable to import.
      'E0611',  # No name in module.
      'W0232',  # Class has no __init__ method.
      'E1002',  # Use of super on an old style class.
      'W0403',  # Relative import used.
      'R0201',  # Method could be a function.
      'E1003',  # Using class name in super.
      'W0613',  # Unused argument.
      'W0105',  # String statement has no effect.
  )
  return input_api.canned_checks.RunPylint(
      input_api, output_api,
      disabled_warnings=pylint_disabled_warnings,
      black_list=blacklist)


def _JsonChecks(input_api, output_api):
  """Run checks on any modified json files."""
  failing_files = []
  for affected_file in input_api.AffectedFiles(None):
    affected_file_path = affected_file.LocalPath()
    is_json = affected_file_path.endswith('.json')
    is_metadata = (affected_file_path.startswith('site/') and
                   affected_file_path.endswith('/METADATA'))
    if is_json or is_metadata:
      try:
        input_api.json.load(open(affected_file_path, 'r'))
      except ValueError:
        failing_files.append(affected_file_path)

  results = []
  if failing_files:
    results.append(
        output_api.PresubmitError(
            'The following files contain invalid json:\n%s\n\n' %
                '\n'.join(failing_files)))
  return results


def _IfDefChecks(input_api, output_api):
  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
  comment_block_middle_pattern = re.compile('^\s+\*.*')
  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
  single_line_comment_pattern = re.compile('^\s*//.*$')
  def is_comment(line):
    return (comment_block_start_pattern.match(line) or
            comment_block_middle_pattern.match(line) or
            comment_block_end_pattern.match(line) or
            single_line_comment_pattern.match(line))

  empty_line_pattern = re.compile('^\s*$')
  def is_empty_line(line):
    return empty_line_pattern.match(line)

  failing_files = []
  for affected_file in input_api.AffectedSourceFiles(None):
    affected_file_path = affected_file.LocalPath()
    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
      f = open(affected_file_path)
      for line in f.xreadlines():
        if is_comment(line) or is_empty_line(line):
          continue
        # The below will be the first real line after comments and newlines.
        if line.startswith('#if 0 '):
          pass
        elif line.startswith('#if ') or line.startswith('#ifdef '):
          failing_files.append(affected_file_path)
        break

  results = []
  if failing_files:
    results.append(
        output_api.PresubmitError(
            'The following files have #if or #ifdef before includes:\n%s\n\n'
            'See https://bug.skia.org/3362 for why this should be fixed.' %
                '\n'.join(failing_files)))
  return results


def _CopyrightChecks(input_api, output_api, source_file_filter=None):
  results = []
  year_pattern = r'\d{4}'
  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
  copyright_pattern = (
      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)

  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
    if 'third_party' in affected_file.LocalPath():
      continue
    contents = input_api.ReadFile(affected_file, 'rb')
    if not re.search(copyright_pattern, contents):
      results.append(output_api.PresubmitError(
          '%s is missing a correct copyright header.' % affected_file))
  return results


def _ToolFlags(input_api, output_api):
  """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
  results = []
  sources = lambda x: ('dm_flags.py'        in x.LocalPath() or
                       'nanobench_flags.py' in x.LocalPath())
  for f in input_api.AffectedSourceFiles(sources):
    if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
      results.append(output_api.PresubmitError('`python %s test` failed' % f))
  return results


def _InfraTests(input_api, output_api):
  """Run the infra tests."""
  results = []
  if not any(f.LocalPath().startswith('infra')
             for f in input_api.AffectedFiles()):
    return results

  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
  try:
    subprocess.check_output(cmd)
  except subprocess.CalledProcessError as e:
    results.append(output_api.PresubmitError(
        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
  return results


def _CheckGNFormatted(input_api, output_api):
  """Make sure any .gn files we're changing have been formatted."""
  results = []
  for f in input_api.AffectedFiles():
    if (not f.LocalPath().endswith('.gn') and
        not f.LocalPath().endswith('.gni')):
      continue

    gn = 'gn.bat' if 'win32' in sys.platform else 'gn'
    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
    try:
      subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
      fix = 'gn format ' + f.LocalPath()
      results.append(output_api.PresubmitError(
          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
  return results


def _CommonChecks(input_api, output_api):
  """Presubmit checks common to upload and commit."""
  results = []
  sources = lambda x: (x.LocalPath().endswith('.h') or
                       x.LocalPath().endswith('.py') or
                       x.LocalPath().endswith('.sh') or
                       x.LocalPath().endswith('.m') or
                       x.LocalPath().endswith('.mm') or
                       x.LocalPath().endswith('.go') or
                       x.LocalPath().endswith('.c') or
                       x.LocalPath().endswith('.cc') or
                       x.LocalPath().endswith('.cpp'))
  results.extend(
      _CheckChangeHasEol(
          input_api, output_api, source_file_filter=sources))
  results.extend(
      input_api.canned_checks.CheckChangeHasNoCR(
          input_api, output_api, source_file_filter=sources))
  results.extend(
      input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
          input_api, output_api, source_file_filter=sources))
  results.extend(_PythonChecks(input_api, output_api))
  results.extend(_JsonChecks(input_api, output_api))
  results.extend(_IfDefChecks(input_api, output_api))
  results.extend(_CopyrightChecks(input_api, output_api,
                                  source_file_filter=sources))
  results.extend(_ToolFlags(input_api, output_api))
  return results


def CheckChangeOnUpload(input_api, output_api):
  """Presubmit checks for the change on upload.

  The following are the presubmit checks:
  * Check change has one and only one EOL.
  """
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  # Run on upload, not commit, since the presubmit bot apparently doesn't have
  # coverage or Go installed.
  results.extend(_InfraTests(input_api, output_api))

  results.extend(_CheckGNFormatted(input_api, output_api))
  return results


def _CheckTreeStatus(input_api, output_api, json_url):
  """Check whether to allow commit.

  Args:
    input_api: input related apis.
    output_api: output related apis.
    json_url: url to download json style status.
  """
  tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
      input_api, output_api, json_url=json_url)
  if not tree_status_results:
    # Check for caution state only if tree is not closed.
    connection = input_api.urllib2.urlopen(json_url)
    status = input_api.json.loads(connection.read())
    connection.close()
    if ('caution' in status['message'].lower() and
        os.isatty(sys.stdout.fileno())):
      # Display a prompt only if we are in an interactive shell. Without this
      # check the commit queue behaves incorrectly because it considers
      # prompts to be failures.
      short_text = 'Tree state is: ' + status['general_state']
      long_text = status['message'] + '\n' + json_url
      tree_status_results.append(
          output_api.PresubmitPromptWarning(
              message=short_text, long_text=long_text))
  else:
    # Tree status is closed. Put in message about contacting sheriff.
    connection = input_api.urllib2.urlopen(
        SKIA_TREE_STATUS_URL + '/current-sheriff')
    sheriff_details = input_api.json.loads(connection.read())
    if sheriff_details:
      tree_status_results[0]._message += (
          '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
          'to submit a build fix\nand do not know how to submit because the '
          'tree is closed') % sheriff_details['username']
  return tree_status_results


class CodeReview(object):
  """Abstracts which codereview tool is used for the specified issue."""

  def __init__(self, input_api):
    self._issue = input_api.change.issue
    self._gerrit = input_api.gerrit

  def GetOwnerEmail(self):
    return self._gerrit.GetChangeOwner(self._issue)

  def GetSubject(self):
    return self._gerrit.GetChangeInfo(self._issue)['subject']

  def GetDescription(self):
    return self._gerrit.GetChangeDescription(self._issue)

  def IsDryRun(self):
    return self._gerrit.GetChangeInfo(
        self._issue)['labels']['Commit-Queue'].get('value', 0) == 1

  def GetReviewers(self):
    code_review_label = (
        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
    return [r['email'] for r in code_review_label.get('all', [])]

  def GetApprovers(self):
    approvers = []
    code_review_label = (
        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
    for m in code_review_label.get('all', []):
      if m.get("value") == 1:
        approvers.append(m["email"])
    return approvers


def _CheckOwnerIsInAuthorsFile(input_api, output_api):
  results = []
  if input_api.change.issue:
    cr = CodeReview(input_api)

    owner_email = cr.GetOwnerEmail()

    # Service accounts don't need to be in AUTHORS.
    for suffix in SERVICE_ACCOUNT_SUFFIX:
      if owner_email.endswith(suffix):
        return results

    try:
      authors_content = ''
      for line in open(AUTHORS_FILE_NAME):
        if not line.startswith('#'):
          authors_content += line
      email_fnmatches = re.findall('<(.*)>', authors_content)
      for email_fnmatch in email_fnmatches:
        if fnmatch.fnmatch(owner_email, email_fnmatch):
          # Found a match, the user is in the AUTHORS file break out of the loop
          break
      else:
        results.append(
          output_api.PresubmitError(
            'The email %s is not in Skia\'s AUTHORS file.\n'
            'Issue owner, this CL must include an addition to the Skia AUTHORS '
            'file.'
            % owner_email))
    except IOError:
      # Do not fail if authors file cannot be found.
      traceback.print_exc()
      input_api.logging.error('AUTHORS file not found!')

  return results


def _CheckLGTMsForPublicAPI(input_api, output_api):
  """Check LGTMs for public API changes.

  For public API files make sure there is an LGTM from the list of owners in
  PUBLIC_API_OWNERS.
  """
  results = []
  requires_owner_check = False
  for affected_file in input_api.AffectedFiles():
    affected_file_path = affected_file.LocalPath()
    file_path, file_ext = os.path.splitext(affected_file_path)
    # We only care about files that end in .h and are under the top-level
    # include dir, but not include/private.
    if (file_ext == '.h' and
        'include' == file_path.split(os.path.sep)[0] and
        'private' not in file_path):
      requires_owner_check = True

  if not requires_owner_check:
    return results

  lgtm_from_owner = False
  if input_api.change.issue:
    cr = CodeReview(input_api)

    if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
      # It is a revert CL, ignore the public api owners check.
      return results

    if cr.IsDryRun():
      # Ignore public api owners check for dry run CLs since they are not
      # going to be committed.
      return results

    if input_api.gerrit:
      for reviewer in cr.GetReviewers():
        if reviewer in PUBLIC_API_OWNERS:
          # If an owner is specified as an reviewer in Gerrit then ignore the
          # public api owners check.
          return results
    else:
      match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
      if match:
        tbr_section = match.group(1).strip().split(' ')[0]
        tbr_entries = tbr_section.split(',')
        for owner in PUBLIC_API_OWNERS:
          if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
            # If an owner is specified in the TBR= line then ignore the public
            # api owners check.
            return results

    if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
      # An owner created the CL that is an automatic LGTM.
      lgtm_from_owner = True

    for approver in cr.GetApprovers():
      if approver in PUBLIC_API_OWNERS:
        # Found an lgtm in a message from an owner.
        lgtm_from_owner = True
        break

  if not lgtm_from_owner:
    results.append(
        output_api.PresubmitError(
            "If this CL adds to or changes Skia's public API, you need an LGTM "
            "from any of %s.  If this CL only removes from or doesn't change "
            "Skia's public API, please add a short note to the CL saying so. "
            "Add one of the owners as a reviewer to your CL as well as to the "
            "TBR= line.  If you don't know if this CL affects Skia's public "
            "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
  return results


def _FooterExists(footers, key, value):
  for k, v in footers:
    if k == key and v == value:
      return True
  return False


def PostUploadHook(cl, change, output_api):
  """git cl upload will call this hook after the issue is created/modified.

  This hook does the following:
  * Adds a link to preview docs changes if there are any docs changes in the CL.
  * Adds 'No-Try: true' if the CL contains only docs changes.
  * Adds 'No-Tree-Checks: true' for non master branch changes since they do not
    need to be gated on the master branch's tree.
  * Adds 'No-Try: true' for non master branch changes since trybots do not yet
    work on them.
  * Adds 'No-Presubmit: true' for non master branch changes since those don't
    run the presubmit checks.
  * Adds extra trybots for the paths defined in PATH_TO_EXTRA_TRYBOTS.
  """

  results = []
  atleast_one_docs_change = False
  all_docs_changes = True
  for affected_file in change.AffectedFiles():
    affected_file_path = affected_file.LocalPath()
    file_path, _ = os.path.splitext(affected_file_path)
    if 'site' == file_path.split(os.path.sep)[0]:
      atleast_one_docs_change = True
    else:
      all_docs_changes = False
    if atleast_one_docs_change and not all_docs_changes:
      break

  issue = cl.issue
  if issue:
    # Skip PostUploadHooks for all auto-commit service account bots. New
    # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
    # the "--use-commit-queue" flag to "git cl upload".
    for suffix in SERVICE_ACCOUNT_SUFFIX:
      if cl.GetIssueOwner().endswith(suffix):
        return results

    original_description_lines, footers = cl.GetDescriptionFooters()
    new_description_lines = list(original_description_lines)

    # If the change includes only doc changes then add No-Try: true in the
    # CL's description if it does not exist yet.
    if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'):
      new_description_lines.append('No-Try: true')
      results.append(
          output_api.PresubmitNotifyResult(
              'This change has only doc changes. Automatically added '
              '\'No-Try: true\' to the CL\'s description'))

    # If there is atleast one docs change then add preview link in the CL's
    # description if it does not already exist there.
    docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue)
    docs_preview_line = 'Docs-Preview: %s' % docs_preview_link
    if (atleast_one_docs_change and
        not _FooterExists(footers, 'Docs-Preview', docs_preview_link)):
      # Automatically add a link to where the docs can be previewed.
      new_description_lines.append(docs_preview_line)
      results.append(
          output_api.PresubmitNotifyResult(
              'Automatically added a link to preview the docs changes to the '
              'CL\'s description'))

    # If the target ref is not master then add 'No-Tree-Checks: true' and
    # 'No-Try: true' to the CL's description if it does not already exist there.
    target_ref = cl.GetRemoteBranch()[1]
    if target_ref != 'refs/remotes/origin/master':
      if not _FooterExists(footers, 'No-Tree-Checks', 'true'):
        new_description_lines.append('No-Tree-Checks: true')
        results.append(
            output_api.PresubmitNotifyResult(
                'Branch changes do not need to rely on the master branch\'s '
                'tree status. Automatically added \'No-Tree-Checks: true\' to '
                'the CL\'s description'))
      if not _FooterExists(footers, 'No-Try', 'true'):
        new_description_lines.append('No-Try: true')
        results.append(
            output_api.PresubmitNotifyResult(
                'Trybots do not yet work for non-master branches. '
                'Automatically added \'No-Try: true\' to the CL\'s '
                'description'))
      if not _FooterExists(footers, 'No-Presubmit', 'true'):
        new_description_lines.append('No-Presubmit: true')
        results.append(
            output_api.PresubmitNotifyResult(
                'Branch changes do not run the presubmit checks.'))

    # Automatically set Cq-Include-Trybots if any of the changed files here
    # begin with the paths of interest.
    bots_to_include = []
    for affected_file in change.AffectedFiles():
      affected_file_path = affected_file.LocalPath()
      for path_prefix, extra_bots in PATH_PREFIX_TO_EXTRA_TRYBOTS.iteritems():
        if affected_file_path.startswith(path_prefix):
          results.append(
              output_api.PresubmitNotifyResult(
                  'Your CL modifies the path %s.\nAutomatically adding %s to '
                  'the CL description.' % (affected_file_path, extra_bots)))
          bots_to_include.append(extra_bots)
    if bots_to_include:
      output_api.EnsureCQIncludeTrybotsAreAdded(
          cl, bots_to_include, new_description_lines)

    # If the description has changed update it.
    if new_description_lines != original_description_lines:
      # Add a new line separating the new contents from the old contents.
      new_description_lines.insert(len(original_description_lines), '')
      cl.UpdateDescriptionFooters(new_description_lines, footers)

    return results


def CheckChangeOnCommit(input_api, output_api):
  """Presubmit checks for the change on commit.

  The following are the presubmit checks:
  * Check change has one and only one EOL.
  * Ensures that the Skia tree is open in
    http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
    state and an error if it is in 'Closed' state.
  """
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  results.extend(
      _CheckTreeStatus(input_api, output_api, json_url=(
          SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
  results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
  results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
  # content of files.
  results.extend(
      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
  return results