Fix tests that rely on the ordering of values in a dictionary.
Reviewed in https://codereview.appspot.com/7311096/.
diff --git a/tests/test_discovery.py b/tests/test_discovery.py
index 1c8b706..46d8e33 100644
--- a/tests/test_discovery.py
+++ b/tests/test_discovery.py
@@ -863,7 +863,7 @@
self.assertTrue(hasattr(new_zoo, 'scopedAnimals'))
self.assertTrue(callable(new_zoo.scopedAnimals))
- self.assertEqual(zoo._dynamic_attrs, new_zoo._dynamic_attrs)
+ self.assertEqual(sorted(zoo._dynamic_attrs), sorted(new_zoo._dynamic_attrs))
self.assertEqual(zoo._baseUrl, new_zoo._baseUrl)
self.assertEqual(zoo._developerKey, new_zoo._developerKey)
self.assertEqual(zoo._requestBuilder, new_zoo._requestBuilder)
diff --git a/tests/test_push.py b/tests/test_push.py
index 5a42835..7c5ce0d 100644
--- a/tests/test_push.py
+++ b/tests/test_push.py
@@ -19,6 +19,7 @@
from apiclient import push
from apiclient import model
from apiclient import http
+from test_discovery import assertUrisEqual
class ClientTokenGeneratorTest(unittest.TestCase):
@@ -92,13 +93,15 @@
def test_creation_no_appengine(self):
c = push.WebhookChannel('http://example.org')
- self.assertEqual('web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
- c.as_header_value())
+ assertUrisEqual(self,
+ 'web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
+ c.as_header_value())
def test_creation_appengine(self):
c = push.WebhookChannel('http://example.org', app_engine=True)
- self.assertEqual('web_hook?url=http%3A%2F%2Fexample.org&app_engine=true',
- c.as_header_value())
+ assertUrisEqual(self,
+ 'web_hook?url=http%3A%2F%2Fexample.org&app_engine=true',
+ c.as_header_value())
class HeadersTest(unittest.TestCase):
@@ -146,15 +149,17 @@
c = push.WebhookChannel('http://example.org')
s = push.Subscription.for_channel(c)
self.assertTrue(s.client_token)
- self.assertEqual('web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
- s.subscribe)
+ assertUrisEqual(self,
+ 'web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
+ s.subscribe)
def test_create_for_channel_client_token(self):
c = push.WebhookChannel('http://example.org')
s = push.Subscription.for_channel(c, client_token='my_token')
self.assertEqual('my_token', s.client_token)
- self.assertEqual('web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
- s.subscribe)
+ assertUrisEqual(self,
+ 'web_hook?url=http%3A%2F%2Fexample.org&app_engine=false',
+ s.subscribe)
def test_subscribe(self):
s = push.Subscription()
diff --git a/upload-diffs.py b/upload-diffs.py
index b9e5847..7b5cc80 100755
--- a/upload-diffs.py
+++ b/upload-diffs.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# coding: utf-8
#
# Copyright 2007 Google Inc.
#
@@ -35,6 +36,7 @@
import ConfigParser
import cookielib
+import errno
import fnmatch
import getpass
import logging
@@ -93,12 +95,6 @@
VCS_CVS = "CVS"
VCS_UNKNOWN = "Unknown"
-# whitelist for non-binary filetypes which do not start with "text/"
-# .mm (Objective-C) shows up as application/x-freemind on my Linux box.
-TEXT_MIMETYPES = ['application/javascript', 'application/json',
- 'application/x-javascript', 'application/xml',
- 'application/x-freemind', 'application/x-sh']
-
VCS_ABBREVIATIONS = {
VCS_MERCURIAL.lower(): VCS_MERCURIAL,
"hg": VCS_MERCURIAL,
@@ -169,7 +165,15 @@
def __init__(self, url, code, msg, headers, args):
urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
self.args = args
- self.reason = args["Error"]
+ self._reason = args["Error"]
+ self.info = args.get("Info", None)
+
+ @property
+ def reason(self):
+ # reason is a property on python 2.7 but a member variable on <=2.6.
+ # self.args is modified so it cannot be used as-is so save the value in
+ # self._reason.
+ return self._reason
class AbstractRpcServer(object):
@@ -177,7 +181,7 @@
def __init__(self, host, auth_function, host_override=None, extra_headers={},
save_cookies=False, account_type=AUTH_ACCOUNT_TYPE):
- """Creates a new HttpRpcServer.
+ """Creates a new AbstractRpcServer.
Args:
host: The host to send requests to.
@@ -219,7 +223,7 @@
def _CreateRequest(self, url, data=None):
"""Creates a new urllib request."""
logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
- req = urllib2.Request(url, data=data)
+ req = urllib2.Request(url, data=data, headers={"Accept": "text/plain"})
if self.host_override:
req.add_header("Host", self.host_override)
for key, value in self.extra_headers.iteritems():
@@ -313,37 +317,42 @@
try:
auth_token = self._GetAuthToken(credentials[0], credentials[1])
except ClientLoginError, e:
+ print >>sys.stderr, ''
if e.reason == "BadAuthentication":
- print >>sys.stderr, "Invalid username or password."
- continue
- if e.reason == "CaptchaRequired":
+ if e.info == "InvalidSecondFactor":
+ print >>sys.stderr, (
+ "Use an application-specific password instead "
+ "of your regular account password.\n"
+ "See http://www.google.com/"
+ "support/accounts/bin/answer.py?answer=185833")
+ else:
+ print >>sys.stderr, "Invalid username or password."
+ elif e.reason == "CaptchaRequired":
print >>sys.stderr, (
"Please go to\n"
"https://www.google.com/accounts/DisplayUnlockCaptcha\n"
"and verify you are a human. Then try again.\n"
"If you are using a Google Apps account the URL is:\n"
"https://www.google.com/a/yourdomain.com/UnlockCaptcha")
- break
- if e.reason == "NotVerified":
+ elif e.reason == "NotVerified":
print >>sys.stderr, "Account not verified."
- break
- if e.reason == "TermsNotAgreed":
+ elif e.reason == "TermsNotAgreed":
print >>sys.stderr, "User has not agreed to TOS."
- break
- if e.reason == "AccountDeleted":
+ elif e.reason == "AccountDeleted":
print >>sys.stderr, "The user account has been deleted."
- break
- if e.reason == "AccountDisabled":
+ elif e.reason == "AccountDisabled":
print >>sys.stderr, "The user account has been disabled."
break
- if e.reason == "ServiceDisabled":
+ elif e.reason == "ServiceDisabled":
print >>sys.stderr, ("The user's access to the service has been "
"disabled.")
- break
- if e.reason == "ServiceUnavailable":
+ elif e.reason == "ServiceUnavailable":
print >>sys.stderr, "The service is not available; try again later."
- break
- raise
+ else:
+ # Unknown error.
+ raise
+ print >>sys.stderr, ''
+ continue
self._GetAuthCookie(auth_token)
return
@@ -398,14 +407,13 @@
raise
elif e.code == 401 or e.code == 302:
self._Authenticate()
-## elif e.code >= 500 and e.code < 600:
-## # Server Error - try again.
-## continue
elif e.code == 301:
# Handle permanent redirect manually.
url = e.info()["location"]
url_loc = urlparse.urlparse(url)
self.host = '%s://%s' % (url_loc[0], url_loc[1])
+ elif e.code >= 500:
+ ErrorExit(e.read())
else:
raise
finally:
@@ -460,8 +468,39 @@
return opener
+class CondensedHelpFormatter(optparse.IndentedHelpFormatter):
+ """Frees more horizontal space by removing indentation from group
+ options and collapsing arguments between short and long, e.g.
+ '-o ARG, --opt=ARG' to -o --opt ARG"""
+
+ def format_heading(self, heading):
+ return "%s:\n" % heading
+
+ def format_option(self, option):
+ self.dedent()
+ res = optparse.HelpFormatter.format_option(self, option)
+ self.indent()
+ return res
+
+ def format_option_strings(self, option):
+ self.set_long_opt_delimiter(" ")
+ optstr = optparse.HelpFormatter.format_option_strings(self, option)
+ optlist = optstr.split(", ")
+ if len(optlist) > 1:
+ if option.takes_value():
+ # strip METAVAR from all but the last option
+ optlist = [x.split()[0] for x in optlist[:-1]] + optlist[-1:]
+ optstr = " ".join(optlist)
+ return optstr
+
+
parser = optparse.OptionParser(
- usage="%prog [options] [-- diff_options] [path...]")
+ usage="%prog [options] [-- diff_options] [path...]",
+ add_help_option=False,
+ formatter=CondensedHelpFormatter()
+)
+parser.add_option("-h", "--help", action="store_true",
+ help="Show this help message and exit.")
parser.add_option("-y", "--assume_yes", action="store_true",
dest="assume_yes", default=False,
help="Assume that the answer to yes/no questions is 'yes'.")
@@ -500,14 +539,13 @@
"valid choices are 'GOOGLE' and 'HOSTED')."))
# Issue
group = parser.add_option_group("Issue options")
-group.add_option("-d", "--description", action="store", dest="description",
- metavar="DESCRIPTION", default=None,
- help="Optional description when creating an issue.")
-group.add_option("-f", "--description_file", action="store",
- dest="description_file", metavar="DESCRIPTION_FILE",
+group.add_option("-t", "--title", action="store", dest="title",
+ help="New issue subject or new patch set title")
+group.add_option("-m", "--message", action="store", dest="message",
default=None,
- help="Optional path of a file that contains "
- "the description when creating an issue.")
+ help="New issue description or new patch set message")
+group.add_option("-F", "--file", action="store", dest="file",
+ default=None, help="Read the message above from file.")
group.add_option("-r", "--reviewers", action="store", dest="reviewers",
metavar="REVIEWERS", default="jcgregorio@google.com",
help="Add reviewers (comma separated email addresses).")
@@ -520,15 +558,11 @@
help="Make the issue restricted to reviewers and those CCed")
# Upload options
group = parser.add_option_group("Patch options")
-group.add_option("-m", "--message", action="store", dest="message",
- metavar="MESSAGE", default=None,
- help="A message to identify the patch. "
- "Will prompt if omitted.")
group.add_option("-i", "--issue", type="int", action="store",
metavar="ISSUE", default=None,
help="Issue number to which to add. Defaults to new issue.")
group.add_option("--base_url", action="store", dest="base_url", default=None,
- help="Base repository URL (listed as \"Base URL\" when "
+ help="Base URL path for files (listed as \"Base URL\" when "
"viewing issue). If omitted, will be guessed automatically "
"for SVN repos and left blank for others.")
group.add_option("--download_base", action="store_true",
@@ -542,6 +576,10 @@
group.add_option("--send_mail", action="store_true",
dest="send_mail", default=False,
help="Send notification email to reviewers.")
+group.add_option("-p", "--send_patch", action="store_true",
+ dest="send_patch", default=False,
+ help="Same as --send_mail, but include diff as an "
+ "attachment, and prepend email subject with 'PATCH:'.")
group.add_option("--vcs", action="store", dest="vcs",
metavar="VCS", default=None,
help=("Version control system (optional, usually upload.py "
@@ -549,6 +587,15 @@
group.add_option("--emulate_svn_auto_props", action="store_true",
dest="emulate_svn_auto_props", default=False,
help=("Emulate Subversion's auto properties feature."))
+# Git-specific
+group = parser.add_option_group("Git-specific options")
+group.add_option("--git_similarity", action="store", dest="git_similarity",
+ metavar="SIM", type="int", default=50,
+ help=("Set the minimum similarity index for detecting renames "
+ "and copies. See `git diff -C`. (default 50)."))
+group.add_option("--git_no_find_copies", action="store_false", default=True,
+ dest="git_find_copies",
+ help=("Prevents git from looking for copies (default off)."))
# Perforce-specific
group = parser.add_option_group("Perforce-specific options "
"(overrides P4 environment variables)")
@@ -565,6 +612,48 @@
metavar="P4_USER", default=None,
help=("Perforce user"))
+
+class KeyringCreds(object):
+ def __init__(self, server, host, email):
+ self.server = server
+ self.host = host
+ self.email = email
+ self.accounts_seen = set()
+
+ def GetUserCredentials(self):
+ """Prompts the user for a username and password.
+
+ Only use keyring on the initial call. If the keyring contains the wrong
+ password, we want to give the user a chance to enter another one.
+ """
+ # Create a local alias to the email variable to avoid Python's crazy
+ # scoping rules.
+ global keyring
+ email = self.email
+ if email is None:
+ email = GetEmail("Email (login for uploading to %s)" % self.server)
+ password = None
+ if keyring and not email in self.accounts_seen:
+ try:
+ password = keyring.get_password(self.host, email)
+ except:
+ # Sadly, we have to trap all errors here as
+ # gnomekeyring.IOError inherits from object. :/
+ print "Failed to get password from keyring"
+ keyring = None
+ if password is not None:
+ print "Using password from system keyring."
+ self.accounts_seen.add(email)
+ else:
+ password = getpass.getpass("Password for %s: " % email)
+ if keyring:
+ answer = raw_input("Store password in system keyring?(y/N) ").strip()
+ if answer == "y":
+ keyring.set_password(self.host, email, password)
+ self.accounts_seen.add(email)
+ return (email, password)
+
+
def GetRpcServer(server, email=None, host_override=None, save_cookies=True,
account_type=AUTH_ACCOUNT_TYPE):
"""Returns an instance of an AbstractRpcServer.
@@ -579,18 +668,16 @@
or 'HOSTED'. Defaults to AUTH_ACCOUNT_TYPE.
Returns:
- A new AbstractRpcServer, on which RPC calls can be made.
+ A new HttpRpcServer, on which RPC calls can be made.
"""
- rpc_server_class = HttpRpcServer
-
# If this is the dev_appserver, use fake authentication.
host = (host_override or server).lower()
if re.match(r'(http://)?localhost([:/]|$)', host):
if email is None:
email = "test@example.com"
logging.info("Using debug user %s. Override with --email" % email)
- server = rpc_server_class(
+ server = HttpRpcServer(
server,
lambda: (email, "password"),
host_override=host_override,
@@ -602,30 +689,11 @@
server.authenticated = True
return server
- def GetUserCredentials():
- """Prompts the user for a username and password."""
- # Create a local alias to the email variable to avoid Python's crazy
- # scoping rules.
- local_email = email
- if local_email is None:
- local_email = GetEmail("Email (login for uploading to %s)" % server)
- password = None
- if keyring:
- password = keyring.get_password(host, local_email)
- if password is not None:
- print "Using password from system keyring."
- else:
- password = getpass.getpass("Password for %s: " % local_email)
- if keyring:
- answer = raw_input("Store password in system keyring?(y/N) ").strip()
- if answer == "y":
- keyring.set_password(host, local_email, password)
- return (local_email, password)
-
- return rpc_server_class(server,
- GetUserCredentials,
- host_override=host_override,
- save_cookies=save_cookies)
+ return HttpRpcServer(server,
+ KeyringCreds(server, host, email).GetUserCredentials,
+ host_override=host_override,
+ save_cookies=save_cookies,
+ account_type=account_type)
def EncodeMultipartFormData(fields, files):
@@ -675,10 +743,10 @@
# Use a shell for subcommands on Windows to get a PATH search.
use_shell = sys.platform.startswith("win")
-def RunShellWithReturnCode(command, print_output=False,
+def RunShellWithReturnCodeAndStderr(command, print_output=False,
universal_newlines=True,
env=os.environ):
- """Executes a command and returns the output from stdout and the return code.
+ """Executes a command and returns the output from stdout, stderr and the return code.
Args:
command: Command to execute.
@@ -687,9 +755,11 @@
universal_newlines: Use universal_newlines flag (default: True).
Returns:
- Tuple (output, return code)
+ Tuple (stdout, stderr, return code)
"""
logging.info("Running %s", command)
+ env = env.copy()
+ env['LC_MESSAGES'] = 'C'
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=use_shell, universal_newlines=universal_newlines,
env=env)
@@ -710,8 +780,15 @@
print >>sys.stderr, errout
p.stdout.close()
p.stderr.close()
- return output, p.returncode
+ return output, errout, p.returncode
+def RunShellWithReturnCode(command, print_output=False,
+ universal_newlines=True,
+ env=os.environ):
+ """Executes a command and returns the output from stdout and the return code."""
+ out, err, retcode = RunShellWithReturnCodeAndStderr(command, print_output,
+ universal_newlines, env)
+ return out, retcode
def RunShell(command, silent_ok=False, universal_newlines=True,
print_output=False, env=os.environ):
@@ -735,6 +812,12 @@
"""
self.options = options
+ def GetGUID(self):
+ """Return string to distinguish the repository from others, for example to
+ query all opened review issues for it"""
+ raise NotImplementedError(
+ "abstract method -- subclass %s must override" % self.__class__)
+
def PostProcessDiff(self, diff):
"""Return the diff with any special post processing this VCS needs, e.g.
to include an svn-style "Index:"."""
@@ -861,15 +944,11 @@
return False
return mimetype.startswith("image/")
- def IsBinary(self, filename):
- """Returns true if the guessed mimetyped isnt't in text group."""
- mimetype = mimetypes.guess_type(filename)[0]
- if not mimetype:
- return False # e.g. README, "real" binaries usually have an extension
- # special case for text files which don't start with text/
- if mimetype in TEXT_MIMETYPES:
- return False
- return not mimetype.startswith("text/")
+ def IsBinaryData(self, data):
+ """Returns true if data contains a null byte."""
+ # Derived from how Mercurial's heuristic, see
+ # http://selenic.com/hg/file/848a6658069e/mercurial/util.py#l229
+ return bool(data and "\0" in data)
class SubversionVCS(VersionControlSystem):
@@ -893,6 +972,9 @@
required = self.options.download_base or self.options.revision is not None
self.svn_base = self._GuessBase(required)
+ def GetGUID(self):
+ return self._GetInfo("Repository UUID")
+
def GuessBase(self, required):
"""Wrapper for _GuessBase."""
return self.svn_base
@@ -904,12 +986,11 @@
required: If true, exits if the url can't be guessed, otherwise None is
returned.
"""
- info = RunShell(["svn", "info"])
- for line in info.splitlines():
- if line.startswith("URL: "):
- url = line.split()[1]
+ url = self._GetInfo("URL")
+ if url:
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
guess = ""
+ # TODO(anatoli) - repository specific hacks should be handled by server
if netloc == "svn.python.org" and scheme == "svn+ssh":
path = "projects" + path
scheme = "http"
@@ -926,6 +1007,18 @@
ErrorExit("Can't find URL in output from svn info")
return None
+ def _GetInfo(self, key):
+ """Parses 'svn info' for current dir. Returns value for key or None"""
+ for line in RunShell(["svn", "info"]).splitlines():
+ if line.startswith(key + ": "):
+ return line.split(":", 1)[1].strip()
+
+ def _EscapeFilename(self, filename):
+ """Escapes filename for SVN commands."""
+ if "@" in filename and not filename.endswith("@"):
+ filename = "%s@" % filename
+ return filename
+
def GenerateDiff(self, args):
cmd = ["svn", "diff"]
if self.options.revision:
@@ -993,7 +1086,8 @@
def GetStatus(self, filename):
"""Returns the status of a file."""
if not self.options.revision:
- status = RunShell(["svn", "status", "--ignore-externals", filename])
+ status = RunShell(["svn", "status", "--ignore-externals",
+ self._EscapeFilename(filename)])
if not status:
ErrorExit("svn status returned no output for %s" % filename)
status_lines = status.splitlines()
@@ -1012,15 +1106,22 @@
else:
dirname, relfilename = os.path.split(filename)
if dirname not in self.svnls_cache:
- cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
- out, returncode = RunShellWithReturnCode(cmd)
+ cmd = ["svn", "list", "-r", self.rev_start,
+ self._EscapeFilename(dirname) or "."]
+ out, err, returncode = RunShellWithReturnCodeAndStderr(cmd)
if returncode:
- ErrorExit("Failed to get status for %s." % filename)
- old_files = out.splitlines()
+ # Directory might not yet exist at start revison
+ # svn: Unable to find repository location for 'abc' in revision nnn
+ if re.match('^svn: Unable to find repository location for .+ in revision \d+', err):
+ old_files = ()
+ else:
+ ErrorExit("Failed to get status for %s:\n%s" % (filename, err))
+ else:
+ old_files = out.splitlines()
args = ["svn", "list"]
if self.rev_end:
args += ["-r", self.rev_end]
- cmd = args + [dirname or "."]
+ cmd = args + [self._EscapeFilename(dirname) or "."]
out, returncode = RunShellWithReturnCode(cmd)
if returncode:
ErrorExit("Failed to run command %s" % cmd)
@@ -1046,17 +1147,18 @@
if status[0] == "A" and status[3] != "+":
# We'll need to upload the new content if we're adding a binary file
# since diff's output won't contain it.
- mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
- silent_ok=True)
+ mimetype = RunShell(["svn", "propget", "svn:mime-type",
+ self._EscapeFilename(filename)], silent_ok=True)
base_content = ""
is_binary = bool(mimetype) and not mimetype.startswith("text/")
- if is_binary and self.IsImage(filename):
+ if is_binary:
new_content = self.ReadFile(filename)
elif (status[0] in ("M", "D", "R") or
(status[0] == "A" and status[3] == "+") or # Copied file.
(status[0] == " " and status[1] == "M")): # Property change.
args = []
if self.options.revision:
+ # filename must not be escaped. We already add an ampersand here.
url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
else:
# Don't change filename, it's needed later.
@@ -1071,24 +1173,24 @@
else:
mimetype = mimetype.strip()
get_base = False
+ # this test for binary is exactly the test prescribed by the
+ # official SVN docs at
+ # http://subversion.apache.org/faq.html#binary-files
is_binary = (bool(mimetype) and
not mimetype.startswith("text/") and
- not mimetype in TEXT_MIMETYPES)
+ mimetype not in ("image/x-xbitmap", "image/x-xpixmap"))
if status[0] == " ":
# Empty base content just to force an upload.
base_content = ""
elif is_binary:
- if self.IsImage(filename):
- get_base = True
- if status[0] == "M":
- if not self.rev_end:
- new_content = self.ReadFile(filename)
- else:
- url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
- new_content = RunShell(["svn", "cat", url],
- universal_newlines=True, silent_ok=True)
- else:
- base_content = ""
+ get_base = True
+ if status[0] == "M":
+ if not self.rev_end:
+ new_content = self.ReadFile(filename)
+ else:
+ url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
+ new_content = RunShell(["svn", "cat", url],
+ universal_newlines=True, silent_ok=True)
else:
get_base = True
@@ -1106,7 +1208,8 @@
silent_ok=True)
else:
base_content, ret_code = RunShellWithReturnCode(
- ["svn", "cat", filename], universal_newlines=universal_newlines)
+ ["svn", "cat", self._EscapeFilename(filename)],
+ universal_newlines=universal_newlines)
if ret_code and status[0] == "R":
# It's a replaced file without local history (see issue208).
# The base file needs to be fetched from the server.
@@ -1144,6 +1247,15 @@
# Map of new filename -> old filename for renames.
self.renames = {}
+ def GetGUID(self):
+ revlist = RunShell("git rev-list --parents HEAD".split()).splitlines()
+ # M-A: Return the 1st root hash, there could be multiple when a
+ # subtree is merged. In that case, more analysis would need to
+ # be done to figure out which HEAD is the 'most representative'.
+ for r in revlist:
+ if ' ' not in r:
+ return r
+
def PostProcessDiff(self, gitdiff):
"""Converts the diff output to include an svn-style "Index:" line as well
as record the hashes of the files, so we can upload them along with our
@@ -1214,9 +1326,33 @@
# this by overriding the environment (but there is still a problem if the
# git config key "diff.external" is used).
env = os.environ.copy()
- if 'GIT_EXTERNAL_DIFF' in env: del env['GIT_EXTERNAL_DIFF']
- return RunShell(["git", "diff", "--no-ext-diff", "--full-index", "-M"]
- + extra_args, env=env)
+ if "GIT_EXTERNAL_DIFF" in env:
+ del env["GIT_EXTERNAL_DIFF"]
+ # -M/-C will not print the diff for the deleted file when a file is renamed.
+ # This is confusing because the original file will not be shown on the
+ # review when a file is renamed. So, get a diff with ONLY deletes, then
+ # append a diff (with rename detection), without deletes.
+ cmd = [
+ "git", "diff", "--no-color", "--no-ext-diff", "--full-index",
+ "--ignore-submodules",
+ ]
+ diff = RunShell(
+ cmd + ["--no-renames", "--diff-filter=D"] + extra_args,
+ env=env, silent_ok=True)
+ if self.options.git_find_copies:
+ similarity_options = ["--find-copies-harder", "-l100000",
+ "-C%s" % self.options.git_similarity ]
+ else:
+ similarity_options = ["-M%s" % self.options.git_similarity ]
+ diff += RunShell(
+ cmd + ["--diff-filter=AMCRT"] + similarity_options + extra_args,
+ env=env, silent_ok=True)
+
+ # The CL could be only file deletion or not. So accept silent diff for both
+ # commands then check for an empty diff manually.
+ if not diff:
+ ErrorExit("No output from %s" % (cmd + extra_args))
+ return diff
def GetUnknownFiles(self):
status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
@@ -1235,14 +1371,14 @@
hash_before, hash_after = self.hashes.get(filename, (None,None))
base_content = None
new_content = None
- is_binary = self.IsBinary(filename)
status = None
if filename in self.renames:
status = "A +" # Match svn attribute name for renames.
if filename not in self.hashes:
# If a rename doesn't change the content, we never get a hash.
- base_content = RunShell(["git", "show", "HEAD:" + filename])
+ base_content = RunShell(
+ ["git", "show", "HEAD:" + filename], silent_ok=True)
elif not hash_before:
status = "A"
base_content = ""
@@ -1251,18 +1387,17 @@
else:
status = "M"
+ is_binary = self.IsBinaryData(base_content)
is_image = self.IsImage(filename)
# Grab the before/after content if we need it.
- # We should include file contents if it's text or it's an image.
- if not is_binary or is_image:
- # Grab the base content if we don't have it already.
- if base_content is None and hash_before:
- base_content = self.GetFileContent(hash_before, is_binary)
- # Only include the "after" file if it's an image; otherwise it
- # it is reconstructed from the diff.
- if is_image and hash_after:
- new_content = self.GetFileContent(hash_after, is_binary)
+ # Grab the base content if we don't have it already.
+ if base_content is None and hash_before:
+ base_content = self.GetFileContent(hash_before, is_binary)
+ # Only include the "after" file if it's an image; otherwise it
+ # it is reconstructed from the diff.
+ if is_image and hash_after:
+ new_content = self.GetFileContent(hash_after, is_binary)
return (base_content, new_content, is_binary, status)
@@ -1273,6 +1408,10 @@
def __init__(self, options):
super(CVSVCS, self).__init__(options)
+ def GetGUID(self):
+ """For now we don't know how to get repository ID for CVS"""
+ return
+
def GetOriginalContent_(self, filename):
RunShell(["cvs", "up", filename], silent_ok=True)
# TODO need detect file content encoding
@@ -1282,7 +1421,6 @@
def GetBaseFile(self, filename):
base_content = None
new_content = None
- is_binary = False
status = "A"
output, retcode = RunShellWithReturnCode(["cvs", "status", filename])
@@ -1302,7 +1440,7 @@
status = "D"
base_content = self.GetOriginalContent_(filename)
- return (base_content, new_content, is_binary, status)
+ return (base_content, new_content, self.IsBinaryData(base_content), status)
def GenerateDiff(self, extra_args):
cmd = ["cvs", "diff", "-u", "-N"]
@@ -1312,7 +1450,7 @@
cmd.extend(extra_args)
data, retcode = RunShellWithReturnCode(cmd)
count = 0
- if retcode == 0:
+ if retcode in [0, 1]:
for line in data.splitlines():
if line.startswith("Index:"):
count += 1
@@ -1324,10 +1462,11 @@
return data
def GetUnknownFiles(self):
- status = RunShell(["cvs", "diff"],
- silent_ok=True)
+ data, retcode = RunShellWithReturnCode(["cvs", "diff"])
+ if retcode not in [0, 1]:
+ ErrorExit("Got error status from 'cvs diff':\n%s" % (data,))
unknown_files = []
- for line in status.split("\n"):
+ for line in data.split("\n"):
if line and line[0] == "?":
unknown_files.append(line)
return unknown_files
@@ -1348,11 +1487,17 @@
else:
self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
+ def GetGUID(self):
+ # See chapter "Uniquely identifying a repository"
+ # http://hgbook.red-bean.com/read/customizing-the-output-of-mercurial.html
+ info = RunShell("hg log -r0 --template {node}".split())
+ return info.strip()
+
def _GetRelPath(self, filename):
"""Get relative path of a file according to the current directory,
given its logical path in the repo."""
- assert filename.startswith(self.subdir), (filename, self.subdir)
- return filename[len(self.subdir):].lstrip(r"\/")
+ absname = os.path.join(self.repo_dir, filename)
+ return os.path.relpath(absname)
def GenerateDiff(self, extra_args):
cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
@@ -1391,9 +1536,8 @@
return unknown_files
def GetBaseFile(self, filename):
- # "hg status" and "hg cat" both take a path relative to the current subdir
- # rather than to the repo root, but "hg diff" has given us the full path
- # to the repo root.
+ # "hg status" and "hg cat" both take a path relative to the current subdir,
+ # but "hg diff" has given us the path relative to the repo root.
base_content = ""
new_content = None
is_binary = False
@@ -1418,15 +1562,15 @@
if status != "A":
base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
silent_ok=True)
- is_binary = "\0" in base_content # Mercurial's heuristic
+ is_binary = self.IsBinaryData(base_content)
if status != "R":
new_content = open(relpath, "rb").read()
- is_binary = is_binary or "\0" in new_content
+ is_binary = is_binary or self.IsBinaryData(new_content)
if is_binary and base_content:
# Fetch again without converting newlines
base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
silent_ok=True, universal_newlines=False)
- if not is_binary or not self.IsImage(relpath):
+ if not is_binary:
new_content = None
return base_content, new_content, is_binary, status
@@ -1435,7 +1579,7 @@
"""Implementation of the VersionControlSystem interface for Perforce."""
def __init__(self, options):
-
+
def ConfirmLogin():
# Make sure we have a valid perforce session
while True:
@@ -1447,31 +1591,35 @@
break
print "Enter perforce password: "
self.RunPerforceCommandWithReturnCode(["login"])
-
+
super(PerforceVCS, self).__init__(options)
-
+
self.p4_changelist = options.p4_changelist
if not self.p4_changelist:
ErrorExit("A changelist id is required")
if (options.revision):
ErrorExit("--rev is not supported for perforce")
-
+
self.p4_port = options.p4_port
self.p4_client = options.p4_client
self.p4_user = options.p4_user
-
+
ConfirmLogin()
-
- if not options.message:
+
+ if not options.title:
description = self.RunPerforceCommand(["describe", self.p4_changelist],
marshal_output=True)
if description and "desc" in description:
# Rietveld doesn't support multi-line descriptions
- raw_message = description["desc"].strip()
- lines = raw_message.splitlines()
+ raw_title = description["desc"].strip()
+ lines = raw_title.splitlines()
if len(lines):
- options.message = lines[0]
-
+ options.title = lines[0]
+
+ def GetGUID(self):
+ """For now we don't know how to get repository ID for Perforce"""
+ return
+
def RunPerforceCommandWithReturnCode(self, extra_args, marshal_output=False,
universal_newlines=True):
args = ["p4"]
@@ -1485,13 +1633,13 @@
if self.p4_user:
args.extend(["-u", self.p4_user])
args.extend(extra_args)
-
+
data, retcode = RunShellWithReturnCode(
args, print_output=False, universal_newlines=universal_newlines)
if marshal_output and data:
data = marshal.loads(data)
return data, retcode
-
+
def RunPerforceCommand(self, extra_args, marshal_output=False,
universal_newlines=True):
# This might be a good place to cache call results, since things like
@@ -1505,7 +1653,7 @@
def GetFileProperties(self, property_key_prefix = "", command = "describe"):
description = self.RunPerforceCommand(["describe", self.p4_changelist],
marshal_output=True)
-
+
changed_files = {}
file_index = 0
# Try depotFile0, depotFile1, ... until we don't find a match
@@ -1534,9 +1682,6 @@
def IsPendingBinary(self, filename):
return self.IsBinaryHelper(filename, "describe")
- def IsBinary(self, filename):
- ErrorExit("IsBinary is not safe: call IsBaseBinary or IsPendingBinary")
-
def IsBinaryHelper(self, filename, command):
file_types = self.GetFileProperties("type", command)
if not filename in file_types:
@@ -1560,52 +1705,52 @@
"branch", # p4 integrate (to a new file), similar to hg "add"
"add", # p4 integrate (to a new file), after modifying the new file
]
-
+
# We only see a different base for "add" if this is a downgraded branch
- # after a file was branched (integrated), then edited.
+ # after a file was branched (integrated), then edited.
if self.GetAction(filename) in actionsWithDifferentBases:
# -Or shows information about pending integrations/moves
fstat_result = self.RunPerforceCommand(["fstat", "-Or", filename],
marshal_output=True)
-
+
baseFileKey = "resolveFromFile0" # I think it's safe to use only file0
if baseFileKey in fstat_result:
return fstat_result[baseFileKey]
-
+
return filename
def GetBaseRevision(self, filename):
base_filename = self.GetBaseFilename(filename)
-
+
have_result = self.RunPerforceCommand(["have", base_filename],
marshal_output=True)
if "haveRev" in have_result:
return have_result["haveRev"]
-
+
def GetLocalFilename(self, filename):
where = self.RunPerforceCommand(["where", filename], marshal_output=True)
if "path" in where:
return where["path"]
- def GenerateDiff(self, args):
+ def GenerateDiff(self, args):
class DiffData:
def __init__(self, perforceVCS, filename, action):
self.perforceVCS = perforceVCS
self.filename = filename
self.action = action
self.base_filename = perforceVCS.GetBaseFilename(filename)
-
+
self.file_body = None
self.base_rev = None
self.prefix = None
self.working_copy = True
self.change_summary = None
-
+
def GenerateDiffHeader(diffData):
header = []
header.append("Index: %s" % diffData.filename)
header.append("=" * 67)
-
+
if diffData.base_filename != diffData.filename:
if diffData.action.startswith("move"):
verb = "rename"
@@ -1613,7 +1758,7 @@
verb = "copy"
header.append("%s from %s" % (verb, diffData.base_filename))
header.append("%s to %s" % (verb, diffData.filename))
-
+
suffix = "\t(revision %s)" % diffData.base_rev
header.append("--- " + diffData.base_filename + suffix)
if diffData.working_copy:
@@ -1622,14 +1767,14 @@
if diffData.change_summary:
header.append(diffData.change_summary)
return header
-
+
def GenerateMergeDiff(diffData, args):
# -du generates a unified diff, which is nearly svn format
diffData.file_body = self.RunPerforceCommand(
["diff", "-du", diffData.filename] + args)
diffData.base_rev = self.GetBaseRevision(diffData.filename)
diffData.prefix = ""
-
+
# We have to replace p4's file status output (the lines starting
# with +++ or ---) to match svn's diff format
lines = diffData.file_body.splitlines()
@@ -1658,13 +1803,13 @@
diffData.change_summary += " @@"
diffData.prefix = "+"
return diffData
-
+
def GenerateDeleteDiff(diffData):
diffData.base_rev = self.GetBaseRevision(diffData.filename)
is_base_binary = self.IsBaseBinary(diffData.filename)
# For deletes, base_filename == filename
- diffData.file_body = self.GetFileContent(diffData.base_filename,
- None,
+ diffData.file_body = self.GetFileContent(diffData.base_filename,
+ None,
is_base_binary)
# Replicate svn's list of changed lines
line_count = len(diffData.file_body.splitlines())
@@ -1674,16 +1819,16 @@
diffData.change_summary += " +0,0 @@"
diffData.prefix = "-"
return diffData
-
+
changed_files = self.GetChangedFiles()
-
+
svndiff = []
filecount = 0
for (filename, action) in changed_files.items():
svn_status = self.PerforceActionToSvnStatus(action)
if svn_status == "SKIP":
continue
-
+
diffData = DiffData(self, filename, action)
# Is it possible to diff a branched file? Stackoverflow says no:
# http://stackoverflow.com/questions/1771314/in-perforce-command-line-how-to-diff-a-file-reopened-for-add
@@ -1696,9 +1841,9 @@
else:
ErrorExit("Unknown file action %s (svn action %s)." % \
(action, svn_status))
-
+
svndiff += GenerateDiffHeader(diffData)
-
+
for line in diffData.file_body.splitlines():
svndiff.append(diffData.prefix + line)
filecount += 1
@@ -1724,16 +1869,16 @@
changed_files = self.GetChangedFiles()
if not filename in changed_files:
ErrorExit("Trying to get base version of unknown file %s." % filename)
-
+
return changed_files[filename]
def GetBaseFile(self, filename):
base_filename = self.GetBaseFilename(filename)
base_content = ""
new_content = None
-
+
status = self.PerforceActionToSvnStatus(self.GetAction(filename))
-
+
if status != "A":
revision = self.GetBaseRevision(base_filename)
if not revision:
@@ -1742,13 +1887,13 @@
base_content = self.GetFileContent(base_filename,
revision,
is_base_binary)
-
+
is_binary = self.IsPendingBinary(filename)
if status != "D" and status != "SKIP":
relpath = self.GetLocalFilename(filename)
- if is_binary and self.IsImage(relpath):
+ if is_binary:
new_content = open(relpath, "rb").read()
-
+
return base_content, new_content, is_binary, status
# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
@@ -1838,42 +1983,46 @@
for attribute, value in options.__dict__.iteritems():
if attribute.startswith("p4") and value != None:
return (VCS_PERFORCE, None)
-
+
+ def RunDetectCommand(vcs_type, command):
+ """Helper to detect VCS by executing command.
+
+ Returns:
+ A pair (vcs, output) or None. Throws exception on error.
+ """
+ try:
+ out, returncode = RunShellWithReturnCode(command)
+ if returncode == 0:
+ return (vcs_type, out.strip())
+ except OSError, (errcode, message):
+ if errcode != errno.ENOENT: # command not found code
+ raise
+
# Mercurial has a command to get the base directory of a repository
# Try running it, but don't die if we don't have hg installed.
# NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
- try:
- out, returncode = RunShellWithReturnCode(["hg", "root"])
- if returncode == 0:
- return (VCS_MERCURIAL, out.strip())
- except OSError, (errno, message):
- if errno != 2: # ENOENT -- they don't have hg installed.
- raise
+ res = RunDetectCommand(VCS_MERCURIAL, ["hg", "root"])
+ if res != None:
+ return res
- # Subversion has a .svn in all working directories.
- if os.path.isdir('.svn'):
- logging.info("Guessed VCS = Subversion")
- return (VCS_SUBVERSION, None)
+ # Subversion from 1.7 has a single centralized .svn folder
+ # ( see http://subversion.apache.org/docs/release-notes/1.7.html#wc-ng )
+ # That's why we use 'svn info' instead of checking for .svn dir
+ res = RunDetectCommand(VCS_SUBVERSION, ["svn", "info"])
+ if res != None:
+ return res
# Git has a command to test if you're in a git tree.
# Try running it, but don't die if we don't have git installed.
- try:
- out, returncode = RunShellWithReturnCode(["git", "rev-parse",
- "--is-inside-work-tree"])
- if returncode == 0:
- return (VCS_GIT, None)
- except OSError, (errno, message):
- if errno != 2: # ENOENT -- they don't have git installed.
- raise
+ res = RunDetectCommand(VCS_GIT, ["git", "rev-parse",
+ "--is-inside-work-tree"])
+ if res != None:
+ return res
# detect CVS repos use `cvs status && $? == 0` rules
- try:
- out, returncode = RunShellWithReturnCode(["cvs", "status"])
- if returncode == 0:
- return (VCS_CVS, None)
- except OSError, (errno, message):
- if errno != 2:
- raise
+ res = RunDetectCommand(VCS_CVS, ["cvs", "status"])
+ if res != None:
+ return res
return (VCS_UNKNOWN, None)
@@ -2058,10 +2207,15 @@
The patchset id is None if the base files are not uploaded by this
script (applies only to SVN checkouts).
"""
- logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
- "%(lineno)s %(message)s "))
- os.environ['LC_ALL'] = 'C'
options, args = parser.parse_args(argv[1:])
+ if options.help:
+ if options.verbose < 2:
+ # hide Perforce options
+ parser.epilog = "Use '--help -v' to show additional Perforce options."
+ parser.option_groups.remove(parser.get_option_group('--p4_port'))
+ parser.print_help()
+ sys.exit(0)
+
global verbosity
verbosity = options.verbose
if verbosity >= 3:
@@ -2098,19 +2252,16 @@
files = vcs.GetBaseFiles(data)
if verbosity >= 1:
print "Upload server:", options.server, "(change with -s/--server)"
- if options.issue:
- prompt = "Message describing this patch set: "
- else:
- prompt = "New issue subject: "
- message = options.message or raw_input(prompt).strip()
- if not message:
- ErrorExit("A non-empty message is required")
rpc_server = GetRpcServer(options.server,
options.email,
options.host,
options.save_cookies,
options.account_type)
- form_fields = [("subject", message)]
+ form_fields = []
+
+ repo_guid = vcs.GetGUID()
+ if repo_guid:
+ form_fields.append(("repo_guid", repo_guid))
if base:
b = urlparse.urlparse(base)
username, netloc = urllib.splituser(b.netloc)
@@ -2131,15 +2282,38 @@
for cc in options.cc.split(','):
CheckReviewer(cc)
form_fields.append(("cc", options.cc))
- description = options.description
- if options.description_file:
- if options.description:
- ErrorExit("Can't specify description and description_file")
- file = open(options.description_file, 'r')
- description = file.read()
+
+ # Process --message, --title and --file.
+ message = options.message or ""
+ title = options.title or ""
+ if options.file:
+ if options.message:
+ ErrorExit("Can't specify both message and message file options")
+ file = open(options.file, 'r')
+ message = file.read()
file.close()
- if description:
- form_fields.append(("description", description))
+ if options.issue:
+ prompt = "Title describing this patch set: "
+ else:
+ prompt = "New issue subject: "
+ title = (
+ title or message.split('\n', 1)[0].strip() or raw_input(prompt).strip())
+ if not title and not options.issue:
+ ErrorExit("A non-empty title is required for a new issue")
+ # For existing issues, it's fine to give a patchset an empty name. Rietveld
+ # doesn't accept that so use a whitespace.
+ title = title or " "
+ if len(title) > 100:
+ title = title[:99] + '…'
+ if title and not options.issue:
+ message = message or title
+
+ form_fields.append(("subject", title))
+ # If it's a new issue send message as description. Otherwise a new
+ # message is created below on upload_complete.
+ if message and not options.issue:
+ form_fields.append(("description", message))
+
# Send a hash of all the base file so the server can determine if a copy
# already exists in an earlier patchset.
base_hashes = ""
@@ -2155,10 +2329,8 @@
print "Warning: Private flag ignored when updating an existing issue."
else:
form_fields.append(("private", "1"))
- # If we're uploading base files, don't send the email before the uploads, so
- # that it contains the file status.
- if options.send_mail and options.download_base:
- form_fields.append(("send_mail", "1"))
+ if options.send_patch:
+ options.send_mail = True
if not options.download_base:
form_fields.append(("content_upload", "1"))
if len(data) > MAX_UPLOAD_SIZE:
@@ -2193,13 +2365,25 @@
if not options.download_base:
vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
- if options.send_mail:
- rpc_server.Send("/" + issue + "/mail", payload="")
+
+ payload = {} # payload for final request
+ if options.send_mail:
+ payload["send_mail"] = "yes"
+ if options.send_patch:
+ payload["attach_patch"] = "yes"
+ if options.issue and message:
+ payload["message"] = message
+ payload = urllib.urlencode(payload)
+ rpc_server.Send("/" + issue + "/upload_complete/" + (patchset or ""),
+ payload=payload)
return issue, patchset
def main():
try:
+ logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
+ "%(lineno)s %(message)s "))
+ os.environ['LC_ALL'] = 'C'
RealMain(sys.argv)
except KeyboardInterrupt:
print