blob: 4f9618512aa6d566c5e31c5c7fcb10a0360aa10d [file] [log] [blame]
epoger@google.com6e1e7852013-07-10 18:09:55 +00001#!/usr/bin/python
epoger@google.com2e0a0612012-05-25 19:48:05 +00002'''
epoger@google.com2e0a0612012-05-25 19:48:05 +00003Copyright 2012 Google Inc.
4
5Use of this source code is governed by a BSD-style license that can be
6found in the LICENSE file.
7'''
8
epoger@google.com61822a22013-07-16 18:56:32 +00009'''
epoger@google.com627858b2013-07-18 18:45:17 +000010Generates a visual diff of all pending changes in the local SVN (or git!)
11checkout.
epoger@google.com61822a22013-07-16 18:56:32 +000012
13Launch with --help to see more information.
14
epoger@google.com627858b2013-07-18 18:45:17 +000015TODO(epoger): Now that this tool supports either git or svn, rename it.
epoger@google.com61822a22013-07-16 18:56:32 +000016TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
17'''
18
epoger@google.com2e0a0612012-05-25 19:48:05 +000019# common Python modules
20import optparse
21import os
22import re
23import shutil
epoger@google.com627858b2013-07-18 18:45:17 +000024import subprocess
epoger@google.com61822a22013-07-16 18:56:32 +000025import sys
epoger@google.com2e0a0612012-05-25 19:48:05 +000026import tempfile
epoger@google.com61822a22013-07-16 18:56:32 +000027import urllib2
epoger@google.com2e0a0612012-05-25 19:48:05 +000028
epoger@google.com61822a22013-07-16 18:56:32 +000029# Imports from within Skia
30#
31# We need to add the 'gm' directory, so that we can import gm_json.py within
32# that directory. That script allows us to parse the actual-results.json file
33# written out by the GM tool.
34# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
35# so any dirs that are already in the PYTHONPATH will be preferred.
36#
37# This assumes that the 'gm' directory has been checked out as a sibling of
38# the 'tools' directory containing this script, which will be the case if
39# 'trunk' was checked out as a single unit.
40GM_DIRECTORY = os.path.realpath(
41 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
42if GM_DIRECTORY not in sys.path:
43 sys.path.append(GM_DIRECTORY)
44import gm_json
45import jsondiff
epoger@google.com2e0a0612012-05-25 19:48:05 +000046import svn
47
48USAGE_STRING = 'Usage: %s [options]'
49HELP_STRING = '''
50
epoger@google.com627858b2013-07-18 18:45:17 +000051Generates a visual diff of all pending changes in the local SVN/git checkout.
epoger@google.com6dbf6cd2012-05-29 21:28:12 +000052
53This includes a list of all files that have been added, deleted, or modified
epoger@google.com627858b2013-07-18 18:45:17 +000054(as far as SVN/git knows about). For any image modifications, pixel diffs will
epoger@google.com6dbf6cd2012-05-29 21:28:12 +000055be generated.
epoger@google.com2e0a0612012-05-25 19:48:05 +000056
57'''
58
epoger@google.com61822a22013-07-16 18:56:32 +000059IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
60
epoger@google.com2e0a0612012-05-25 19:48:05 +000061TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
62
63OPTION_DEST_DIR = '--dest-dir'
epoger@google.com2e0a0612012-05-25 19:48:05 +000064OPTION_PATH_TO_SKDIFF = '--path-to-skdiff'
epoger@google.com61822a22013-07-16 18:56:32 +000065OPTION_SOURCE_DIR = '--source-dir'
epoger@google.com2e0a0612012-05-25 19:48:05 +000066
67def RunCommand(command):
68 """Run a command, raising an exception if it fails.
69
70 @param command the command as a single string
71 """
72 print 'running command [%s]...' % command
73 retval = os.system(command)
74 if retval is not 0:
75 raise Exception('command [%s] failed' % command)
76
77def FindPathToSkDiff(user_set_path=None):
78 """Return path to an existing skdiff binary, or raise an exception if we
79 cannot find one.
80
81 @param user_set_path if None, the user did not specify a path, so look in
82 some likely places; otherwise, only check at this path
83 """
84 if user_set_path is not None:
85 if os.path.isfile(user_set_path):
86 return user_set_path
87 raise Exception('unable to find skdiff at user-set path %s' %
88 user_set_path)
89 trunk_path = os.path.join(os.path.dirname(__file__), os.pardir)
90 possible_paths = [os.path.join(trunk_path, 'out', 'Release', 'skdiff'),
91 os.path.join(trunk_path, 'out', 'Debug', 'skdiff')]
92 for try_path in possible_paths:
93 if os.path.isfile(try_path):
94 return try_path
95 raise Exception('cannot find skdiff in paths %s; maybe you need to '
96 'specify the %s option or build skdiff?' % (
97 possible_paths, OPTION_PATH_TO_SKDIFF))
98
epoger@google.com61822a22013-07-16 18:56:32 +000099def _DownloadUrlToFile(source_url, dest_path):
100 """Download source_url, and save its contents to dest_path.
101 Raises an exception if there were any problems."""
epoger@google.com48bceed2013-07-16 23:37:01 +0000102 try:
103 reader = urllib2.urlopen(source_url)
104 writer = open(dest_path, 'wb')
105 writer.write(reader.read())
106 writer.close()
107 except BaseException as e:
108 raise Exception(
109 '%s: unable to download source_url %s to dest_path %s' % (
110 e, source_url, dest_path))
epoger@google.com61822a22013-07-16 18:56:32 +0000111
112def _CreateGSUrl(imagename, hash_type, hash_digest):
113 """Return the HTTP URL we can use to download this particular version of
114 the actually-generated GM image with this imagename.
115
116 imagename: name of the test image, e.g. 'perlinnoise_msaa4.png'
117 hash_type: string indicating the hash type used to generate hash_digest,
118 e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
119 hash_digest: the hash digest of the image to retrieve
120 """
121 return gm_json.CreateGmActualUrl(
122 test_name=IMAGE_FILENAME_RE.match(imagename).group(1),
123 hash_type=hash_type,
124 hash_digest=hash_digest)
125
126def _CallJsonDiff(old_json_path, new_json_path,
127 old_flattened_dir, new_flattened_dir,
128 filename_prefix):
129 """Using jsondiff.py, write the images that differ between two GM
130 expectations summary files (old and new) into old_flattened_dir and
131 new_flattened_dir.
132
133 filename_prefix: prefix to prepend to filenames of all images we write
134 into the flattened directories
135 """
136 json_differ = jsondiff.GMDiffer()
137 diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path,
138 newfile=new_json_path)
epoger@google.com48bceed2013-07-16 23:37:01 +0000139 print 'Downloading %d before-and-after image pairs...' % len(diff_dict)
epoger@google.com61822a22013-07-16 18:56:32 +0000140 for (imagename, results) in diff_dict.iteritems():
epoger@google.com61822a22013-07-16 18:56:32 +0000141 # TODO(epoger): Currently, this assumes that all images have been
142 # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
epoger@google.com48bceed2013-07-16 23:37:01 +0000143
144 old_checksum = results['old']
145 if old_checksum:
146 old_image_url = _CreateGSUrl(
147 imagename=imagename,
148 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
149 hash_digest=old_checksum)
djsollen@google.com64f031c2013-10-07 14:02:58 +0000150 _DownloadUrlToFile(
151 source_url=old_image_url,
152 dest_path=os.path.join(old_flattened_dir,
153 filename_prefix + imagename))
epoger@google.com48bceed2013-07-16 23:37:01 +0000154
155 new_checksum = results['new']
156 if new_checksum:
157 new_image_url = _CreateGSUrl(
158 imagename=imagename,
159 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
160 hash_digest=new_checksum)
161 _DownloadUrlToFile(
162 source_url=new_image_url,
163 dest_path=os.path.join(new_flattened_dir,
164 filename_prefix + imagename))
epoger@google.com61822a22013-07-16 18:56:32 +0000165
epoger@google.com627858b2013-07-18 18:45:17 +0000166def _RunCommand(args):
167 """Run a command (from self._directory) and return stdout as a single
168 string.
169
170 @param args a list of arguments
171 """
172 proc = subprocess.Popen(args,
173 stdout=subprocess.PIPE,
174 stderr=subprocess.PIPE)
175 (stdout, stderr) = proc.communicate()
176 if proc.returncode is not 0:
177 raise Exception('command "%s" failed: %s' % (args, stderr))
178 return stdout
179
180def _GitGetModifiedFiles():
181 """Returns a list of locally modified files within the current working dir.
182
183 TODO(epoger): Move this into a git utility package?
184 """
185 return _RunCommand(['git', 'ls-files', '-m']).splitlines()
186
187def _GitExportBaseVersionOfFile(file_within_repo, dest_path):
188 """Retrieves a copy of the base version of a file within the repository.
189
190 @param file_within_repo path to the file within the repo whose base
191 version you wish to obtain
192 @param dest_path destination to which to write the base content
193
194 TODO(epoger): Move this into a git utility package?
195 """
196 # TODO(epoger): Replace use of "git show" command with lower-level git
197 # commands? senorblanco points out that "git show" is a "porcelain"
198 # command, intended for human use, as opposed to the "plumbing" commands
199 # generally more suitable for scripting. (See
200 # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain )
201 #
202 # For now, though, "git show" is the most straightforward implementation
203 # I could come up with. I tried using "git cat-file", but I had trouble
204 # getting it to work as desired.
205 args = ['git', 'show', os.path.join('HEAD:.', file_within_repo)]
206 with open(dest_path, 'wb') as outfile:
207 proc = subprocess.Popen(args, stdout=outfile)
208 proc.communicate()
209 if proc.returncode is not 0:
210 raise Exception('command "%s" failed' % args)
211
epoger@google.com61822a22013-07-16 18:56:32 +0000212def SvnDiff(path_to_skdiff, dest_dir, source_dir):
213 """Generates a visual diff of all pending changes in source_dir.
epoger@google.com2e0a0612012-05-25 19:48:05 +0000214
215 @param path_to_skdiff
216 @param dest_dir existing directory within which to write results
epoger@google.com61822a22013-07-16 18:56:32 +0000217 @param source_dir
epoger@google.com2e0a0612012-05-25 19:48:05 +0000218 """
219 # Validate parameters, filling in default values if necessary and possible.
epoger@google.com61822a22013-07-16 18:56:32 +0000220 path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
epoger@google.com2e0a0612012-05-25 19:48:05 +0000221 if not dest_dir:
222 dest_dir = tempfile.mkdtemp()
epoger@google.com61822a22013-07-16 18:56:32 +0000223 dest_dir = os.path.abspath(dest_dir)
224
225 os.chdir(source_dir)
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000226 svn_repo = svn.Svn('.')
227 using_svn = True
228 try:
229 svn_repo.GetInfo()
230 except:
231 using_svn = False
epoger@google.com2e0a0612012-05-25 19:48:05 +0000232
233 # Prepare temporary directories.
234 modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
235 original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
236 diff_dir = os.path.join(dest_dir, 'diffs')
237 for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
238 shutil.rmtree(dir, ignore_errors=True)
239 os.mkdir(dir)
240
epoger@google.com6dbf6cd2012-05-29 21:28:12 +0000241 # Get a list of all locally modified (including added/deleted) files,
242 # descending subdirectories.
epoger@google.com627858b2013-07-18 18:45:17 +0000243 if using_svn:
epoger@google.com627858b2013-07-18 18:45:17 +0000244 modified_file_paths = svn_repo.GetFilesWithStatus(
245 svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
246 else:
247 modified_file_paths = _GitGetModifiedFiles()
epoger@google.com2e0a0612012-05-25 19:48:05 +0000248
249 # For each modified file:
250 # 1. copy its current contents into modified_flattened_dir
251 # 2. copy its original contents into original_flattened_dir
252 for modified_file_path in modified_file_paths:
epoger@google.com61822a22013-07-16 18:56:32 +0000253 if modified_file_path.endswith('.json'):
254 # Special handling for JSON files, in the hopes that they
255 # contain GM result summaries.
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000256 original_file = tempfile.NamedTemporaryFile(delete = False)
257 original_file.close()
epoger@google.com627858b2013-07-18 18:45:17 +0000258 if using_svn:
259 svn_repo.ExportBaseVersionOfFile(
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000260 modified_file_path, original_file.name)
epoger@google.com627858b2013-07-18 18:45:17 +0000261 else:
262 _GitExportBaseVersionOfFile(
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000263 modified_file_path, original_file.name)
bungeman@google.comcc5fd0a2013-10-10 14:16:23 +0000264 modified_dir = os.path.dirname(modified_file_path)
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000265 platform_prefix = (re.sub(re.escape(os.sep), '__',
bungeman@google.comcc5fd0a2013-10-10 14:16:23 +0000266 os.path.splitdrive(modified_dir)[1])
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000267 + '__')
268 _CallJsonDiff(old_json_path=original_file.name,
epoger@google.com61822a22013-07-16 18:56:32 +0000269 new_json_path=modified_file_path,
270 old_flattened_dir=original_flattened_dir,
271 new_flattened_dir=modified_flattened_dir,
272 filename_prefix=platform_prefix)
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000273 os.remove(original_file.name)
epoger@google.com61822a22013-07-16 18:56:32 +0000274 else:
bungeman@google.com3c8d9cb2013-10-07 19:57:35 +0000275 dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path)
epoger@google.com61822a22013-07-16 18:56:32 +0000276 # If the file had STATUS_DELETED, it won't exist anymore...
277 if os.path.isfile(modified_file_path):
278 shutil.copyfile(modified_file_path,
epoger@google.com48bceed2013-07-16 23:37:01 +0000279 os.path.join(modified_flattened_dir,
280 dest_filename))
epoger@google.com627858b2013-07-18 18:45:17 +0000281 if using_svn:
282 svn_repo.ExportBaseVersionOfFile(
283 modified_file_path,
284 os.path.join(original_flattened_dir, dest_filename))
285 else:
286 _GitExportBaseVersionOfFile(
287 modified_file_path,
288 os.path.join(original_flattened_dir, dest_filename))
epoger@google.com2e0a0612012-05-25 19:48:05 +0000289
290 # Run skdiff: compare original_flattened_dir against modified_flattened_dir
291 RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
292 modified_flattened_dir, diff_dir))
293 print '\nskdiff results are ready in file://%s/index.html' % diff_dir
294
295def RaiseUsageException():
296 raise Exception('%s\nRun with --help for more detail.' % (
297 USAGE_STRING % __file__))
298
299def Main(options, args):
300 """Allow other scripts to call this script with fake command-line args.
301 """
302 num_args = len(args)
303 if num_args != 0:
304 RaiseUsageException()
epoger@google.com61822a22013-07-16 18:56:32 +0000305 SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir,
306 source_dir=options.source_dir)
epoger@google.com2e0a0612012-05-25 19:48:05 +0000307
308if __name__ == '__main__':
309 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
310 parser.add_option(OPTION_DEST_DIR,
311 action='store', type='string', default=None,
312 help='existing directory within which to write results; '
313 'if not set, will create a temporary directory which '
314 'will remain in place after this script completes')
315 parser.add_option(OPTION_PATH_TO_SKDIFF,
316 action='store', type='string', default=None,
317 help='path to already-built skdiff tool; if not set, '
318 'will search for it in typical directories near this '
319 'script')
epoger@google.com61822a22013-07-16 18:56:32 +0000320 parser.add_option(OPTION_SOURCE_DIR,
epoger@google.come94a7d22013-07-23 19:37:03 +0000321 action='store', type='string',
322 default=os.path.join('expectations', 'gm'),
epoger@google.com61822a22013-07-16 18:56:32 +0000323 help='root directory within which to compare all ' +
324 'files; defaults to "%default"')
epoger@google.com2e0a0612012-05-25 19:48:05 +0000325 (options, args) = parser.parse_args()
326 Main(options, args)