| #!/usr/bin/python |
| ''' |
| Copyright 2012 Google Inc. |
| |
| Use of this source code is governed by a BSD-style license that can be |
| found in the LICENSE file. |
| ''' |
| |
| ''' |
| Generates a visual diff of all pending changes in the local SVN (or git!) |
| checkout. |
| |
| Launch with --help to see more information. |
| |
| TODO(epoger): Now that this tool supports either git or svn, rename it. |
| TODO(epoger): Fix indentation in this file (2-space indents, not 4-space). |
| ''' |
| |
| # common Python modules |
| import optparse |
| import os |
| import posixpath |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import urllib2 |
| |
| # Imports from within Skia |
| # |
| # We need to add the 'gm' directory, so that we can import gm_json.py within |
| # that directory. That script allows us to parse the actual-results.json file |
| # written out by the GM tool. |
| # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
| # so any dirs that are already in the PYTHONPATH will be preferred. |
| # |
| # This assumes that the 'gm' directory has been checked out as a sibling of |
| # the 'tools' directory containing this script, which will be the case if |
| # 'trunk' was checked out as a single unit. |
| GM_DIRECTORY = os.path.realpath( |
| os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) |
| if GM_DIRECTORY not in sys.path: |
| sys.path.append(GM_DIRECTORY) |
| import gm_json |
| import jsondiff |
| import svn |
| |
| USAGE_STRING = 'Usage: %s [options]' |
| HELP_STRING = ''' |
| |
| Generates a visual diff of all pending changes in the local SVN/git checkout. |
| |
| This includes a list of all files that have been added, deleted, or modified |
| (as far as SVN/git knows about). For any image modifications, pixel diffs will |
| be generated. |
| |
| ''' |
| |
| IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
| |
| TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir) |
| |
| OPTION_DEST_DIR = '--dest-dir' |
| OPTION_PATH_TO_SKDIFF = '--path-to-skdiff' |
| OPTION_SOURCE_DIR = '--source-dir' |
| |
| def RunCommand(command): |
| """Run a command, raising an exception if it fails. |
| |
| @param command the command as a single string |
| """ |
| print 'running command [%s]...' % command |
| retval = os.system(command) |
| if retval is not 0: |
| raise Exception('command [%s] failed' % command) |
| |
| def FindPathToSkDiff(user_set_path=None): |
| """Return path to an existing skdiff binary, or raise an exception if we |
| cannot find one. |
| |
| @param user_set_path if None, the user did not specify a path, so look in |
| some likely places; otherwise, only check at this path |
| """ |
| if user_set_path is not None: |
| if os.path.isfile(user_set_path): |
| return user_set_path |
| raise Exception('unable to find skdiff at user-set path %s' % |
| user_set_path) |
| trunk_path = os.path.join(os.path.dirname(__file__), os.pardir) |
| |
| extension = '' |
| if os.name is 'nt': |
| extension = '.exe' |
| |
| possible_paths = [os.path.join(trunk_path, 'out', 'Release', |
| 'skdiff' + extension), |
| os.path.join(trunk_path, 'out', 'Debug', |
| 'skdiff' + extension)] |
| for try_path in possible_paths: |
| if os.path.isfile(try_path): |
| return try_path |
| raise Exception('cannot find skdiff in paths %s; maybe you need to ' |
| 'specify the %s option or build skdiff?' % ( |
| possible_paths, OPTION_PATH_TO_SKDIFF)) |
| |
| def _DownloadUrlToFile(source_url, dest_path): |
| """Download source_url, and save its contents to dest_path. |
| Raises an exception if there were any problems.""" |
| try: |
| reader = urllib2.urlopen(source_url) |
| writer = open(dest_path, 'wb') |
| writer.write(reader.read()) |
| writer.close() |
| except BaseException as e: |
| raise Exception( |
| '%s: unable to download source_url %s to dest_path %s' % ( |
| e, source_url, dest_path)) |
| |
| def _CreateGSUrl(imagename, hash_type, hash_digest): |
| """Return the HTTP URL we can use to download this particular version of |
| the actually-generated GM image with this imagename. |
| |
| imagename: name of the test image, e.g. 'perlinnoise_msaa4.png' |
| hash_type: string indicating the hash type used to generate hash_digest, |
| e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 |
| hash_digest: the hash digest of the image to retrieve |
| """ |
| return gm_json.CreateGmActualUrl( |
| test_name=IMAGE_FILENAME_RE.match(imagename).group(1), |
| hash_type=hash_type, |
| hash_digest=hash_digest) |
| |
| def _CallJsonDiff(old_json_path, new_json_path, |
| old_flattened_dir, new_flattened_dir, |
| filename_prefix): |
| """Using jsondiff.py, write the images that differ between two GM |
| expectations summary files (old and new) into old_flattened_dir and |
| new_flattened_dir. |
| |
| filename_prefix: prefix to prepend to filenames of all images we write |
| into the flattened directories |
| """ |
| json_differ = jsondiff.GMDiffer() |
| diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path, |
| newfile=new_json_path) |
| print 'Downloading %d before-and-after image pairs...' % len(diff_dict) |
| for (imagename, results) in diff_dict.iteritems(): |
| # TODO(epoger): Currently, this assumes that all images have been |
| # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 |
| |
| old_checksum = results['old'] |
| if old_checksum: |
| old_image_url = _CreateGSUrl( |
| imagename=imagename, |
| hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, |
| hash_digest=old_checksum) |
| _DownloadUrlToFile( |
| source_url=old_image_url, |
| dest_path=os.path.join(old_flattened_dir, |
| filename_prefix + imagename)) |
| |
| new_checksum = results['new'] |
| if new_checksum: |
| new_image_url = _CreateGSUrl( |
| imagename=imagename, |
| hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, |
| hash_digest=new_checksum) |
| _DownloadUrlToFile( |
| source_url=new_image_url, |
| dest_path=os.path.join(new_flattened_dir, |
| filename_prefix + imagename)) |
| |
| def _RunCommand(args): |
| """Run a command (from self._directory) and return stdout as a single |
| string. |
| |
| @param args a list of arguments |
| """ |
| proc = subprocess.Popen(args, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE) |
| (stdout, stderr) = proc.communicate() |
| if proc.returncode is not 0: |
| raise Exception('command "%s" failed: %s' % (args, stderr)) |
| return stdout |
| |
| def _GitGetModifiedFiles(): |
| """Returns a list of locally modified files within the current working dir. |
| |
| TODO(epoger): Move this into a git utility package? |
| """ |
| return _RunCommand(['git', 'ls-files', '-m']).splitlines() |
| |
| def _GitExportBaseVersionOfFile(file_within_repo, dest_path): |
| """Retrieves a copy of the base version of a file within the repository. |
| |
| @param file_within_repo path to the file within the repo whose base |
| version you wish to obtain |
| @param dest_path destination to which to write the base content |
| |
| TODO(epoger): Move this into a git utility package? |
| """ |
| # TODO(epoger): Replace use of "git show" command with lower-level git |
| # commands? senorblanco points out that "git show" is a "porcelain" |
| # command, intended for human use, as opposed to the "plumbing" commands |
| # generally more suitable for scripting. (See |
| # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain ) |
| # |
| # For now, though, "git show" is the most straightforward implementation |
| # I could come up with. I tried using "git cat-file", but I had trouble |
| # getting it to work as desired. |
| # Note that git expects / rather than \ as a path separator even on |
| # windows. |
| args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)] |
| with open(dest_path, 'wb') as outfile: |
| proc = subprocess.Popen(args, stdout=outfile) |
| proc.communicate() |
| if proc.returncode is not 0: |
| raise Exception('command "%s" failed' % args) |
| |
| def SvnDiff(path_to_skdiff, dest_dir, source_dir): |
| """Generates a visual diff of all pending changes in source_dir. |
| |
| @param path_to_skdiff |
| @param dest_dir existing directory within which to write results |
| @param source_dir |
| """ |
| # Validate parameters, filling in default values if necessary and possible. |
| path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff)) |
| if not dest_dir: |
| dest_dir = tempfile.mkdtemp() |
| dest_dir = os.path.abspath(dest_dir) |
| |
| os.chdir(source_dir) |
| svn_repo = svn.Svn('.') |
| using_svn = True |
| try: |
| svn_repo.GetInfo() |
| except: |
| using_svn = False |
| |
| # Prepare temporary directories. |
| modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened') |
| original_flattened_dir = os.path.join(dest_dir, 'original_flattened') |
| diff_dir = os.path.join(dest_dir, 'diffs') |
| for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] : |
| shutil.rmtree(dir, ignore_errors=True) |
| os.mkdir(dir) |
| |
| # Get a list of all locally modified (including added/deleted) files, |
| # descending subdirectories. |
| if using_svn: |
| modified_file_paths = svn_repo.GetFilesWithStatus( |
| svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED) |
| else: |
| modified_file_paths = _GitGetModifiedFiles() |
| |
| # For each modified file: |
| # 1. copy its current contents into modified_flattened_dir |
| # 2. copy its original contents into original_flattened_dir |
| for modified_file_path in modified_file_paths: |
| if modified_file_path.endswith('.json'): |
| # Special handling for JSON files, in the hopes that they |
| # contain GM result summaries. |
| original_file = tempfile.NamedTemporaryFile(delete = False) |
| original_file.close() |
| if using_svn: |
| svn_repo.ExportBaseVersionOfFile( |
| modified_file_path, original_file.name) |
| else: |
| _GitExportBaseVersionOfFile( |
| modified_file_path, original_file.name) |
| modified_dir = os.path.dirname(modified_file_path) |
| platform_prefix = (re.sub(re.escape(os.sep), '__', |
| os.path.splitdrive(modified_dir)[1]) |
| + '__') |
| _CallJsonDiff(old_json_path=original_file.name, |
| new_json_path=modified_file_path, |
| old_flattened_dir=original_flattened_dir, |
| new_flattened_dir=modified_flattened_dir, |
| filename_prefix=platform_prefix) |
| os.remove(original_file.name) |
| else: |
| dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path) |
| # If the file had STATUS_DELETED, it won't exist anymore... |
| if os.path.isfile(modified_file_path): |
| shutil.copyfile(modified_file_path, |
| os.path.join(modified_flattened_dir, |
| dest_filename)) |
| if using_svn: |
| svn_repo.ExportBaseVersionOfFile( |
| modified_file_path, |
| os.path.join(original_flattened_dir, dest_filename)) |
| else: |
| _GitExportBaseVersionOfFile( |
| modified_file_path, |
| os.path.join(original_flattened_dir, dest_filename)) |
| |
| # Run skdiff: compare original_flattened_dir against modified_flattened_dir |
| RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir, |
| modified_flattened_dir, diff_dir)) |
| print '\nskdiff results are ready in file://%s/index.html' % diff_dir |
| |
| def RaiseUsageException(): |
| raise Exception('%s\nRun with --help for more detail.' % ( |
| USAGE_STRING % __file__)) |
| |
| def Main(options, args): |
| """Allow other scripts to call this script with fake command-line args. |
| """ |
| num_args = len(args) |
| if num_args != 0: |
| RaiseUsageException() |
| SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir, |
| source_dir=options.source_dir) |
| |
| if __name__ == '__main__': |
| parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) |
| parser.add_option(OPTION_DEST_DIR, |
| action='store', type='string', default=None, |
| help='existing directory within which to write results; ' |
| 'if not set, will create a temporary directory which ' |
| 'will remain in place after this script completes') |
| parser.add_option(OPTION_PATH_TO_SKDIFF, |
| action='store', type='string', default=None, |
| help='path to already-built skdiff tool; if not set, ' |
| 'will search for it in typical directories near this ' |
| 'script') |
| parser.add_option(OPTION_SOURCE_DIR, |
| action='store', type='string', |
| default=os.path.join('expectations', 'gm'), |
| help='root directory within which to compare all ' + |
| 'files; defaults to "%default"') |
| (options, args) = parser.parse_args() |
| Main(options, args) |