Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 1 | """Utilities for comparing files and directories. |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 2 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 3 | Classes: |
| 4 | dircmp |
| 5 | |
| 6 | Functions: |
Andrew M. Kuchling | 83e879d | 2003-02-06 19:38:45 +0000 | [diff] [blame] | 7 | cmp(f1, f2, shallow=1) -> int |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 8 | cmpfiles(a, b, common) -> ([], [], []) |
| 9 | |
| 10 | """ |
| 11 | |
| 12 | import os |
| 13 | import stat |
Andrew M. Kuchling | 83e879d | 2003-02-06 19:38:45 +0000 | [diff] [blame] | 14 | import warnings |
Raymond Hettinger | eeca37e | 2003-09-02 05:42:02 +0000 | [diff] [blame] | 15 | from itertools import ifilter, ifilterfalse, imap, izip |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 16 | |
Skip Montanaro | eccd02a | 2001-01-20 23:34:12 +0000 | [diff] [blame] | 17 | __all__ = ["cmp","dircmp","cmpfiles"] |
| 18 | |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 19 | _cache = {} |
| 20 | BUFSIZE=8*1024 |
| 21 | |
Raymond Hettinger | f3fa946 | 2004-12-05 01:58:09 +0000 | [diff] [blame] | 22 | def cmp(f1, f2, shallow=1): |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 23 | """Compare two files. |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 24 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 25 | Arguments: |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 26 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 27 | f1 -- First file name |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 28 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 29 | f2 -- Second file name |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 30 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 31 | shallow -- Just check stat signature (do not read the files). |
| 32 | defaults to 1. |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 33 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 34 | Return value: |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 35 | |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 36 | True if the files are the same, False otherwise. |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 37 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 38 | This function uses a cache for past comparisons and the results, |
| 39 | with a cache invalidation mechanism relying on stale signatures. |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 40 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 41 | """ |
Andrew M. Kuchling | 83e879d | 2003-02-06 19:38:45 +0000 | [diff] [blame] | 42 | |
Andrew M. Kuchling | 8eb4044 | 2003-02-06 17:50:01 +0000 | [diff] [blame] | 43 | s1 = _sig(os.stat(f1)) |
| 44 | s2 = _sig(os.stat(f2)) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 45 | if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 46 | return False |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 47 | if shallow and s1 == s2: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 48 | return True |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 49 | if s1[1] != s2[1]: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 50 | return False |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 51 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 52 | result = _cache.get((f1, f2)) |
| 53 | if result and (s1, s2) == result[:2]: |
| 54 | return result[2] |
| 55 | outcome = _do_cmp(f1, f2) |
| 56 | _cache[f1, f2] = s1, s2, outcome |
| 57 | return outcome |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 58 | |
| 59 | def _sig(st): |
Raymond Hettinger | 32200ae | 2002-06-01 19:51:15 +0000 | [diff] [blame] | 60 | return (stat.S_IFMT(st.st_mode), |
| 61 | st.st_size, |
| 62 | st.st_mtime) |
Guido van Rossum | 2d72687 | 1999-10-26 14:02:01 +0000 | [diff] [blame] | 63 | |
| 64 | def _do_cmp(f1, f2): |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 65 | bufsize = BUFSIZE |
| 66 | fp1 = open(f1, 'rb') |
| 67 | fp2 = open(f2, 'rb') |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 68 | while True: |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 69 | b1 = fp1.read(bufsize) |
| 70 | b2 = fp2.read(bufsize) |
| 71 | if b1 != b2: |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 72 | return False |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 73 | if not b1: |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 74 | return True |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 75 | |
| 76 | # Directory comparison class. |
| 77 | # |
| 78 | class dircmp: |
| 79 | """A class that manages the comparison of 2 directories. |
| 80 | |
| 81 | dircmp(a,b,ignore=None,hide=None) |
| 82 | A and B are directories. |
| 83 | IGNORE is a list of names to ignore, |
| 84 | defaults to ['RCS', 'CVS', 'tags']. |
| 85 | HIDE is a list of names to hide, |
| 86 | defaults to [os.curdir, os.pardir]. |
| 87 | |
| 88 | High level usage: |
| 89 | x = dircmp(dir1, dir2) |
| 90 | x.report() -> prints a report on the differences between dir1 and dir2 |
| 91 | or |
| 92 | x.report_partial_closure() -> prints report on differences between dir1 |
| 93 | and dir2, and reports on common immediate subdirectories. |
| 94 | x.report_full_closure() -> like report_partial_closure, |
| 95 | but fully recursive. |
| 96 | |
| 97 | Attributes: |
| 98 | left_list, right_list: The files in dir1 and dir2, |
| 99 | filtered by hide and ignore. |
| 100 | common: a list of names in both dir1 and dir2. |
| 101 | left_only, right_only: names only in dir1, dir2. |
| 102 | common_dirs: subdirectories in both dir1 and dir2. |
| 103 | common_files: files in both dir1 and dir2. |
| 104 | common_funny: names in both dir1 and dir2 where the type differs between |
| 105 | dir1 and dir2, or the name is not stat-able. |
| 106 | same_files: list of identical files. |
| 107 | diff_files: list of filenames which differ. |
| 108 | funny_files: list of files which could not be compared. |
| 109 | subdirs: a dictionary of dircmp objects, keyed by names in common_dirs. |
| 110 | """ |
| 111 | |
| 112 | def __init__(self, a, b, ignore=None, hide=None): # Initialize |
| 113 | self.left = a |
| 114 | self.right = b |
| 115 | if hide is None: |
| 116 | self.hide = [os.curdir, os.pardir] # Names never to be shown |
| 117 | else: |
| 118 | self.hide = hide |
| 119 | if ignore is None: |
| 120 | self.ignore = ['RCS', 'CVS', 'tags'] # Names ignored in comparison |
| 121 | else: |
| 122 | self.ignore = ignore |
| 123 | |
| 124 | def phase0(self): # Compare everything except common subdirectories |
| 125 | self.left_list = _filter(os.listdir(self.left), |
| 126 | self.hide+self.ignore) |
| 127 | self.right_list = _filter(os.listdir(self.right), |
| 128 | self.hide+self.ignore) |
| 129 | self.left_list.sort() |
| 130 | self.right_list.sort() |
| 131 | |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 132 | def phase1(self): # Compute common names |
Raymond Hettinger | eeca37e | 2003-09-02 05:42:02 +0000 | [diff] [blame] | 133 | a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list)) |
| 134 | b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list)) |
Guido van Rossum | c1f779c | 2007-07-03 08:25:58 +0000 | [diff] [blame] | 135 | self.common = list(map(a.__getitem__, ifilter(b.__contains__, a))) |
| 136 | self.left_only = list(map(a.__getitem__, ifilterfalse(b.__contains__, a))) |
| 137 | self.right_only = list(map(b.__getitem__, ifilterfalse(a.__contains__, b))) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 138 | |
| 139 | def phase2(self): # Distinguish files, directories, funnies |
| 140 | self.common_dirs = [] |
| 141 | self.common_files = [] |
| 142 | self.common_funny = [] |
| 143 | |
| 144 | for x in self.common: |
| 145 | a_path = os.path.join(self.left, x) |
| 146 | b_path = os.path.join(self.right, x) |
| 147 | |
| 148 | ok = 1 |
| 149 | try: |
Andrew M. Kuchling | 8eb4044 | 2003-02-06 17:50:01 +0000 | [diff] [blame] | 150 | a_stat = os.stat(a_path) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 151 | except os.error as why: |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 152 | # print 'Can\'t stat', a_path, ':', why[1] |
| 153 | ok = 0 |
| 154 | try: |
Andrew M. Kuchling | 8eb4044 | 2003-02-06 17:50:01 +0000 | [diff] [blame] | 155 | b_stat = os.stat(b_path) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 156 | except os.error as why: |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 157 | # print 'Can\'t stat', b_path, ':', why[1] |
| 158 | ok = 0 |
| 159 | |
| 160 | if ok: |
Raymond Hettinger | 32200ae | 2002-06-01 19:51:15 +0000 | [diff] [blame] | 161 | a_type = stat.S_IFMT(a_stat.st_mode) |
| 162 | b_type = stat.S_IFMT(b_stat.st_mode) |
Fred Drake | 8152d32 | 2000-12-12 23:20:45 +0000 | [diff] [blame] | 163 | if a_type != b_type: |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 164 | self.common_funny.append(x) |
| 165 | elif stat.S_ISDIR(a_type): |
| 166 | self.common_dirs.append(x) |
| 167 | elif stat.S_ISREG(a_type): |
| 168 | self.common_files.append(x) |
| 169 | else: |
| 170 | self.common_funny.append(x) |
| 171 | else: |
| 172 | self.common_funny.append(x) |
| 173 | |
| 174 | def phase3(self): # Find out differences between common files |
| 175 | xx = cmpfiles(self.left, self.right, self.common_files) |
| 176 | self.same_files, self.diff_files, self.funny_files = xx |
| 177 | |
| 178 | def phase4(self): # Find out differences between common subdirectories |
| 179 | # A new dircmp object is created for each common subdirectory, |
| 180 | # these are stored in a dictionary indexed by filename. |
| 181 | # The hide and ignore properties are inherited from the parent |
| 182 | self.subdirs = {} |
| 183 | for x in self.common_dirs: |
| 184 | a_x = os.path.join(self.left, x) |
| 185 | b_x = os.path.join(self.right, x) |
| 186 | self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide) |
| 187 | |
| 188 | def phase4_closure(self): # Recursively call phase4() on subdirectories |
| 189 | self.phase4() |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 190 | for sd in self.subdirs.values(): |
Raymond Hettinger | e0d4972 | 2002-06-02 18:55:56 +0000 | [diff] [blame] | 191 | sd.phase4_closure() |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 192 | |
| 193 | def report(self): # Print a report on the differences between a and b |
| 194 | # Output format is purposely lousy |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 195 | print('diff', self.left, self.right) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 196 | if self.left_only: |
| 197 | self.left_only.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 198 | print('Only in', self.left, ':', self.left_only) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 199 | if self.right_only: |
| 200 | self.right_only.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 201 | print('Only in', self.right, ':', self.right_only) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 202 | if self.same_files: |
| 203 | self.same_files.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 204 | print('Identical files :', self.same_files) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 205 | if self.diff_files: |
| 206 | self.diff_files.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 207 | print('Differing files :', self.diff_files) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 208 | if self.funny_files: |
| 209 | self.funny_files.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 210 | print('Trouble with common files :', self.funny_files) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 211 | if self.common_dirs: |
| 212 | self.common_dirs.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 213 | print('Common subdirectories :', self.common_dirs) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 214 | if self.common_funny: |
| 215 | self.common_funny.sort() |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 216 | print('Common funny cases :', self.common_funny) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 217 | |
| 218 | def report_partial_closure(self): # Print reports on self and on subdirs |
| 219 | self.report() |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 220 | for sd in self.subdirs.values(): |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 221 | print() |
Raymond Hettinger | e0d4972 | 2002-06-02 18:55:56 +0000 | [diff] [blame] | 222 | sd.report() |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 223 | |
| 224 | def report_full_closure(self): # Report on self and subdirs recursively |
| 225 | self.report() |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 226 | for sd in self.subdirs.values(): |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 227 | print() |
Raymond Hettinger | e0d4972 | 2002-06-02 18:55:56 +0000 | [diff] [blame] | 228 | sd.report_full_closure() |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 229 | |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 230 | methodmap = dict(subdirs=phase4, |
| 231 | same_files=phase3, diff_files=phase3, funny_files=phase3, |
| 232 | common_dirs = phase2, common_files=phase2, common_funny=phase2, |
| 233 | common=phase1, left_only=phase1, right_only=phase1, |
| 234 | left_list=phase0, right_list=phase0) |
| 235 | |
| 236 | def __getattr__(self, attr): |
| 237 | if attr not in self.methodmap: |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 238 | raise AttributeError(attr) |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 239 | self.methodmap[attr](self) |
| 240 | return getattr(self, attr) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 241 | |
Raymond Hettinger | f3fa946 | 2004-12-05 01:58:09 +0000 | [diff] [blame] | 242 | def cmpfiles(a, b, common, shallow=1): |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 243 | """Compare common files in two directories. |
| 244 | |
Fred Drake | 2b0d98b | 2000-07-03 08:18:47 +0000 | [diff] [blame] | 245 | a, b -- directory names |
| 246 | common -- list of file names found in both directories |
| 247 | shallow -- if true, do comparison based solely on stat() information |
Fred Drake | 2b0d98b | 2000-07-03 08:18:47 +0000 | [diff] [blame] | 248 | |
| 249 | Returns a tuple of three lists: |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 250 | files that compare equal |
| 251 | files that are different |
Fred Drake | 2b0d98b | 2000-07-03 08:18:47 +0000 | [diff] [blame] | 252 | filenames that aren't regular files. |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 253 | |
Fred Drake | 2b0d98b | 2000-07-03 08:18:47 +0000 | [diff] [blame] | 254 | """ |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 255 | res = ([], [], []) |
| 256 | for x in common: |
Fred Drake | 2b0d98b | 2000-07-03 08:18:47 +0000 | [diff] [blame] | 257 | ax = os.path.join(a, x) |
| 258 | bx = os.path.join(b, x) |
Andrew M. Kuchling | 8eb4044 | 2003-02-06 17:50:01 +0000 | [diff] [blame] | 259 | res[_cmp(ax, bx, shallow)].append(x) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 260 | return res |
| 261 | |
| 262 | |
| 263 | # Compare two files. |
| 264 | # Return: |
Tim Peters | 88869f9 | 2001-01-14 23:36:06 +0000 | [diff] [blame] | 265 | # 0 for equal |
| 266 | # 1 for different |
| 267 | # 2 for funny cases (can't stat, etc.) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 268 | # |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 269 | def _cmp(a, b, sh, abs=abs, cmp=cmp): |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 270 | try: |
Andrew M. Kuchling | 8eb4044 | 2003-02-06 17:50:01 +0000 | [diff] [blame] | 271 | return not abs(cmp(a, b, sh)) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 272 | except os.error: |
| 273 | return 2 |
| 274 | |
| 275 | |
| 276 | # Return a copy with items that occur in skip removed. |
| 277 | # |
Raymond Hettinger | 05595e9 | 2003-02-27 00:05:31 +0000 | [diff] [blame] | 278 | def _filter(flist, skip): |
| 279 | return list(ifilterfalse(skip.__contains__, flist)) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 280 | |
| 281 | |
| 282 | # Demonstration and testing. |
| 283 | # |
| 284 | def demo(): |
| 285 | import sys |
| 286 | import getopt |
| 287 | options, args = getopt.getopt(sys.argv[1:], 'r') |
Fred Drake | 8152d32 | 2000-12-12 23:20:45 +0000 | [diff] [blame] | 288 | if len(args) != 2: |
Andrew M. Kuchling | 83e879d | 2003-02-06 19:38:45 +0000 | [diff] [blame] | 289 | raise getopt.GetoptError('need exactly two args', None) |
Guido van Rossum | 63b08ac | 2000-06-29 14:13:28 +0000 | [diff] [blame] | 290 | dd = dircmp(args[0], args[1]) |
| 291 | if ('-r', '') in options: |
| 292 | dd.report_full_closure() |
| 293 | else: |
| 294 | dd.report() |
| 295 | |
| 296 | if __name__ == '__main__': |
| 297 | demo() |