Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 1 | |
| 2 | """ |
| 3 | csv.py - read/write/investigate CSV files |
| 4 | """ |
| 5 | |
| 6 | import re |
| 7 | from _csv import Error, __version__, writer, reader, register_dialect, \ |
| 8 | unregister_dialect, get_dialect, list_dialects, \ |
Andrew McNamara | 31d8896 | 2005-01-12 03:45:10 +0000 | [diff] [blame] | 9 | field_size_limit, \ |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 10 | QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ |
| 11 | __doc__ |
Andrew McNamara | 7130ff5 | 2005-01-11 02:22:47 +0000 | [diff] [blame] | 12 | from _csv import Dialect as _Dialect |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 13 | |
Guido van Rossum | 68937b4 | 2007-05-18 00:51:22 +0000 | [diff] [blame] | 14 | from io import StringIO |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 15 | |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 16 | __all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", |
Brett Cannon | e8d0bf9 | 2008-08-18 00:51:19 +0000 | [diff] [blame] | 17 | "Error", "Dialect", "__doc__", "excel", "excel_tab", |
| 18 | "field_size_limit", "reader", "writer", |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 19 | "register_dialect", "get_dialect", "list_dialects", "Sniffer", |
| 20 | "unregister_dialect", "__version__", "DictReader", "DictWriter" ] |
| 21 | |
| 22 | class Dialect: |
Skip Montanaro | f26285c | 2005-01-05 06:54:58 +0000 | [diff] [blame] | 23 | """Describe an Excel dialect. |
| 24 | |
| 25 | This must be subclassed (see csv.excel). Valid attributes are: |
| 26 | delimiter, quotechar, escapechar, doublequote, skipinitialspace, |
| 27 | lineterminator, quoting. |
| 28 | |
| 29 | """ |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 30 | _name = "" |
| 31 | _valid = False |
| 32 | # placeholders |
| 33 | delimiter = None |
| 34 | quotechar = None |
| 35 | escapechar = None |
| 36 | doublequote = None |
| 37 | skipinitialspace = None |
| 38 | lineterminator = None |
| 39 | quoting = None |
| 40 | |
| 41 | def __init__(self): |
| 42 | if self.__class__ != Dialect: |
| 43 | self._valid = True |
Andrew McNamara | 7130ff5 | 2005-01-11 02:22:47 +0000 | [diff] [blame] | 44 | self._validate() |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 45 | |
| 46 | def _validate(self): |
Andrew McNamara | 7130ff5 | 2005-01-11 02:22:47 +0000 | [diff] [blame] | 47 | try: |
| 48 | _Dialect(self) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 49 | except TypeError as e: |
Andrew McNamara | 7130ff5 | 2005-01-11 02:22:47 +0000 | [diff] [blame] | 50 | # We do this for compatibility with py2.3 |
| 51 | raise Error(str(e)) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 52 | |
| 53 | class excel(Dialect): |
Skip Montanaro | f26285c | 2005-01-05 06:54:58 +0000 | [diff] [blame] | 54 | """Describe the usual properties of Excel-generated CSV files.""" |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 55 | delimiter = ',' |
| 56 | quotechar = '"' |
| 57 | doublequote = True |
| 58 | skipinitialspace = False |
| 59 | lineterminator = '\r\n' |
| 60 | quoting = QUOTE_MINIMAL |
| 61 | register_dialect("excel", excel) |
| 62 | |
| 63 | class excel_tab(excel): |
Skip Montanaro | f26285c | 2005-01-05 06:54:58 +0000 | [diff] [blame] | 64 | """Describe the usual properties of Excel-generated TAB-delimited files.""" |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 65 | delimiter = '\t' |
| 66 | register_dialect("excel-tab", excel_tab) |
| 67 | |
| 68 | |
| 69 | class DictReader: |
Skip Montanaro | dffeed3 | 2003-10-03 14:03:01 +0000 | [diff] [blame] | 70 | def __init__(self, f, fieldnames=None, restkey=None, restval=None, |
Skip Montanaro | 3f7a948 | 2003-09-06 19:52:12 +0000 | [diff] [blame] | 71 | dialect="excel", *args, **kwds): |
Skip Montanaro | af8fcfa | 2008-08-09 19:44:22 +0000 | [diff] [blame] | 72 | self._fieldnames = fieldnames # list of keys for the dict |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 73 | self.restkey = restkey # key to catch long rows |
| 74 | self.restval = restval # default value for short rows |
Skip Montanaro | 3f7a948 | 2003-09-06 19:52:12 +0000 | [diff] [blame] | 75 | self.reader = reader(f, dialect, *args, **kwds) |
Christian Heimes | 4fbc72b | 2008-03-22 00:47:35 +0000 | [diff] [blame] | 76 | self.dialect = dialect |
| 77 | self.line_num = 0 |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 78 | |
| 79 | def __iter__(self): |
| 80 | return self |
| 81 | |
Skip Montanaro | af8fcfa | 2008-08-09 19:44:22 +0000 | [diff] [blame] | 82 | @property |
| 83 | def fieldnames(self): |
| 84 | if self._fieldnames is None: |
| 85 | try: |
| 86 | self._fieldnames = next(self.reader) |
| 87 | except StopIteration: |
| 88 | pass |
| 89 | self.line_num = self.reader.line_num |
| 90 | return self._fieldnames |
| 91 | |
| 92 | @fieldnames.setter |
| 93 | def fieldnames(self, value): |
| 94 | self._fieldnames = value |
| 95 | |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 96 | def __next__(self): |
Skip Montanaro | af8fcfa | 2008-08-09 19:44:22 +0000 | [diff] [blame] | 97 | if self.line_num == 0: |
| 98 | # Used only for its side effect. |
| 99 | self.fieldnames |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 100 | row = next(self.reader) |
Christian Heimes | 4fbc72b | 2008-03-22 00:47:35 +0000 | [diff] [blame] | 101 | self.line_num = self.reader.line_num |
Skip Montanaro | dffeed3 | 2003-10-03 14:03:01 +0000 | [diff] [blame] | 102 | |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 103 | # unlike the basic reader, we prefer not to return blanks, |
| 104 | # because we will typically wind up with a dict full of None |
| 105 | # values |
| 106 | while row == []: |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 107 | row = next(self.reader) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 108 | d = dict(zip(self.fieldnames, row)) |
| 109 | lf = len(self.fieldnames) |
| 110 | lr = len(row) |
| 111 | if lf < lr: |
| 112 | d[self.restkey] = row[lf:] |
| 113 | elif lf > lr: |
| 114 | for key in self.fieldnames[lr:]: |
| 115 | d[key] = self.restval |
| 116 | return d |
| 117 | |
| 118 | |
| 119 | class DictWriter: |
| 120 | def __init__(self, f, fieldnames, restval="", extrasaction="raise", |
Skip Montanaro | 3f7a948 | 2003-09-06 19:52:12 +0000 | [diff] [blame] | 121 | dialect="excel", *args, **kwds): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 122 | self.fieldnames = fieldnames # list of keys for the dict |
| 123 | self.restval = restval # for writing short dicts |
| 124 | if extrasaction.lower() not in ("raise", "ignore"): |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 125 | raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" |
| 126 | % extrasaction) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 127 | self.extrasaction = extrasaction |
Skip Montanaro | 3f7a948 | 2003-09-06 19:52:12 +0000 | [diff] [blame] | 128 | self.writer = writer(f, dialect, *args, **kwds) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 129 | |
R. David Murray | be0698b | 2010-02-23 22:57:58 +0000 | [diff] [blame^] | 130 | def writeheader(self): |
| 131 | header = dict(zip(self.fieldnames, self.fieldnames)) |
| 132 | self.writerow(header) |
| 133 | |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 134 | def _dict_to_list(self, rowdict): |
| 135 | if self.extrasaction == "raise": |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 136 | wrong_fields = [k for k in rowdict if k not in self.fieldnames] |
| 137 | if wrong_fields: |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 138 | raise ValueError("dict contains fields not in fieldnames: " |
| 139 | + ", ".join(wrong_fields)) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 140 | return [rowdict.get(key, self.restval) for key in self.fieldnames] |
| 141 | |
| 142 | def writerow(self, rowdict): |
| 143 | return self.writer.writerow(self._dict_to_list(rowdict)) |
| 144 | |
| 145 | def writerows(self, rowdicts): |
| 146 | rows = [] |
| 147 | for rowdict in rowdicts: |
| 148 | rows.append(self._dict_to_list(rowdict)) |
| 149 | return self.writer.writerows(rows) |
| 150 | |
Raymond Hettinger | 39a5592 | 2003-06-12 03:01:55 +0000 | [diff] [blame] | 151 | # Guard Sniffer's type checking against builds that exclude complex() |
| 152 | try: |
| 153 | complex |
| 154 | except NameError: |
| 155 | complex = float |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 156 | |
| 157 | class Sniffer: |
| 158 | ''' |
| 159 | "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 160 | Returns a Dialect object. |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 161 | ''' |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 162 | def __init__(self): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 163 | # in case there is more than one possible delimiter |
| 164 | self.preferred = [',', '\t', ';', ' ', ':'] |
| 165 | |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 166 | |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 167 | def sniff(self, sample, delimiters=None): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 168 | """ |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 169 | Returns a dialect (or None) corresponding to the sample |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 170 | """ |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 171 | |
Benjamin Peterson | 4ac9ce4 | 2009-10-04 14:49:41 +0000 | [diff] [blame] | 172 | quotechar, doublequote, delimiter, skipinitialspace = \ |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 173 | self._guess_quote_and_delimiter(sample, delimiters) |
Skip Montanaro | 39b29be | 2005-12-30 05:09:48 +0000 | [diff] [blame] | 174 | if not delimiter: |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 175 | delimiter, skipinitialspace = self._guess_delimiter(sample, |
| 176 | delimiters) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 177 | |
Skip Montanaro | 39b29be | 2005-12-30 05:09:48 +0000 | [diff] [blame] | 178 | if not delimiter: |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 179 | raise Error("Could not determine delimiter") |
Skip Montanaro | 39b29be | 2005-12-30 05:09:48 +0000 | [diff] [blame] | 180 | |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 181 | class dialect(Dialect): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 182 | _name = "sniffed" |
| 183 | lineterminator = '\r\n' |
Fred Drake | 7c852f3 | 2003-04-25 14:27:00 +0000 | [diff] [blame] | 184 | quoting = QUOTE_MINIMAL |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 185 | # escapechar = '' |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 186 | |
Benjamin Peterson | 4ac9ce4 | 2009-10-04 14:49:41 +0000 | [diff] [blame] | 187 | dialect.doublequote = doublequote |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 188 | dialect.delimiter = delimiter |
| 189 | # _csv.reader won't accept a quotechar of '' |
| 190 | dialect.quotechar = quotechar or '"' |
| 191 | dialect.skipinitialspace = skipinitialspace |
| 192 | |
| 193 | return dialect |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 194 | |
| 195 | |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 196 | def _guess_quote_and_delimiter(self, data, delimiters): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 197 | """ |
| 198 | Looks for text enclosed between two identical quotes |
| 199 | (the probable quotechar) which are preceded and followed |
| 200 | by the same character (the probable delimiter). |
| 201 | For example: |
| 202 | ,'some text', |
| 203 | The quote with the most wins, same with the delimiter. |
| 204 | If there is no quotechar the delimiter can't be determined |
| 205 | this way. |
| 206 | """ |
| 207 | |
| 208 | matches = [] |
| 209 | for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?", |
| 210 | '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?", |
| 211 | '(?P<delim>>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)', # ,".*?" |
| 212 | '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) |
Fred Drake | 6f7b213 | 2003-09-02 16:01:07 +0000 | [diff] [blame] | 213 | regexp = re.compile(restr, re.DOTALL | re.MULTILINE) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 214 | matches = regexp.findall(data) |
| 215 | if matches: |
| 216 | break |
| 217 | |
| 218 | if not matches: |
Benjamin Peterson | 4ac9ce4 | 2009-10-04 14:49:41 +0000 | [diff] [blame] | 219 | # (quotechar, doublequote, delimiter, skipinitialspace) |
| 220 | return ('', False, None, 0) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 221 | quotes = {} |
| 222 | delims = {} |
| 223 | spaces = 0 |
| 224 | for m in matches: |
| 225 | n = regexp.groupindex['quote'] - 1 |
| 226 | key = m[n] |
| 227 | if key: |
| 228 | quotes[key] = quotes.get(key, 0) + 1 |
| 229 | try: |
| 230 | n = regexp.groupindex['delim'] - 1 |
| 231 | key = m[n] |
| 232 | except KeyError: |
| 233 | continue |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 234 | if key and (delimiters is None or key in delimiters): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 235 | delims[key] = delims.get(key, 0) + 1 |
| 236 | try: |
| 237 | n = regexp.groupindex['space'] - 1 |
| 238 | except KeyError: |
| 239 | continue |
| 240 | if m[n]: |
| 241 | spaces += 1 |
| 242 | |
Guido van Rossum | 89da5d7 | 2006-08-22 00:21:25 +0000 | [diff] [blame] | 243 | quotechar = max(quotes, key=quotes.get) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 244 | |
| 245 | if delims: |
Guido van Rossum | 89da5d7 | 2006-08-22 00:21:25 +0000 | [diff] [blame] | 246 | delim = max(delims, key=delims.get) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 247 | skipinitialspace = delims[delim] == spaces |
| 248 | if delim == '\n': # most likely a file with a single column |
| 249 | delim = '' |
| 250 | else: |
| 251 | # there is *no* delimiter, it's a single column of quoted data |
| 252 | delim = '' |
| 253 | skipinitialspace = 0 |
| 254 | |
Benjamin Peterson | 4ac9ce4 | 2009-10-04 14:49:41 +0000 | [diff] [blame] | 255 | # if we see an extra quote between delimiters, we've got a |
| 256 | # double quoted format |
| 257 | dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ |
| 258 | {'delim':delim, 'quote':quotechar}, re.MULTILINE) |
| 259 | |
| 260 | |
| 261 | |
| 262 | if dq_regexp.search(data): |
| 263 | doublequote = True |
| 264 | else: |
| 265 | doublequote = False |
| 266 | |
| 267 | return (quotechar, doublequote, delim, skipinitialspace) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 268 | |
| 269 | |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 270 | def _guess_delimiter(self, data, delimiters): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 271 | """ |
| 272 | The delimiter /should/ occur the same number of times on |
| 273 | each row. However, due to malformed data, it may not. We don't want |
| 274 | an all or nothing approach, so we allow for small variations in this |
| 275 | number. |
| 276 | 1) build a table of the frequency of each character on every line. |
| 277 | 2) build a table of freqencies of this frequency (meta-frequency?), |
| 278 | e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, |
| 279 | 7 times in 2 rows' |
| 280 | 3) use the mode of the meta-frequency to determine the /expected/ |
| 281 | frequency for that character |
| 282 | 4) find out how often the character actually meets that goal |
| 283 | 5) the character that best meets its goal is the delimiter |
| 284 | For performance reasons, the data is evaluated in chunks, so it can |
| 285 | try and evaluate the smallest portion of the data possible, evaluating |
| 286 | additional chunks as necessary. |
| 287 | """ |
| 288 | |
Guido van Rossum | c1f779c | 2007-07-03 08:25:58 +0000 | [diff] [blame] | 289 | data = list(filter(None, data.split('\n'))) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 290 | |
| 291 | ascii = [chr(c) for c in range(127)] # 7-bit ASCII |
| 292 | |
| 293 | # build frequency tables |
| 294 | chunkLength = min(10, len(data)) |
| 295 | iteration = 0 |
| 296 | charFrequency = {} |
| 297 | modes = {} |
| 298 | delims = {} |
| 299 | start, end = 0, min(chunkLength, len(data)) |
| 300 | while start < len(data): |
| 301 | iteration += 1 |
| 302 | for line in data[start:end]: |
| 303 | for char in ascii: |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 304 | metaFrequency = charFrequency.get(char, {}) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 305 | # must count even if frequency is 0 |
Skip Montanaro | 91bb70c | 2005-12-28 15:37:25 +0000 | [diff] [blame] | 306 | freq = line.count(char) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 307 | # value is the mode |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 308 | metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 |
| 309 | charFrequency[char] = metaFrequency |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 310 | |
| 311 | for char in charFrequency.keys(): |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 312 | items = list(charFrequency[char].items()) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 313 | if len(items) == 1 and items[0][0] == 0: |
| 314 | continue |
| 315 | # get the mode of the frequencies |
| 316 | if len(items) > 1: |
Guido van Rossum | 89da5d7 | 2006-08-22 00:21:25 +0000 | [diff] [blame] | 317 | modes[char] = max(items, key=lambda x: x[1]) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 318 | # adjust the mode - subtract the sum of all |
| 319 | # other frequencies |
| 320 | items.remove(modes[char]) |
| 321 | modes[char] = (modes[char][0], modes[char][1] |
Guido van Rossum | 89da5d7 | 2006-08-22 00:21:25 +0000 | [diff] [blame] | 322 | - sum(item[1] for item in items)) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 323 | else: |
| 324 | modes[char] = items[0] |
| 325 | |
| 326 | # build a list of possible delimiters |
| 327 | modeList = modes.items() |
| 328 | total = float(chunkLength * iteration) |
| 329 | # (rows of consistent data) / (number of rows) = 100% |
| 330 | consistency = 1.0 |
| 331 | # minimum consistency threshold |
| 332 | threshold = 0.9 |
| 333 | while len(delims) == 0 and consistency >= threshold: |
| 334 | for k, v in modeList: |
| 335 | if v[0] > 0 and v[1] > 0: |
Skip Montanaro | 7789237 | 2003-05-19 15:33:36 +0000 | [diff] [blame] | 336 | if ((v[1]/total) >= consistency and |
| 337 | (delimiters is None or k in delimiters)): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 338 | delims[k] = v |
| 339 | consistency -= 0.01 |
| 340 | |
| 341 | if len(delims) == 1: |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 342 | delim = list(delims.keys())[0] |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 343 | skipinitialspace = (data[0].count(delim) == |
| 344 | data[0].count("%c " % delim)) |
| 345 | return (delim, skipinitialspace) |
| 346 | |
| 347 | # analyze another chunkLength lines |
| 348 | start = end |
| 349 | end += chunkLength |
| 350 | |
| 351 | if not delims: |
| 352 | return ('', 0) |
| 353 | |
| 354 | # if there's more than one, fall back to a 'preferred' list |
| 355 | if len(delims) > 1: |
| 356 | for d in self.preferred: |
| 357 | if d in delims.keys(): |
| 358 | skipinitialspace = (data[0].count(d) == |
| 359 | data[0].count("%c " % d)) |
| 360 | return (d, skipinitialspace) |
| 361 | |
Skip Montanaro | 39b29be | 2005-12-30 05:09:48 +0000 | [diff] [blame] | 362 | # nothing else indicates a preference, pick the character that |
| 363 | # dominates(?) |
| 364 | items = [(v,k) for (k,v) in delims.items()] |
| 365 | items.sort() |
| 366 | delim = items[-1][1] |
| 367 | |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 368 | skipinitialspace = (data[0].count(delim) == |
| 369 | data[0].count("%c " % delim)) |
| 370 | return (delim, skipinitialspace) |
| 371 | |
| 372 | |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 373 | def has_header(self, sample): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 374 | # Creates a dictionary of types of data in each column. If any |
| 375 | # column is of a single type (say, integers), *except* for the first |
| 376 | # row, then the first row is presumed to be labels. If the type |
| 377 | # can't be determined, it is assumed to be a string in which case |
| 378 | # the length of the string is the determining factor: if all of the |
| 379 | # rows except for the first are the same length, it's a header. |
| 380 | # Finally, a 'vote' is taken at the end for each column, adding or |
| 381 | # subtracting from the likelihood of the first row being a header. |
| 382 | |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 383 | rdr = reader(StringIO(sample), self.sniff(sample)) |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 384 | |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 385 | header = next(rdr) # assume first row is header |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 386 | |
| 387 | columns = len(header) |
| 388 | columnTypes = {} |
| 389 | for i in range(columns): columnTypes[i] = None |
| 390 | |
| 391 | checked = 0 |
Skip Montanaro | 1448d47 | 2003-04-25 14:47:16 +0000 | [diff] [blame] | 392 | for row in rdr: |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 393 | # arbitrary number of rows to check, to keep it sane |
| 394 | if checked > 20: |
| 395 | break |
| 396 | checked += 1 |
| 397 | |
| 398 | if len(row) != columns: |
| 399 | continue # skip rows that have irregular number of columns |
| 400 | |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 401 | for col in list(columnTypes.keys()): |
Raymond Hettinger | 39a5592 | 2003-06-12 03:01:55 +0000 | [diff] [blame] | 402 | |
Amaury Forgeot d'Arc | a461873 | 2008-04-24 18:26:53 +0000 | [diff] [blame] | 403 | for thisType in [int, float, complex]: |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 404 | try: |
Raymond Hettinger | 39a5592 | 2003-06-12 03:01:55 +0000 | [diff] [blame] | 405 | thisType(row[col]) |
| 406 | break |
Raymond Hettinger | abe14e6 | 2003-06-12 03:59:17 +0000 | [diff] [blame] | 407 | except (ValueError, OverflowError): |
Raymond Hettinger | 39a5592 | 2003-06-12 03:01:55 +0000 | [diff] [blame] | 408 | pass |
| 409 | else: |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 410 | # fallback to length of string |
| 411 | thisType = len(row[col]) |
| 412 | |
| 413 | if thisType != columnTypes[col]: |
| 414 | if columnTypes[col] is None: # add new column type |
| 415 | columnTypes[col] = thisType |
| 416 | else: |
| 417 | # type is inconsistent, remove column from |
| 418 | # consideration |
| 419 | del columnTypes[col] |
| 420 | |
| 421 | # finally, compare results against first row and "vote" |
| 422 | # on whether it's a header |
| 423 | hasHeader = 0 |
| 424 | for col, colType in columnTypes.items(): |
| 425 | if type(colType) == type(0): # it's a length |
| 426 | if len(header[col]) != colType: |
| 427 | hasHeader += 1 |
| 428 | else: |
| 429 | hasHeader -= 1 |
| 430 | else: # attempt typecast |
| 431 | try: |
Raymond Hettinger | 39a5592 | 2003-06-12 03:01:55 +0000 | [diff] [blame] | 432 | colType(header[col]) |
Raymond Hettinger | f31cb0c | 2003-06-12 04:05:00 +0000 | [diff] [blame] | 433 | except (ValueError, TypeError): |
Skip Montanaro | 04ae705 | 2003-04-24 20:21:31 +0000 | [diff] [blame] | 434 | hasHeader += 1 |
| 435 | else: |
| 436 | hasHeader -= 1 |
| 437 | |
| 438 | return hasHeader > 0 |