Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Argument Clinic |
| 4 | # Copyright 2012-2013 by Larry Hastings. |
| 5 | # Licensed to the PSF under a contributor agreement. |
| 6 | # |
| 7 | |
| 8 | import abc |
| 9 | import ast |
| 10 | import atexit |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 11 | import collections |
| 12 | import contextlib |
| 13 | import functools |
| 14 | import hashlib |
| 15 | import inspect |
| 16 | import io |
| 17 | import itertools |
| 18 | import os |
| 19 | import re |
| 20 | import shlex |
| 21 | import sys |
| 22 | import tempfile |
| 23 | import textwrap |
| 24 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 25 | # TODO: |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 26 | # |
| 27 | # soon: |
| 28 | # |
| 29 | # * allow mixing any two of {positional-only, positional-or-keyword, |
| 30 | # keyword-only} |
| 31 | # * dict constructor uses positional-only and keyword-only |
| 32 | # * max and min use positional only with an optional group |
| 33 | # and keyword-only |
| 34 | # |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 35 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 36 | version = '1' |
| 37 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 38 | _empty = inspect._empty |
| 39 | _void = inspect._void |
| 40 | |
| 41 | |
| 42 | class Unspecified: |
| 43 | def __repr__(self): |
| 44 | return '<Unspecified>' |
| 45 | |
| 46 | unspecified = Unspecified() |
| 47 | |
| 48 | |
| 49 | class Null: |
| 50 | def __repr__(self): |
| 51 | return '<Null>' |
| 52 | |
| 53 | NULL = Null() |
| 54 | |
| 55 | |
| 56 | def _text_accumulator(): |
| 57 | text = [] |
| 58 | def output(): |
| 59 | s = ''.join(text) |
| 60 | text.clear() |
| 61 | return s |
| 62 | return text, text.append, output |
| 63 | |
| 64 | |
| 65 | def text_accumulator(): |
| 66 | """ |
| 67 | Creates a simple text accumulator / joiner. |
| 68 | |
| 69 | Returns a pair of callables: |
| 70 | append, output |
| 71 | "append" appends a string to the accumulator. |
| 72 | "output" returns the contents of the accumulator |
| 73 | joined together (''.join(accumulator)) and |
| 74 | empties the accumulator. |
| 75 | """ |
| 76 | text, append, output = _text_accumulator() |
| 77 | return append, output |
| 78 | |
| 79 | |
| 80 | def fail(*args, filename=None, line_number=None): |
| 81 | joined = " ".join([str(a) for a in args]) |
| 82 | add, output = text_accumulator() |
| 83 | add("Error") |
| 84 | if clinic: |
| 85 | if filename is None: |
| 86 | filename = clinic.filename |
| 87 | if clinic.block_parser and (line_number is None): |
| 88 | line_number = clinic.block_parser.line_number |
| 89 | if filename is not None: |
| 90 | add(' in file "' + filename + '"') |
| 91 | if line_number is not None: |
| 92 | add(" on line " + str(line_number)) |
| 93 | add(':\n') |
| 94 | add(joined) |
| 95 | print(output()) |
| 96 | sys.exit(-1) |
| 97 | |
| 98 | |
| 99 | |
| 100 | def quoted_for_c_string(s): |
| 101 | for old, new in ( |
| 102 | ('"', '\\"'), |
| 103 | ("'", "\\'"), |
| 104 | ): |
| 105 | s = s.replace(old, new) |
| 106 | return s |
| 107 | |
Larry Hastings | dfcd467 | 2013-10-27 02:49:39 -0700 | [diff] [blame] | 108 | is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match |
| 109 | |
| 110 | def is_legal_py_identifier(s): |
| 111 | return all(is_legal_c_identifier(field) for field in s.split('.')) |
| 112 | |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 113 | # added "module", "self", "cls", and "null" just to be safe |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 114 | # (clinic will generate variables with these names) |
| 115 | c_keywords = set(""" |
| 116 | asm auto break case char cls const continue default do double |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 117 | else enum extern float for goto if inline int long module null |
| 118 | register return self short signed sizeof static struct switch |
| 119 | typedef typeof union unsigned void volatile while |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 120 | """.strip().split()) |
| 121 | |
Larry Hastings | dfcd467 | 2013-10-27 02:49:39 -0700 | [diff] [blame] | 122 | def ensure_legal_c_identifier(s): |
| 123 | # for now, just complain if what we're given isn't legal |
| 124 | if not is_legal_c_identifier(s): |
| 125 | fail("Illegal C identifier: {}".format(s)) |
| 126 | # but if we picked a C keyword, pick something else |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 127 | if s in c_keywords: |
| 128 | return s + "_value" |
| 129 | return s |
| 130 | |
| 131 | def rstrip_lines(s): |
| 132 | text, add, output = _text_accumulator() |
| 133 | for line in s.split('\n'): |
| 134 | add(line.rstrip()) |
| 135 | add('\n') |
| 136 | text.pop() |
| 137 | return output() |
| 138 | |
| 139 | def linear_format(s, **kwargs): |
| 140 | """ |
| 141 | Perform str.format-like substitution, except: |
| 142 | * The strings substituted must be on lines by |
| 143 | themselves. (This line is the "source line".) |
| 144 | * If the substitution text is empty, the source line |
| 145 | is removed in the output. |
| 146 | * If the substitution text is not empty: |
| 147 | * Each line of the substituted text is indented |
| 148 | by the indent of the source line. |
| 149 | * A newline will be added to the end. |
| 150 | """ |
| 151 | |
| 152 | add, output = text_accumulator() |
| 153 | for line in s.split('\n'): |
| 154 | indent, curly, trailing = line.partition('{') |
| 155 | if not curly: |
| 156 | add(line) |
| 157 | add('\n') |
| 158 | continue |
| 159 | |
| 160 | name, curl, trailing = trailing.partition('}') |
| 161 | if not curly or name not in kwargs: |
| 162 | add(line) |
| 163 | add('\n') |
| 164 | continue |
| 165 | |
| 166 | if trailing: |
| 167 | fail("Text found after {" + name + "} block marker! It must be on a line by itself.") |
| 168 | if indent.strip(): |
| 169 | fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") |
| 170 | |
| 171 | value = kwargs[name] |
| 172 | if not value: |
| 173 | continue |
| 174 | |
| 175 | value = textwrap.indent(rstrip_lines(value), indent) |
| 176 | add(value) |
| 177 | add('\n') |
| 178 | |
| 179 | return output()[:-1] |
| 180 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 181 | def version_splitter(s): |
| 182 | """Splits a version string into a tuple of integers. |
| 183 | |
| 184 | The following ASCII characters are allowed, and employ |
| 185 | the following conversions: |
| 186 | a -> -3 |
| 187 | b -> -2 |
| 188 | c -> -1 |
| 189 | (This permits Python-style version strings such as "1.4b3".) |
| 190 | """ |
| 191 | version = [] |
| 192 | accumulator = [] |
| 193 | def flush(): |
| 194 | if not accumulator: |
| 195 | raise ValueError('Malformed version string: ' + repr(s)) |
| 196 | version.append(int(''.join(accumulator))) |
| 197 | accumulator.clear() |
| 198 | |
| 199 | for c in s: |
| 200 | if c.isdigit(): |
| 201 | accumulator.append(c) |
| 202 | elif c == '.': |
| 203 | flush() |
| 204 | elif c in 'abc': |
| 205 | flush() |
| 206 | version.append('abc'.index(c) - 3) |
| 207 | else: |
| 208 | raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) |
| 209 | flush() |
| 210 | return tuple(version) |
| 211 | |
| 212 | def version_comparitor(version1, version2): |
| 213 | iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) |
| 214 | for i, (a, b) in enumerate(iterator): |
| 215 | if a < b: |
| 216 | return -1 |
| 217 | if a > b: |
| 218 | return 1 |
| 219 | return 0 |
| 220 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 221 | |
| 222 | class CRenderData: |
| 223 | def __init__(self): |
| 224 | |
| 225 | # The C statements to declare variables. |
| 226 | # Should be full lines with \n eol characters. |
| 227 | self.declarations = [] |
| 228 | |
| 229 | # The C statements required to initialize the variables before the parse call. |
| 230 | # Should be full lines with \n eol characters. |
| 231 | self.initializers = [] |
| 232 | |
| 233 | # The entries for the "keywords" array for PyArg_ParseTuple. |
| 234 | # Should be individual strings representing the names. |
| 235 | self.keywords = [] |
| 236 | |
| 237 | # The "format units" for PyArg_ParseTuple. |
| 238 | # Should be individual strings that will get |
| 239 | self.format_units = [] |
| 240 | |
| 241 | # The varargs arguments for PyArg_ParseTuple. |
| 242 | self.parse_arguments = [] |
| 243 | |
| 244 | # The parameter declarations for the impl function. |
| 245 | self.impl_parameters = [] |
| 246 | |
| 247 | # The arguments to the impl function at the time it's called. |
| 248 | self.impl_arguments = [] |
| 249 | |
| 250 | # For return converters: the name of the variable that |
| 251 | # should receive the value returned by the impl. |
| 252 | self.return_value = "return_value" |
| 253 | |
| 254 | # For return converters: the code to convert the return |
| 255 | # value from the parse function. This is also where |
| 256 | # you should check the _return_value for errors, and |
| 257 | # "goto exit" if there are any. |
| 258 | self.return_conversion = [] |
| 259 | |
| 260 | # The C statements required to clean up after the impl call. |
| 261 | self.cleanup = [] |
| 262 | |
| 263 | |
| 264 | class Language(metaclass=abc.ABCMeta): |
| 265 | |
| 266 | start_line = "" |
| 267 | body_prefix = "" |
| 268 | stop_line = "" |
| 269 | checksum_line = "" |
| 270 | |
| 271 | @abc.abstractmethod |
| 272 | def render(self, block): |
| 273 | pass |
| 274 | |
| 275 | def validate(self): |
| 276 | def assert_only_one(field, token='dsl_name'): |
| 277 | line = getattr(self, field) |
| 278 | token = '{' + token + '}' |
| 279 | if len(line.split(token)) != 2: |
| 280 | fail(self.__class__.__name__ + " " + field + " must contain " + token + " exactly once!") |
| 281 | assert_only_one('start_line') |
| 282 | assert_only_one('stop_line') |
| 283 | assert_only_one('checksum_line') |
| 284 | assert_only_one('checksum_line', 'checksum') |
| 285 | |
| 286 | if len(self.body_prefix.split('{dsl_name}')) >= 3: |
| 287 | fail(self.__class__.__name__ + " body_prefix may contain " + token + " once at most!") |
| 288 | |
| 289 | |
| 290 | |
| 291 | class PythonLanguage(Language): |
| 292 | |
| 293 | language = 'Python' |
Larry Hastings | 61272b7 | 2014-01-07 12:41:53 -0800 | [diff] [blame] | 294 | start_line = "#/*[{dsl_name} input]" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 295 | body_prefix = "#" |
Larry Hastings | 61272b7 | 2014-01-07 12:41:53 -0800 | [diff] [blame] | 296 | stop_line = "#[{dsl_name} start generated code]*/" |
| 297 | checksum_line = "#/*[{dsl_name} end generated code: checksum={checksum}]*/" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 298 | |
| 299 | |
| 300 | def permute_left_option_groups(l): |
| 301 | """ |
| 302 | Given [1, 2, 3], should yield: |
| 303 | () |
| 304 | (3,) |
| 305 | (2, 3) |
| 306 | (1, 2, 3) |
| 307 | """ |
| 308 | yield tuple() |
| 309 | accumulator = [] |
| 310 | for group in reversed(l): |
| 311 | accumulator = list(group) + accumulator |
| 312 | yield tuple(accumulator) |
| 313 | |
| 314 | |
| 315 | def permute_right_option_groups(l): |
| 316 | """ |
| 317 | Given [1, 2, 3], should yield: |
| 318 | () |
| 319 | (1,) |
| 320 | (1, 2) |
| 321 | (1, 2, 3) |
| 322 | """ |
| 323 | yield tuple() |
| 324 | accumulator = [] |
| 325 | for group in l: |
| 326 | accumulator.extend(group) |
| 327 | yield tuple(accumulator) |
| 328 | |
| 329 | |
| 330 | def permute_optional_groups(left, required, right): |
| 331 | """ |
| 332 | Generator function that computes the set of acceptable |
| 333 | argument lists for the provided iterables of |
| 334 | argument groups. (Actually it generates a tuple of tuples.) |
| 335 | |
| 336 | Algorithm: prefer left options over right options. |
| 337 | |
| 338 | If required is empty, left must also be empty. |
| 339 | """ |
| 340 | required = tuple(required) |
| 341 | result = [] |
| 342 | |
| 343 | if not required: |
| 344 | assert not left |
| 345 | |
| 346 | accumulator = [] |
| 347 | counts = set() |
| 348 | for r in permute_right_option_groups(right): |
| 349 | for l in permute_left_option_groups(left): |
| 350 | t = l + required + r |
| 351 | if len(t) in counts: |
| 352 | continue |
| 353 | counts.add(len(t)) |
| 354 | accumulator.append(t) |
| 355 | |
| 356 | accumulator.sort(key=len) |
| 357 | return tuple(accumulator) |
| 358 | |
| 359 | |
| 360 | class CLanguage(Language): |
| 361 | |
Larry Hastings | 61272b7 | 2014-01-07 12:41:53 -0800 | [diff] [blame] | 362 | body_prefix = "#" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 363 | language = 'C' |
Larry Hastings | 61272b7 | 2014-01-07 12:41:53 -0800 | [diff] [blame] | 364 | start_line = "/*[{dsl_name} input]" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 365 | body_prefix = "" |
Larry Hastings | 61272b7 | 2014-01-07 12:41:53 -0800 | [diff] [blame] | 366 | stop_line = "[{dsl_name} start generated code]*/" |
| 367 | checksum_line = "/*[{dsl_name} end generated code: checksum={checksum}]*/" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 368 | |
| 369 | def render(self, signatures): |
| 370 | function = None |
| 371 | for o in signatures: |
| 372 | if isinstance(o, Function): |
| 373 | if function: |
| 374 | fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) |
| 375 | function = o |
| 376 | return self.render_function(function) |
| 377 | |
| 378 | def docstring_for_c_string(self, f): |
| 379 | text, add, output = _text_accumulator() |
| 380 | # turn docstring into a properly quoted C string |
| 381 | for line in f.docstring.split('\n'): |
| 382 | add('"') |
| 383 | add(quoted_for_c_string(line)) |
| 384 | add('\\n"\n') |
| 385 | |
| 386 | text.pop() |
| 387 | add('"') |
| 388 | return ''.join(text) |
| 389 | |
| 390 | impl_prototype_template = "{c_basename}_impl({impl_parameters})" |
| 391 | |
| 392 | @staticmethod |
| 393 | def template_base(*args): |
| 394 | flags = '|'.join(f for f in args if f) |
| 395 | return """ |
| 396 | PyDoc_STRVAR({c_basename}__doc__, |
| 397 | {docstring}); |
| 398 | |
| 399 | #define {methoddef_name} \\ |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 400 | {{"{name}", (PyCFunction){c_basename}, {methoddef_flags}, {c_basename}__doc__}}, |
| 401 | """.replace('{methoddef_flags}', flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 402 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 403 | def meth_noargs_template(self, methoddef_flags=""): |
| 404 | return self.template_base("METH_NOARGS", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 405 | static {impl_return_type} |
| 406 | {impl_prototype}; |
| 407 | |
| 408 | static PyObject * |
Larry Hastings | 3cceb38 | 2014-01-04 11:09:09 -0800 | [diff] [blame] | 409 | {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 410 | {{ |
| 411 | PyObject *return_value = NULL; |
| 412 | {declarations} |
| 413 | {initializers} |
| 414 | |
| 415 | {return_value} = {c_basename}_impl({impl_arguments}); |
| 416 | {return_conversion} |
| 417 | |
| 418 | {exit_label} |
| 419 | {cleanup} |
| 420 | return return_value; |
| 421 | }} |
| 422 | |
| 423 | static {impl_return_type} |
| 424 | {impl_prototype} |
| 425 | """ |
| 426 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 427 | def meth_o_template(self, methoddef_flags=""): |
| 428 | return self.template_base("METH_O", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 429 | static PyObject * |
| 430 | {c_basename}({impl_parameters}) |
| 431 | """ |
| 432 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 433 | def meth_o_return_converter_template(self, methoddef_flags=""): |
| 434 | return self.template_base("METH_O", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 435 | static {impl_return_type} |
| 436 | {impl_prototype}; |
| 437 | |
| 438 | static PyObject * |
| 439 | {c_basename}({impl_parameters}) |
| 440 | {{ |
| 441 | PyObject *return_value = NULL; |
| 442 | {declarations} |
| 443 | {initializers} |
| 444 | _return_value = {c_basename}_impl({impl_arguments}); |
| 445 | {return_conversion} |
| 446 | |
| 447 | {exit_label} |
| 448 | {cleanup} |
| 449 | return return_value; |
| 450 | }} |
| 451 | |
| 452 | static {impl_return_type} |
| 453 | {impl_prototype} |
| 454 | """ |
| 455 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 456 | def option_group_template(self, methoddef_flags=""): |
| 457 | return self.template_base("METH_VARARGS", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 458 | static {impl_return_type} |
| 459 | {impl_prototype}; |
| 460 | |
| 461 | static PyObject * |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 462 | {c_basename}({self_type}{self_name}, PyObject *args) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 463 | {{ |
| 464 | PyObject *return_value = NULL; |
| 465 | {declarations} |
| 466 | {initializers} |
| 467 | |
| 468 | {option_group_parsing} |
| 469 | {return_value} = {c_basename}_impl({impl_arguments}); |
| 470 | {return_conversion} |
| 471 | |
| 472 | {exit_label} |
| 473 | {cleanup} |
| 474 | return return_value; |
| 475 | }} |
| 476 | |
| 477 | static {impl_return_type} |
| 478 | {impl_prototype} |
| 479 | """ |
| 480 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 481 | def keywords_template(self, methoddef_flags=""): |
| 482 | return self.template_base("METH_VARARGS|METH_KEYWORDS", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 483 | static {impl_return_type} |
| 484 | {impl_prototype}; |
| 485 | |
| 486 | static PyObject * |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 487 | {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 488 | {{ |
| 489 | PyObject *return_value = NULL; |
| 490 | static char *_keywords[] = {{{keywords}, NULL}}; |
| 491 | {declarations} |
| 492 | {initializers} |
| 493 | |
| 494 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, |
| 495 | "{format_units}:{name}", _keywords, |
| 496 | {parse_arguments})) |
| 497 | goto exit; |
| 498 | {return_value} = {c_basename}_impl({impl_arguments}); |
| 499 | {return_conversion} |
| 500 | |
| 501 | {exit_label} |
| 502 | {cleanup} |
| 503 | return return_value; |
| 504 | }} |
| 505 | |
| 506 | static {impl_return_type} |
| 507 | {impl_prototype} |
| 508 | """ |
| 509 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 510 | def positional_only_template(self, methoddef_flags=""): |
| 511 | return self.template_base("METH_VARARGS", methoddef_flags) + """ |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 512 | static {impl_return_type} |
| 513 | {impl_prototype}; |
| 514 | |
| 515 | static PyObject * |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 516 | {c_basename}({self_type}{self_name}, PyObject *args) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 517 | {{ |
| 518 | PyObject *return_value = NULL; |
| 519 | {declarations} |
| 520 | {initializers} |
| 521 | |
| 522 | if (!PyArg_ParseTuple(args, |
| 523 | "{format_units}:{name}", |
| 524 | {parse_arguments})) |
| 525 | goto exit; |
| 526 | {return_value} = {c_basename}_impl({impl_arguments}); |
| 527 | {return_conversion} |
| 528 | |
| 529 | {exit_label} |
| 530 | {cleanup} |
| 531 | return return_value; |
| 532 | }} |
| 533 | |
| 534 | static {impl_return_type} |
| 535 | {impl_prototype} |
| 536 | """ |
| 537 | |
| 538 | @staticmethod |
| 539 | def group_to_variable_name(group): |
| 540 | adjective = "left_" if group < 0 else "right_" |
| 541 | return "group_" + adjective + str(abs(group)) |
| 542 | |
| 543 | def render_option_group_parsing(self, f, template_dict): |
| 544 | # positional only, grouped, optional arguments! |
| 545 | # can be optional on the left or right. |
| 546 | # here's an example: |
| 547 | # |
| 548 | # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] |
| 549 | # |
| 550 | # Here group D are required, and all other groups are optional. |
| 551 | # (Group D's "group" is actually None.) |
| 552 | # We can figure out which sets of arguments we have based on |
| 553 | # how many arguments are in the tuple. |
| 554 | # |
| 555 | # Note that you need to count up on both sides. For example, |
| 556 | # you could have groups C+D, or C+D+E, or C+D+E+F. |
| 557 | # |
| 558 | # What if the number of arguments leads us to an ambiguous result? |
| 559 | # Clinic prefers groups on the left. So in the above example, |
| 560 | # five arguments would map to B+C, not C+D. |
| 561 | |
| 562 | add, output = text_accumulator() |
| 563 | parameters = list(f.parameters.values()) |
| 564 | |
| 565 | groups = [] |
| 566 | group = None |
| 567 | left = [] |
| 568 | right = [] |
| 569 | required = [] |
| 570 | last = unspecified |
| 571 | |
| 572 | for p in parameters: |
| 573 | group_id = p.group |
| 574 | if group_id != last: |
| 575 | last = group_id |
| 576 | group = [] |
| 577 | if group_id < 0: |
| 578 | left.append(group) |
| 579 | elif group_id == 0: |
| 580 | group = required |
| 581 | else: |
| 582 | right.append(group) |
| 583 | group.append(p) |
| 584 | |
| 585 | count_min = sys.maxsize |
| 586 | count_max = -1 |
| 587 | |
| 588 | add("switch (PyTuple_Size(args)) {{\n") |
| 589 | for subset in permute_optional_groups(left, required, right): |
| 590 | count = len(subset) |
| 591 | count_min = min(count_min, count) |
| 592 | count_max = max(count_max, count) |
| 593 | |
Larry Hastings | 583baa8 | 2014-01-12 08:49:30 -0800 | [diff] [blame^] | 594 | if count == 0: |
| 595 | add(""" case 0: |
| 596 | break; |
| 597 | """) |
| 598 | continue |
| 599 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 600 | group_ids = {p.group for p in subset} # eliminate duplicates |
| 601 | d = {} |
| 602 | d['count'] = count |
| 603 | d['name'] = f.name |
| 604 | d['groups'] = sorted(group_ids) |
| 605 | d['format_units'] = "".join(p.converter.format_unit for p in subset) |
| 606 | |
| 607 | parse_arguments = [] |
| 608 | for p in subset: |
| 609 | p.converter.parse_argument(parse_arguments) |
| 610 | d['parse_arguments'] = ", ".join(parse_arguments) |
| 611 | |
| 612 | group_ids.discard(0) |
| 613 | lines = [self.group_to_variable_name(g) + " = 1;" for g in group_ids] |
| 614 | lines = "\n".join(lines) |
| 615 | |
| 616 | s = """ |
| 617 | case {count}: |
| 618 | if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) |
| 619 | return NULL; |
| 620 | {group_booleans} |
| 621 | break; |
| 622 | """[1:] |
| 623 | s = linear_format(s, group_booleans=lines) |
| 624 | s = s.format_map(d) |
| 625 | add(s) |
| 626 | |
| 627 | add(" default:\n") |
| 628 | s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' |
| 629 | add(s.format(f.full_name, count_min, count_max)) |
| 630 | add(' return NULL;\n') |
| 631 | add("}}") |
| 632 | template_dict['option_group_parsing'] = output() |
| 633 | |
| 634 | def render_function(self, f): |
| 635 | if not f: |
| 636 | return "" |
| 637 | |
| 638 | add, output = text_accumulator() |
| 639 | data = CRenderData() |
| 640 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 641 | parameters = list(f.parameters.values()) |
| 642 | converters = [p.converter for p in parameters] |
| 643 | |
| 644 | template_dict = {} |
| 645 | |
| 646 | full_name = f.full_name |
| 647 | template_dict['full_name'] = full_name |
| 648 | |
| 649 | name = full_name.rpartition('.')[2] |
| 650 | template_dict['name'] = name |
| 651 | |
| 652 | c_basename = f.c_basename or full_name.replace(".", "_") |
| 653 | template_dict['c_basename'] = c_basename |
| 654 | |
| 655 | methoddef_name = "{}_METHODDEF".format(c_basename.upper()) |
| 656 | template_dict['methoddef_name'] = methoddef_name |
| 657 | |
| 658 | template_dict['docstring'] = self.docstring_for_c_string(f) |
| 659 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 660 | positional = has_option_groups = False |
| 661 | |
| 662 | if parameters: |
| 663 | last_group = 0 |
| 664 | |
| 665 | for p in parameters: |
| 666 | c = p.converter |
| 667 | |
| 668 | # insert group variable |
| 669 | group = p.group |
| 670 | if last_group != group: |
| 671 | last_group = group |
| 672 | if group: |
| 673 | group_name = self.group_to_variable_name(group) |
| 674 | data.impl_arguments.append(group_name) |
| 675 | data.declarations.append("int " + group_name + " = 0;") |
| 676 | data.impl_parameters.append("int " + group_name) |
| 677 | has_option_groups = True |
| 678 | c.render(p, data) |
| 679 | |
| 680 | positional = parameters[-1].kind == inspect.Parameter.POSITIONAL_ONLY |
| 681 | if has_option_groups: |
| 682 | assert positional |
| 683 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 684 | # now insert our "self" (or whatever) parameters |
| 685 | # (we deliberately don't call render on self converters) |
| 686 | stock_self = self_converter('self', f) |
| 687 | template_dict['self_name'] = stock_self.name |
| 688 | template_dict['self_type'] = stock_self.type |
| 689 | data.impl_parameters.insert(0, f.self_converter.type + ("" if f.self_converter.type.endswith('*') else " ") + f.self_converter.name) |
| 690 | if f.self_converter.type != stock_self.type: |
| 691 | self_cast = '(' + f.self_converter.type + ')' |
| 692 | else: |
| 693 | self_cast = '' |
| 694 | data.impl_arguments.insert(0, self_cast + stock_self.name) |
| 695 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 696 | f.return_converter.render(f, data) |
| 697 | template_dict['impl_return_type'] = f.return_converter.type |
| 698 | |
| 699 | template_dict['declarations'] = "\n".join(data.declarations) |
| 700 | template_dict['initializers'] = "\n\n".join(data.initializers) |
| 701 | template_dict['keywords'] = '"' + '", "'.join(data.keywords) + '"' |
| 702 | template_dict['format_units'] = ''.join(data.format_units) |
| 703 | template_dict['parse_arguments'] = ', '.join(data.parse_arguments) |
| 704 | template_dict['impl_parameters'] = ", ".join(data.impl_parameters) |
| 705 | template_dict['impl_arguments'] = ", ".join(data.impl_arguments) |
| 706 | template_dict['return_conversion'] = "".join(data.return_conversion).rstrip() |
| 707 | template_dict['cleanup'] = "".join(data.cleanup) |
| 708 | template_dict['return_value'] = data.return_value |
| 709 | |
| 710 | template_dict['impl_prototype'] = self.impl_prototype_template.format_map(template_dict) |
| 711 | |
| 712 | default_return_converter = (not f.return_converter or |
| 713 | f.return_converter.type == 'PyObject *') |
| 714 | |
| 715 | if not parameters: |
Larry Hastings | 3cceb38 | 2014-01-04 11:09:09 -0800 | [diff] [blame] | 716 | template = self.meth_noargs_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 717 | elif (len(parameters) == 1 and |
| 718 | parameters[0].kind == inspect.Parameter.POSITIONAL_ONLY and |
| 719 | not converters[0].is_optional() and |
| 720 | isinstance(converters[0], object_converter) and |
| 721 | converters[0].format_unit == 'O'): |
| 722 | if default_return_converter: |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 723 | template = self.meth_o_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 724 | else: |
| 725 | # HACK |
| 726 | # we're using "impl_parameters" for the |
| 727 | # non-impl function, because that works |
| 728 | # better for METH_O. but that means we |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 729 | # must supress actually declaring the |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 730 | # impl's parameters as variables in the |
| 731 | # non-impl. but since it's METH_O, we |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 732 | # only have one anyway, so |
| 733 | # we don't have any problem finding it. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 734 | declarations_copy = list(data.declarations) |
| 735 | before, pyobject, after = declarations_copy[0].partition('PyObject *') |
| 736 | assert not before, "hack failed, see comment" |
| 737 | assert pyobject, "hack failed, see comment" |
| 738 | assert after and after[0].isalpha(), "hack failed, see comment" |
| 739 | del declarations_copy[0] |
| 740 | template_dict['declarations'] = "\n".join(declarations_copy) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 741 | template = self.meth_o_return_converter_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 742 | elif has_option_groups: |
| 743 | self.render_option_group_parsing(f, template_dict) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 744 | template = self.option_group_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 745 | template = linear_format(template, |
| 746 | option_group_parsing=template_dict['option_group_parsing']) |
| 747 | elif positional: |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 748 | template = self.positional_only_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 749 | else: |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 750 | template = self.keywords_template(f.methoddef_flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 751 | |
| 752 | template = linear_format(template, |
| 753 | declarations=template_dict['declarations'], |
| 754 | return_conversion=template_dict['return_conversion'], |
| 755 | initializers=template_dict['initializers'], |
| 756 | cleanup=template_dict['cleanup'], |
| 757 | ) |
| 758 | |
| 759 | # Only generate the "exit:" label |
| 760 | # if we have any gotos |
| 761 | need_exit_label = "goto exit;" in template |
| 762 | template = linear_format(template, |
| 763 | exit_label="exit:" if need_exit_label else '' |
| 764 | ) |
| 765 | |
| 766 | return template.format_map(template_dict) |
| 767 | |
| 768 | |
| 769 | @contextlib.contextmanager |
| 770 | def OverrideStdioWith(stdout): |
| 771 | saved_stdout = sys.stdout |
| 772 | sys.stdout = stdout |
| 773 | try: |
| 774 | yield |
| 775 | finally: |
| 776 | assert sys.stdout is stdout |
| 777 | sys.stdout = saved_stdout |
| 778 | |
| 779 | |
| 780 | def create_regex(before, after): |
| 781 | """Create an re object for matching marker lines.""" |
| 782 | pattern = r'^{}(\w+){}$' |
| 783 | return re.compile(pattern.format(re.escape(before), re.escape(after))) |
| 784 | |
| 785 | |
| 786 | class Block: |
| 787 | r""" |
| 788 | Represents a single block of text embedded in |
| 789 | another file. If dsl_name is None, the block represents |
| 790 | verbatim text, raw original text from the file, in |
| 791 | which case "input" will be the only non-false member. |
| 792 | If dsl_name is not None, the block represents a Clinic |
| 793 | block. |
| 794 | |
| 795 | input is always str, with embedded \n characters. |
| 796 | input represents the original text from the file; |
| 797 | if it's a Clinic block, it is the original text with |
| 798 | the body_prefix and redundant leading whitespace removed. |
| 799 | |
| 800 | dsl_name is either str or None. If str, it's the text |
| 801 | found on the start line of the block between the square |
| 802 | brackets. |
| 803 | |
| 804 | signatures is either list or None. If it's a list, |
| 805 | it may only contain clinic.Module, clinic.Class, and |
| 806 | clinic.Function objects. At the moment it should |
| 807 | contain at most one of each. |
| 808 | |
| 809 | output is either str or None. If str, it's the output |
| 810 | from this block, with embedded '\n' characters. |
| 811 | |
| 812 | indent is either str or None. It's the leading whitespace |
| 813 | that was found on every line of input. (If body_prefix is |
| 814 | not empty, this is the indent *after* removing the |
| 815 | body_prefix.) |
| 816 | |
| 817 | preindent is either str or None. It's the whitespace that |
| 818 | was found in front of every line of input *before* the |
| 819 | "body_prefix" (see the Language object). If body_prefix |
| 820 | is empty, preindent must always be empty too. |
| 821 | |
| 822 | To illustrate indent and preindent: Assume that '_' |
| 823 | represents whitespace. If the block processed was in a |
| 824 | Python file, and looked like this: |
| 825 | ____#/*[python] |
| 826 | ____#__for a in range(20): |
| 827 | ____#____print(a) |
| 828 | ____#[python]*/ |
| 829 | "preindent" would be "____" and "indent" would be "__". |
| 830 | |
| 831 | """ |
| 832 | def __init__(self, input, dsl_name=None, signatures=None, output=None, indent='', preindent=''): |
| 833 | assert isinstance(input, str) |
| 834 | self.input = input |
| 835 | self.dsl_name = dsl_name |
| 836 | self.signatures = signatures or [] |
| 837 | self.output = output |
| 838 | self.indent = indent |
| 839 | self.preindent = preindent |
| 840 | |
| 841 | |
| 842 | class BlockParser: |
| 843 | """ |
| 844 | Block-oriented parser for Argument Clinic. |
| 845 | Iterator, yields Block objects. |
| 846 | """ |
| 847 | |
| 848 | def __init__(self, input, language, *, verify=True): |
| 849 | """ |
| 850 | "input" should be a str object |
| 851 | with embedded \n characters. |
| 852 | |
| 853 | "language" should be a Language object. |
| 854 | """ |
| 855 | language.validate() |
| 856 | |
| 857 | self.input = collections.deque(reversed(input.splitlines(keepends=True))) |
| 858 | self.block_start_line_number = self.line_number = 0 |
| 859 | |
| 860 | self.language = language |
| 861 | before, _, after = language.start_line.partition('{dsl_name}') |
| 862 | assert _ == '{dsl_name}' |
| 863 | self.start_re = create_regex(before, after) |
| 864 | self.verify = verify |
| 865 | self.last_checksum_re = None |
| 866 | self.last_dsl_name = None |
| 867 | self.dsl_name = None |
| 868 | |
| 869 | def __iter__(self): |
| 870 | return self |
| 871 | |
| 872 | def __next__(self): |
| 873 | if not self.input: |
| 874 | raise StopIteration |
| 875 | |
| 876 | if self.dsl_name: |
| 877 | return_value = self.parse_clinic_block(self.dsl_name) |
| 878 | self.dsl_name = None |
| 879 | return return_value |
| 880 | return self.parse_verbatim_block() |
| 881 | |
| 882 | def is_start_line(self, line): |
| 883 | match = self.start_re.match(line.lstrip()) |
| 884 | return match.group(1) if match else None |
| 885 | |
| 886 | def _line(self): |
| 887 | self.line_number += 1 |
| 888 | return self.input.pop() |
| 889 | |
| 890 | def parse_verbatim_block(self): |
| 891 | add, output = text_accumulator() |
| 892 | self.block_start_line_number = self.line_number |
| 893 | |
| 894 | while self.input: |
| 895 | line = self._line() |
| 896 | dsl_name = self.is_start_line(line) |
| 897 | if dsl_name: |
| 898 | self.dsl_name = dsl_name |
| 899 | break |
| 900 | add(line) |
| 901 | |
| 902 | return Block(output()) |
| 903 | |
| 904 | def parse_clinic_block(self, dsl_name): |
| 905 | input_add, input_output = text_accumulator() |
| 906 | self.block_start_line_number = self.line_number + 1 |
Larry Hastings | 9026113 | 2014-01-07 12:21:08 -0800 | [diff] [blame] | 907 | stop_line = self.language.stop_line.format(dsl_name=dsl_name) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 908 | body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) |
| 909 | |
Larry Hastings | 9026113 | 2014-01-07 12:21:08 -0800 | [diff] [blame] | 910 | def is_stop_line(line): |
| 911 | # make sure to recognize stop line even if it |
| 912 | # doesn't end with EOL (it could be the very end of the file) |
| 913 | if not line.startswith(stop_line): |
| 914 | return False |
| 915 | remainder = line[len(stop_line):] |
| 916 | return (not remainder) or remainder.isspace() |
| 917 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 918 | # consume body of program |
| 919 | while self.input: |
| 920 | line = self._line() |
Larry Hastings | 9026113 | 2014-01-07 12:21:08 -0800 | [diff] [blame] | 921 | if is_stop_line(line) or self.is_start_line(line): |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 922 | break |
| 923 | if body_prefix: |
| 924 | line = line.lstrip() |
| 925 | assert line.startswith(body_prefix) |
| 926 | line = line[len(body_prefix):] |
| 927 | input_add(line) |
| 928 | |
| 929 | # consume output and checksum line, if present. |
| 930 | if self.last_dsl_name == dsl_name: |
| 931 | checksum_re = self.last_checksum_re |
| 932 | else: |
| 933 | before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, checksum='{checksum}').partition('{checksum}') |
| 934 | assert _ == '{checksum}' |
| 935 | checksum_re = create_regex(before, after) |
| 936 | self.last_dsl_name = dsl_name |
| 937 | self.last_checksum_re = checksum_re |
| 938 | |
| 939 | # scan forward for checksum line |
| 940 | output_add, output_output = text_accumulator() |
| 941 | checksum = None |
| 942 | while self.input: |
| 943 | line = self._line() |
| 944 | match = checksum_re.match(line.lstrip()) |
| 945 | checksum = match.group(1) if match else None |
| 946 | if checksum: |
| 947 | break |
| 948 | output_add(line) |
| 949 | if self.is_start_line(line): |
| 950 | break |
| 951 | |
Larry Hastings | ef3b1fb | 2013-10-22 23:26:23 -0700 | [diff] [blame] | 952 | output = output_output() |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 953 | if checksum: |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 954 | if self.verify: |
| 955 | computed = compute_checksum(output) |
| 956 | if checksum != computed: |
| 957 | fail("Checksum mismatch!\nExpected: {}\nComputed: {}".format(checksum, computed)) |
| 958 | else: |
| 959 | # put back output |
Larry Hastings | eb31e9d | 2014-01-06 11:10:08 -0800 | [diff] [blame] | 960 | output_lines = output.splitlines(keepends=True) |
| 961 | self.line_number -= len(output_lines) |
| 962 | self.input.extend(reversed(output_lines)) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 963 | output = None |
| 964 | |
| 965 | return Block(input_output(), dsl_name, output=output) |
| 966 | |
| 967 | |
| 968 | class BlockPrinter: |
| 969 | |
| 970 | def __init__(self, language, f=None): |
| 971 | self.language = language |
| 972 | self.f = f or io.StringIO() |
| 973 | |
| 974 | def print_block(self, block): |
| 975 | input = block.input |
| 976 | output = block.output |
| 977 | dsl_name = block.dsl_name |
| 978 | write = self.f.write |
| 979 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 980 | assert not ((dsl_name == None) ^ (output == None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) |
| 981 | |
| 982 | if not dsl_name: |
| 983 | write(input) |
| 984 | return |
| 985 | |
| 986 | write(self.language.start_line.format(dsl_name=dsl_name)) |
| 987 | write("\n") |
| 988 | |
| 989 | body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) |
| 990 | if not body_prefix: |
| 991 | write(input) |
| 992 | else: |
| 993 | for line in input.split('\n'): |
| 994 | write(body_prefix) |
| 995 | write(line) |
| 996 | write("\n") |
| 997 | |
| 998 | write(self.language.stop_line.format(dsl_name=dsl_name)) |
| 999 | write("\n") |
| 1000 | |
| 1001 | output = block.output |
| 1002 | if output: |
| 1003 | write(output) |
| 1004 | if not output.endswith('\n'): |
| 1005 | write('\n') |
| 1006 | |
| 1007 | write(self.language.checksum_line.format(dsl_name=dsl_name, checksum=compute_checksum(output))) |
| 1008 | write("\n") |
| 1009 | |
| 1010 | |
| 1011 | # maps strings to Language objects. |
| 1012 | # "languages" maps the name of the language ("C", "Python"). |
| 1013 | # "extensions" maps the file extension ("c", "py"). |
| 1014 | languages = { 'C': CLanguage, 'Python': PythonLanguage } |
Larry Hastings | 6d2ea21 | 2014-01-05 02:50:45 -0800 | [diff] [blame] | 1015 | extensions = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } |
| 1016 | extensions['py'] = PythonLanguage |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1017 | |
| 1018 | |
| 1019 | # maps strings to callables. |
| 1020 | # these callables must be of the form: |
| 1021 | # def foo(name, default, *, ...) |
| 1022 | # The callable may have any number of keyword-only parameters. |
| 1023 | # The callable must return a CConverter object. |
| 1024 | # The callable should not call builtins.print. |
| 1025 | converters = {} |
| 1026 | |
| 1027 | # maps strings to callables. |
| 1028 | # these callables follow the same rules as those for "converters" above. |
| 1029 | # note however that they will never be called with keyword-only parameters. |
| 1030 | legacy_converters = {} |
| 1031 | |
| 1032 | |
| 1033 | # maps strings to callables. |
| 1034 | # these callables must be of the form: |
| 1035 | # def foo(*, ...) |
| 1036 | # The callable may have any number of keyword-only parameters. |
| 1037 | # The callable must return a CConverter object. |
| 1038 | # The callable should not call builtins.print. |
| 1039 | return_converters = {} |
| 1040 | |
| 1041 | class Clinic: |
| 1042 | def __init__(self, language, printer=None, *, verify=True, filename=None): |
| 1043 | # maps strings to Parser objects. |
| 1044 | # (instantiated from the "parsers" global.) |
| 1045 | self.parsers = {} |
| 1046 | self.language = language |
| 1047 | self.printer = printer or BlockPrinter(language) |
| 1048 | self.verify = verify |
| 1049 | self.filename = filename |
| 1050 | self.modules = collections.OrderedDict() |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 1051 | self.classes = collections.OrderedDict() |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1052 | |
| 1053 | global clinic |
| 1054 | clinic = self |
| 1055 | |
| 1056 | def parse(self, input): |
| 1057 | printer = self.printer |
| 1058 | self.block_parser = BlockParser(input, self.language, verify=self.verify) |
| 1059 | for block in self.block_parser: |
| 1060 | dsl_name = block.dsl_name |
| 1061 | if dsl_name: |
| 1062 | if dsl_name not in self.parsers: |
| 1063 | assert dsl_name in parsers, "No parser to handle {!r} block.".format(dsl_name) |
| 1064 | self.parsers[dsl_name] = parsers[dsl_name](self) |
| 1065 | parser = self.parsers[dsl_name] |
| 1066 | parser.parse(block) |
| 1067 | printer.print_block(block) |
| 1068 | return printer.f.getvalue() |
| 1069 | |
| 1070 | def _module_and_class(self, fields): |
| 1071 | """ |
| 1072 | fields should be an iterable of field names. |
| 1073 | returns a tuple of (module, class). |
| 1074 | the module object could actually be self (a clinic object). |
| 1075 | this function is only ever used to find the parent of where |
| 1076 | a new class/module should go. |
| 1077 | """ |
| 1078 | in_classes = False |
| 1079 | parent = module = self |
| 1080 | cls = None |
| 1081 | so_far = [] |
| 1082 | |
| 1083 | for field in fields: |
| 1084 | so_far.append(field) |
| 1085 | if not in_classes: |
| 1086 | child = parent.modules.get(field) |
| 1087 | if child: |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 1088 | parent = module = child |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1089 | continue |
| 1090 | in_classes = True |
| 1091 | if not hasattr(parent, 'classes'): |
| 1092 | return module, cls |
| 1093 | child = parent.classes.get(field) |
| 1094 | if not child: |
| 1095 | fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") |
| 1096 | cls = parent = child |
| 1097 | |
| 1098 | return module, cls |
| 1099 | |
| 1100 | |
| 1101 | def parse_file(filename, *, verify=True, output=None, encoding='utf-8'): |
| 1102 | extension = os.path.splitext(filename)[1][1:] |
| 1103 | if not extension: |
| 1104 | fail("Can't extract file type for file " + repr(filename)) |
| 1105 | |
| 1106 | try: |
| 1107 | language = extensions[extension]() |
| 1108 | except KeyError: |
| 1109 | fail("Can't identify file type for file " + repr(filename)) |
| 1110 | |
| 1111 | clinic = Clinic(language, verify=verify, filename=filename) |
| 1112 | |
| 1113 | with open(filename, 'r', encoding=encoding) as f: |
Larry Hastings | dcd340e | 2013-11-23 14:58:45 -0800 | [diff] [blame] | 1114 | raw = f.read() |
| 1115 | |
| 1116 | cooked = clinic.parse(raw) |
| 1117 | if cooked == raw: |
| 1118 | return |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1119 | |
| 1120 | directory = os.path.dirname(filename) or '.' |
| 1121 | |
| 1122 | with tempfile.TemporaryDirectory(prefix="clinic", dir=directory) as tmpdir: |
Larry Hastings | dcd340e | 2013-11-23 14:58:45 -0800 | [diff] [blame] | 1123 | bytes = cooked.encode(encoding) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1124 | tmpfilename = os.path.join(tmpdir, os.path.basename(filename)) |
| 1125 | with open(tmpfilename, "wb") as f: |
| 1126 | f.write(bytes) |
| 1127 | os.replace(tmpfilename, output or filename) |
| 1128 | |
| 1129 | |
| 1130 | def compute_checksum(input): |
| 1131 | input = input or '' |
| 1132 | return hashlib.sha1(input.encode('utf-8')).hexdigest() |
| 1133 | |
| 1134 | |
| 1135 | |
| 1136 | |
| 1137 | class PythonParser: |
| 1138 | def __init__(self, clinic): |
| 1139 | pass |
| 1140 | |
| 1141 | def parse(self, block): |
| 1142 | s = io.StringIO() |
| 1143 | with OverrideStdioWith(s): |
| 1144 | exec(block.input) |
| 1145 | block.output = s.getvalue() |
| 1146 | |
| 1147 | |
| 1148 | class Module: |
| 1149 | def __init__(self, name, module=None): |
| 1150 | self.name = name |
| 1151 | self.module = self.parent = module |
| 1152 | |
| 1153 | self.modules = collections.OrderedDict() |
| 1154 | self.classes = collections.OrderedDict() |
| 1155 | self.functions = [] |
| 1156 | |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 1157 | def __repr__(self): |
| 1158 | return "<clinic.Module " + repr(self.name) + " at " + str(id(self)) + ">" |
| 1159 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1160 | class Class: |
| 1161 | def __init__(self, name, module=None, cls=None): |
| 1162 | self.name = name |
| 1163 | self.module = module |
| 1164 | self.cls = cls |
| 1165 | self.parent = cls or module |
| 1166 | |
| 1167 | self.classes = collections.OrderedDict() |
| 1168 | self.functions = [] |
| 1169 | |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 1170 | def __repr__(self): |
| 1171 | return "<clinic.Class " + repr(self.name) + " at " + str(id(self)) + ">" |
| 1172 | |
| 1173 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1174 | DATA, CALLABLE, METHOD, STATIC_METHOD, CLASS_METHOD = range(5) |
| 1175 | |
| 1176 | class Function: |
| 1177 | """ |
| 1178 | Mutable duck type for inspect.Function. |
| 1179 | |
| 1180 | docstring - a str containing |
| 1181 | * embedded line breaks |
| 1182 | * text outdented to the left margin |
| 1183 | * no trailing whitespace. |
| 1184 | It will always be true that |
| 1185 | (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) |
| 1186 | """ |
| 1187 | |
| 1188 | def __init__(self, parameters=None, *, name, |
| 1189 | module, cls=None, c_basename=None, |
| 1190 | full_name=None, |
| 1191 | return_converter, return_annotation=_empty, |
| 1192 | docstring=None, kind=CALLABLE, coexist=False): |
| 1193 | self.parameters = parameters or collections.OrderedDict() |
| 1194 | self.return_annotation = return_annotation |
| 1195 | self.name = name |
| 1196 | self.full_name = full_name |
| 1197 | self.module = module |
| 1198 | self.cls = cls |
| 1199 | self.parent = cls or module |
| 1200 | self.c_basename = c_basename |
| 1201 | self.return_converter = return_converter |
| 1202 | self.docstring = docstring or '' |
| 1203 | self.kind = kind |
| 1204 | self.coexist = coexist |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1205 | self.self_converter = None |
| 1206 | |
| 1207 | @property |
| 1208 | def methoddef_flags(self): |
| 1209 | flags = [] |
| 1210 | if self.kind == CLASS_METHOD: |
| 1211 | flags.append('METH_CLASS') |
| 1212 | elif self.kind == STATIC_METHOD: |
| 1213 | flags.append('METH_STATIC') |
| 1214 | else: |
| 1215 | assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind) |
| 1216 | if self.coexist: |
| 1217 | flags.append('METH_COEXIST') |
| 1218 | return '|'.join(flags) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1219 | |
| 1220 | def __repr__(self): |
| 1221 | return '<clinic.Function ' + self.name + '>' |
| 1222 | |
| 1223 | |
| 1224 | class Parameter: |
| 1225 | """ |
| 1226 | Mutable duck type of inspect.Parameter. |
| 1227 | """ |
| 1228 | |
| 1229 | def __init__(self, name, kind, *, default=_empty, |
| 1230 | function, converter, annotation=_empty, |
| 1231 | docstring=None, group=0): |
| 1232 | self.name = name |
| 1233 | self.kind = kind |
| 1234 | self.default = default |
| 1235 | self.function = function |
| 1236 | self.converter = converter |
| 1237 | self.annotation = annotation |
| 1238 | self.docstring = docstring or '' |
| 1239 | self.group = group |
| 1240 | |
| 1241 | def __repr__(self): |
| 1242 | return '<clinic.Parameter ' + self.name + '>' |
| 1243 | |
| 1244 | def is_keyword_only(self): |
| 1245 | return self.kind == inspect.Parameter.KEYWORD_ONLY |
| 1246 | |
| 1247 | py_special_values = { |
| 1248 | NULL: "None", |
| 1249 | } |
| 1250 | |
| 1251 | def py_repr(o): |
| 1252 | special = py_special_values.get(o) |
| 1253 | if special: |
| 1254 | return special |
| 1255 | return repr(o) |
| 1256 | |
| 1257 | |
| 1258 | c_special_values = { |
| 1259 | NULL: "NULL", |
| 1260 | None: "Py_None", |
| 1261 | } |
| 1262 | |
| 1263 | def c_repr(o): |
| 1264 | special = c_special_values.get(o) |
| 1265 | if special: |
| 1266 | return special |
| 1267 | if isinstance(o, str): |
| 1268 | return '"' + quoted_for_c_string(o) + '"' |
| 1269 | return repr(o) |
| 1270 | |
| 1271 | def add_c_converter(f, name=None): |
| 1272 | if not name: |
| 1273 | name = f.__name__ |
| 1274 | if not name.endswith('_converter'): |
| 1275 | return f |
| 1276 | name = name[:-len('_converter')] |
| 1277 | converters[name] = f |
| 1278 | return f |
| 1279 | |
| 1280 | def add_default_legacy_c_converter(cls): |
| 1281 | # automatically add converter for default format unit |
| 1282 | # (but without stomping on the existing one if it's already |
| 1283 | # set, in case you subclass) |
| 1284 | if ((cls.format_unit != 'O&') and |
| 1285 | (cls.format_unit not in legacy_converters)): |
| 1286 | legacy_converters[cls.format_unit] = cls |
| 1287 | return cls |
| 1288 | |
| 1289 | def add_legacy_c_converter(format_unit, **kwargs): |
| 1290 | """ |
| 1291 | Adds a legacy converter. |
| 1292 | """ |
| 1293 | def closure(f): |
| 1294 | if not kwargs: |
| 1295 | added_f = f |
| 1296 | else: |
| 1297 | added_f = functools.partial(f, **kwargs) |
| 1298 | legacy_converters[format_unit] = added_f |
| 1299 | return f |
| 1300 | return closure |
| 1301 | |
| 1302 | class CConverterAutoRegister(type): |
| 1303 | def __init__(cls, name, bases, classdict): |
| 1304 | add_c_converter(cls) |
| 1305 | add_default_legacy_c_converter(cls) |
| 1306 | |
| 1307 | class CConverter(metaclass=CConverterAutoRegister): |
| 1308 | """ |
| 1309 | For the init function, self, name, function, and default |
| 1310 | must be keyword-or-positional parameters. All other |
| 1311 | parameters (including "required" and "doc_default") |
| 1312 | must be keyword-only. |
| 1313 | """ |
| 1314 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1315 | # The C type to use for this variable. |
| 1316 | # 'type' should be a Python string specifying the type, e.g. "int". |
| 1317 | # If this is a pointer type, the type string should end with ' *'. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1318 | type = None |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1319 | |
| 1320 | # The Python default value for this parameter, as a Python value. |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1321 | # Or the magic value "unspecified" if there is no default. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1322 | default = unspecified |
| 1323 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1324 | # "default" as it should appear in the documentation, as a string. |
| 1325 | # Or None if there is no default. |
| 1326 | doc_default = None |
| 1327 | |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1328 | # "default" converted into a str for rendering into Python code. |
| 1329 | py_default = None |
| 1330 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1331 | # "default" converted into a C value, as a string. |
| 1332 | # Or None if there is no default. |
| 1333 | c_default = None |
| 1334 | |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1335 | # The default value used to initialize the C variable when |
| 1336 | # there is no default, but not specifying a default may |
| 1337 | # result in an "uninitialized variable" warning. This can |
| 1338 | # easily happen when using option groups--although |
| 1339 | # properly-written code won't actually use the variable, |
| 1340 | # the variable does get passed in to the _impl. (Ah, if |
| 1341 | # only dataflow analysis could inline the static function!) |
| 1342 | # |
| 1343 | # This value is specified as a string. |
| 1344 | # Every non-abstract subclass should supply a valid value. |
| 1345 | c_ignored_default = 'NULL' |
| 1346 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1347 | # The C converter *function* to be used, if any. |
| 1348 | # (If this is not None, format_unit must be 'O&'.) |
| 1349 | converter = None |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1350 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1351 | # Should Argument Clinic add a '&' before the name of |
| 1352 | # the variable when passing it into the _impl function? |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1353 | impl_by_reference = False |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1354 | |
| 1355 | # Should Argument Clinic add a '&' before the name of |
| 1356 | # the variable when passing it into PyArg_ParseTuple (AndKeywords)? |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1357 | parse_by_reference = True |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1358 | |
| 1359 | ############################################################# |
| 1360 | ############################################################# |
| 1361 | ## You shouldn't need to read anything below this point to ## |
| 1362 | ## write your own converter functions. ## |
| 1363 | ############################################################# |
| 1364 | ############################################################# |
| 1365 | |
| 1366 | # The "format unit" to specify for this variable when |
| 1367 | # parsing arguments using PyArg_ParseTuple (AndKeywords). |
| 1368 | # Custom converters should always use the default value of 'O&'. |
| 1369 | format_unit = 'O&' |
| 1370 | |
| 1371 | # What encoding do we want for this variable? Only used |
| 1372 | # by format units starting with 'e'. |
| 1373 | encoding = None |
| 1374 | |
Larry Hastings | 77561cc | 2014-01-07 12:13:13 -0800 | [diff] [blame] | 1375 | # Should this object be required to be a subclass of a specific type? |
| 1376 | # If not None, should be a string representing a pointer to a |
| 1377 | # PyTypeObject (e.g. "&PyUnicode_Type"). |
| 1378 | # Only used by the 'O!' format unit (and the "object" converter). |
| 1379 | subclass_of = None |
| 1380 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1381 | # Do we want an adjacent '_length' variable for this variable? |
| 1382 | # Only used by format units ending with '#'. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1383 | length = False |
| 1384 | |
Larry Hastings | 16c5191 | 2014-01-07 11:53:01 -0800 | [diff] [blame] | 1385 | def __init__(self, name, function, default=unspecified, *, doc_default=None, c_default=None, py_default=None, required=False, annotation=unspecified, **kwargs): |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1386 | self.function = function |
| 1387 | self.name = name |
| 1388 | |
| 1389 | if default is not unspecified: |
| 1390 | self.default = default |
Larry Hastings | 16c5191 | 2014-01-07 11:53:01 -0800 | [diff] [blame] | 1391 | self.py_default = py_default if py_default is not None else py_repr(default) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1392 | self.doc_default = doc_default if doc_default is not None else self.py_default |
Larry Hastings | 16c5191 | 2014-01-07 11:53:01 -0800 | [diff] [blame] | 1393 | self.c_default = c_default if c_default is not None else c_repr(default) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1394 | elif doc_default is not None: |
| 1395 | fail(function.fullname + " argument " + name + " specified a 'doc_default' without having a 'default'") |
| 1396 | if annotation != unspecified: |
| 1397 | fail("The 'annotation' parameter is not currently permitted.") |
| 1398 | self.required = required |
| 1399 | self.converter_init(**kwargs) |
| 1400 | |
| 1401 | def converter_init(self): |
| 1402 | pass |
| 1403 | |
| 1404 | def is_optional(self): |
| 1405 | return (self.default is not unspecified) and (not self.required) |
| 1406 | |
| 1407 | def render(self, parameter, data): |
| 1408 | """ |
| 1409 | parameter is a clinic.Parameter instance. |
| 1410 | data is a CRenderData instance. |
| 1411 | """ |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1412 | self.parameter = parameter |
Larry Hastings | 9026113 | 2014-01-07 12:21:08 -0800 | [diff] [blame] | 1413 | original_name = self.name |
| 1414 | name = ensure_legal_c_identifier(original_name) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1415 | |
| 1416 | # declarations |
| 1417 | d = self.declaration() |
| 1418 | data.declarations.append(d) |
| 1419 | |
| 1420 | # initializers |
| 1421 | initializers = self.initialize() |
| 1422 | if initializers: |
| 1423 | data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) |
| 1424 | |
| 1425 | # impl_arguments |
| 1426 | s = ("&" if self.impl_by_reference else "") + name |
| 1427 | data.impl_arguments.append(s) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1428 | if self.length: |
| 1429 | data.impl_arguments.append(self.length_name()) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1430 | |
| 1431 | # keywords |
Larry Hastings | 9026113 | 2014-01-07 12:21:08 -0800 | [diff] [blame] | 1432 | data.keywords.append(original_name) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1433 | |
| 1434 | # format_units |
| 1435 | if self.is_optional() and '|' not in data.format_units: |
| 1436 | data.format_units.append('|') |
| 1437 | if parameter.is_keyword_only() and '$' not in data.format_units: |
| 1438 | data.format_units.append('$') |
| 1439 | data.format_units.append(self.format_unit) |
| 1440 | |
| 1441 | # parse_arguments |
| 1442 | self.parse_argument(data.parse_arguments) |
| 1443 | |
| 1444 | # impl_parameters |
| 1445 | data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1446 | if self.length: |
| 1447 | data.impl_parameters.append("Py_ssize_clean_t " + self.length_name()) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1448 | |
| 1449 | # cleanup |
| 1450 | cleanup = self.cleanup() |
| 1451 | if cleanup: |
| 1452 | data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") |
| 1453 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1454 | def length_name(self): |
| 1455 | """Computes the name of the associated "length" variable.""" |
| 1456 | if not self.length: |
| 1457 | return None |
| 1458 | return ensure_legal_c_identifier(self.name) + "_length" |
| 1459 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1460 | # Why is this one broken out separately? |
| 1461 | # For "positional-only" function parsing, |
| 1462 | # which generates a bunch of PyArg_ParseTuple calls. |
| 1463 | def parse_argument(self, list): |
| 1464 | assert not (self.converter and self.encoding) |
| 1465 | if self.format_unit == 'O&': |
| 1466 | assert self.converter |
| 1467 | list.append(self.converter) |
| 1468 | |
| 1469 | if self.encoding: |
Larry Hastings | 77561cc | 2014-01-07 12:13:13 -0800 | [diff] [blame] | 1470 | list.append(c_repr(self.encoding)) |
| 1471 | elif self.subclass_of: |
| 1472 | list.append(self.subclass_of) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1473 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1474 | legal_name = ensure_legal_c_identifier(self.name) |
| 1475 | s = ("&" if self.parse_by_reference else "") + legal_name |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1476 | list.append(s) |
| 1477 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1478 | if self.length: |
| 1479 | list.append("&" + self.length_name()) |
| 1480 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1481 | # |
| 1482 | # All the functions after here are intended as extension points. |
| 1483 | # |
| 1484 | |
| 1485 | def simple_declaration(self, by_reference=False): |
| 1486 | """ |
| 1487 | Computes the basic declaration of the variable. |
| 1488 | Used in computing the prototype declaration and the |
| 1489 | variable declaration. |
| 1490 | """ |
| 1491 | prototype = [self.type] |
| 1492 | if by_reference or not self.type.endswith('*'): |
| 1493 | prototype.append(" ") |
| 1494 | if by_reference: |
| 1495 | prototype.append('*') |
Larry Hastings | dfcd467 | 2013-10-27 02:49:39 -0700 | [diff] [blame] | 1496 | prototype.append(ensure_legal_c_identifier(self.name)) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1497 | return "".join(prototype) |
| 1498 | |
| 1499 | def declaration(self): |
| 1500 | """ |
| 1501 | The C statement to declare this variable. |
| 1502 | """ |
| 1503 | declaration = [self.simple_declaration()] |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1504 | default = self.c_default |
| 1505 | if not default and self.parameter.group: |
| 1506 | default = self.c_ignored_default |
| 1507 | if default: |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1508 | declaration.append(" = ") |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1509 | declaration.append(default) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1510 | declaration.append(";") |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1511 | if self.length: |
| 1512 | declaration.append('\nPy_ssize_clean_t ') |
| 1513 | declaration.append(self.length_name()) |
| 1514 | declaration.append(';') |
Larry Hastings | 3f144c2 | 2014-01-06 10:34:00 -0800 | [diff] [blame] | 1515 | s = "".join(declaration) |
| 1516 | # double up curly-braces, this string will be used |
| 1517 | # as part of a format_map() template later |
| 1518 | s = s.replace("{", "{{") |
| 1519 | s = s.replace("}", "}}") |
| 1520 | return s |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1521 | |
| 1522 | def initialize(self): |
| 1523 | """ |
| 1524 | The C statements required to set up this variable before parsing. |
| 1525 | Returns a string containing this code indented at column 0. |
| 1526 | If no initialization is necessary, returns an empty string. |
| 1527 | """ |
| 1528 | return "" |
| 1529 | |
| 1530 | def cleanup(self): |
| 1531 | """ |
| 1532 | The C statements required to clean up after this variable. |
| 1533 | Returns a string containing this code indented at column 0. |
| 1534 | If no cleanup is necessary, returns an empty string. |
| 1535 | """ |
| 1536 | return "" |
| 1537 | |
| 1538 | |
| 1539 | class bool_converter(CConverter): |
| 1540 | type = 'int' |
| 1541 | format_unit = 'p' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1542 | c_ignored_default = '0' |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1543 | |
| 1544 | def converter_init(self): |
| 1545 | self.default = bool(self.default) |
| 1546 | self.c_default = str(int(self.default)) |
| 1547 | |
| 1548 | class char_converter(CConverter): |
| 1549 | type = 'char' |
| 1550 | format_unit = 'c' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1551 | c_ignored_default = "'\0'" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1552 | |
| 1553 | @add_legacy_c_converter('B', bitwise=True) |
| 1554 | class byte_converter(CConverter): |
| 1555 | type = 'byte' |
| 1556 | format_unit = 'b' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1557 | c_ignored_default = "'\0'" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1558 | |
| 1559 | def converter_init(self, *, bitwise=False): |
| 1560 | if bitwise: |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1561 | self.format_unit = 'B' |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1562 | |
| 1563 | class short_converter(CConverter): |
| 1564 | type = 'short' |
| 1565 | format_unit = 'h' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1566 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1567 | |
| 1568 | class unsigned_short_converter(CConverter): |
| 1569 | type = 'unsigned short' |
| 1570 | format_unit = 'H' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1571 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1572 | |
| 1573 | def converter_init(self, *, bitwise=False): |
| 1574 | if not bitwise: |
| 1575 | fail("Unsigned shorts must be bitwise (for now).") |
| 1576 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1577 | @add_legacy_c_converter('C', types='str') |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1578 | class int_converter(CConverter): |
| 1579 | type = 'int' |
| 1580 | format_unit = 'i' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1581 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1582 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1583 | def converter_init(self, *, types='int'): |
| 1584 | if types == 'str': |
| 1585 | self.format_unit = 'C' |
| 1586 | elif types != 'int': |
| 1587 | fail("int_converter: illegal 'types' argument") |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1588 | |
| 1589 | class unsigned_int_converter(CConverter): |
| 1590 | type = 'unsigned int' |
| 1591 | format_unit = 'I' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1592 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1593 | |
| 1594 | def converter_init(self, *, bitwise=False): |
| 1595 | if not bitwise: |
| 1596 | fail("Unsigned ints must be bitwise (for now).") |
| 1597 | |
| 1598 | class long_converter(CConverter): |
| 1599 | type = 'long' |
| 1600 | format_unit = 'l' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1601 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1602 | |
| 1603 | class unsigned_long_converter(CConverter): |
| 1604 | type = 'unsigned long' |
| 1605 | format_unit = 'k' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1606 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1607 | |
| 1608 | def converter_init(self, *, bitwise=False): |
| 1609 | if not bitwise: |
| 1610 | fail("Unsigned longs must be bitwise (for now).") |
| 1611 | |
| 1612 | class PY_LONG_LONG_converter(CConverter): |
| 1613 | type = 'PY_LONG_LONG' |
| 1614 | format_unit = 'L' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1615 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1616 | |
| 1617 | class unsigned_PY_LONG_LONG_converter(CConverter): |
| 1618 | type = 'unsigned PY_LONG_LONG' |
| 1619 | format_unit = 'K' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1620 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1621 | |
| 1622 | def converter_init(self, *, bitwise=False): |
| 1623 | if not bitwise: |
| 1624 | fail("Unsigned PY_LONG_LONGs must be bitwise (for now).") |
| 1625 | |
| 1626 | class Py_ssize_t_converter(CConverter): |
| 1627 | type = 'Py_ssize_t' |
| 1628 | format_unit = 'n' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1629 | c_ignored_default = "0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1630 | |
| 1631 | |
| 1632 | class float_converter(CConverter): |
| 1633 | type = 'float' |
| 1634 | format_unit = 'f' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1635 | c_ignored_default = "0.0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1636 | |
| 1637 | class double_converter(CConverter): |
| 1638 | type = 'double' |
| 1639 | format_unit = 'd' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1640 | c_ignored_default = "0.0" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1641 | |
| 1642 | |
| 1643 | class Py_complex_converter(CConverter): |
| 1644 | type = 'Py_complex' |
| 1645 | format_unit = 'D' |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1646 | c_ignored_default = "{0.0, 0.0}" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1647 | |
| 1648 | |
| 1649 | class object_converter(CConverter): |
| 1650 | type = 'PyObject *' |
| 1651 | format_unit = 'O' |
| 1652 | |
Larry Hastings | 77561cc | 2014-01-07 12:13:13 -0800 | [diff] [blame] | 1653 | def converter_init(self, *, type=None, subclass_of=None): |
| 1654 | if subclass_of: |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1655 | self.format_unit = 'O!' |
Larry Hastings | 77561cc | 2014-01-07 12:13:13 -0800 | [diff] [blame] | 1656 | self.subclass_of = subclass_of |
| 1657 | if type is not None: |
| 1658 | self.type = type |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1659 | |
| 1660 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1661 | @add_legacy_c_converter('s#', length=True) |
| 1662 | @add_legacy_c_converter('y', type="bytes") |
| 1663 | @add_legacy_c_converter('y#', type="bytes", length=True) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1664 | @add_legacy_c_converter('z', nullable=True) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1665 | @add_legacy_c_converter('z#', nullable=True, length=True) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1666 | class str_converter(CConverter): |
| 1667 | type = 'const char *' |
| 1668 | format_unit = 's' |
| 1669 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1670 | def converter_init(self, *, encoding=None, types="str", |
| 1671 | length=False, nullable=False, zeroes=False): |
| 1672 | |
| 1673 | types = set(types.strip().split()) |
| 1674 | bytes_type = set(("bytes",)) |
| 1675 | str_type = set(("str",)) |
| 1676 | all_3_type = set(("bytearray",)) | bytes_type | str_type |
| 1677 | is_bytes = types == bytes_type |
| 1678 | is_str = types == str_type |
| 1679 | is_all_3 = types == all_3_type |
| 1680 | |
| 1681 | self.length = bool(length) |
| 1682 | format_unit = None |
| 1683 | |
| 1684 | if encoding: |
| 1685 | self.encoding = encoding |
| 1686 | |
| 1687 | if is_str and not (length or zeroes or nullable): |
| 1688 | format_unit = 'es' |
| 1689 | elif is_all_3 and not (length or zeroes or nullable): |
| 1690 | format_unit = 'et' |
| 1691 | elif is_str and length and zeroes and not nullable: |
| 1692 | format_unit = 'es#' |
| 1693 | elif is_all_3 and length and not (nullable or zeroes): |
| 1694 | format_unit = 'et#' |
| 1695 | |
| 1696 | if format_unit.endswith('#'): |
Larry Hastings | 2f9a9aa | 2013-11-24 04:23:35 -0800 | [diff] [blame] | 1697 | print("Warning: code using format unit ", repr(format_unit), "probably doesn't work properly.") |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1698 | # TODO set pointer to NULL |
| 1699 | # TODO add cleanup for buffer |
| 1700 | pass |
| 1701 | |
| 1702 | else: |
| 1703 | if zeroes: |
| 1704 | fail("str_converter: illegal combination of arguments (zeroes is only legal with an encoding)") |
| 1705 | |
| 1706 | if is_bytes and not (nullable or length): |
| 1707 | format_unit = 'y' |
| 1708 | elif is_bytes and length and not nullable: |
| 1709 | format_unit = 'y#' |
| 1710 | elif is_str and not (nullable or length): |
| 1711 | format_unit = 's' |
| 1712 | elif is_str and length and not nullable: |
| 1713 | format_unit = 's#' |
| 1714 | elif is_str and nullable and not length: |
| 1715 | format_unit = 'z' |
| 1716 | elif is_str and nullable and length: |
| 1717 | format_unit = 'z#' |
| 1718 | |
| 1719 | if not format_unit: |
| 1720 | fail("str_converter: illegal combination of arguments") |
| 1721 | self.format_unit = format_unit |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1722 | |
| 1723 | |
| 1724 | class PyBytesObject_converter(CConverter): |
| 1725 | type = 'PyBytesObject *' |
| 1726 | format_unit = 'S' |
| 1727 | |
| 1728 | class PyByteArrayObject_converter(CConverter): |
| 1729 | type = 'PyByteArrayObject *' |
| 1730 | format_unit = 'Y' |
| 1731 | |
| 1732 | class unicode_converter(CConverter): |
| 1733 | type = 'PyObject *' |
| 1734 | format_unit = 'U' |
| 1735 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1736 | @add_legacy_c_converter('u#', length=True) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1737 | @add_legacy_c_converter('Z', nullable=True) |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1738 | @add_legacy_c_converter('Z#', nullable=True, length=True) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1739 | class Py_UNICODE_converter(CConverter): |
| 1740 | type = 'Py_UNICODE *' |
| 1741 | format_unit = 'u' |
| 1742 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1743 | def converter_init(self, *, nullable=False, length=False): |
| 1744 | format_unit = 'Z' if nullable else 'u' |
| 1745 | if length: |
| 1746 | format_unit += '#' |
| 1747 | self.length = True |
| 1748 | self.format_unit = format_unit |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1749 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1750 | # |
| 1751 | # We define three string conventions for buffer types in the 'types' argument: |
| 1752 | # 'buffer' : any object supporting the buffer interface |
| 1753 | # 'rwbuffer': any object supporting the buffer interface, but must be writeable |
| 1754 | # 'robuffer': any object supporting the buffer interface, but must not be writeable |
| 1755 | # |
| 1756 | @add_legacy_c_converter('s*', types='str bytes bytearray buffer') |
| 1757 | @add_legacy_c_converter('z*', types='str bytes bytearray buffer', nullable=True) |
| 1758 | @add_legacy_c_converter('w*', types='bytearray rwbuffer') |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1759 | class Py_buffer_converter(CConverter): |
| 1760 | type = 'Py_buffer' |
| 1761 | format_unit = 'y*' |
| 1762 | impl_by_reference = True |
Larry Hastings | abc716b | 2013-11-20 09:13:52 -0800 | [diff] [blame] | 1763 | c_ignored_default = "{NULL, NULL, 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL}" |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1764 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1765 | def converter_init(self, *, types='bytes bytearray buffer', nullable=False): |
Larry Hastings | 3f144c2 | 2014-01-06 10:34:00 -0800 | [diff] [blame] | 1766 | if self.default != unspecified: |
| 1767 | fail("There is no legal default value for Py_buffer ") |
| 1768 | self.c_default = self.c_ignored_default |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1769 | types = set(types.strip().split()) |
| 1770 | bytes_type = set(('bytes',)) |
| 1771 | bytearray_type = set(('bytearray',)) |
| 1772 | buffer_type = set(('buffer',)) |
| 1773 | rwbuffer_type = set(('rwbuffer',)) |
| 1774 | robuffer_type = set(('robuffer',)) |
| 1775 | str_type = set(('str',)) |
| 1776 | bytes_bytearray_buffer_type = bytes_type | bytearray_type | buffer_type |
| 1777 | |
| 1778 | format_unit = None |
| 1779 | if types == (str_type | bytes_bytearray_buffer_type): |
| 1780 | format_unit = 's*' if not nullable else 'z*' |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1781 | else: |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1782 | if nullable: |
| 1783 | fail('Py_buffer_converter: illegal combination of arguments (nullable=True)') |
| 1784 | elif types == (bytes_bytearray_buffer_type): |
| 1785 | format_unit = 'y*' |
| 1786 | elif types == (bytearray_type | rwuffer_type): |
| 1787 | format_unit = 'w*' |
| 1788 | if not format_unit: |
| 1789 | fail("Py_buffer_converter: illegal combination of arguments") |
| 1790 | |
| 1791 | self.format_unit = format_unit |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1792 | |
| 1793 | def cleanup(self): |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1794 | name = ensure_legal_c_identifier(self.name) |
| 1795 | return "".join(["if (", name, ".buf)\n PyBuffer_Release(&", name, ");\n"]) |
| 1796 | |
| 1797 | |
| 1798 | class self_converter(CConverter): |
| 1799 | """ |
| 1800 | A special-case converter: |
| 1801 | this is the default converter used for "self". |
| 1802 | """ |
| 1803 | type = "PyObject *" |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1804 | def converter_init(self, *, type=None): |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1805 | f = self.function |
| 1806 | if f.kind == CALLABLE: |
| 1807 | if f.cls: |
| 1808 | self.name = "self" |
| 1809 | else: |
| 1810 | self.name = "module" |
| 1811 | self.type = "PyModuleDef *" |
| 1812 | elif f.kind == STATIC_METHOD: |
| 1813 | self.name = "null" |
| 1814 | self.type = "void *" |
| 1815 | elif f.kind == CLASS_METHOD: |
| 1816 | self.name = "cls" |
| 1817 | self.type = "PyTypeObject *" |
| 1818 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1819 | if type: |
| 1820 | self.type = type |
| 1821 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 1822 | def render(self, parameter, data): |
| 1823 | fail("render() should never be called on self_converter instances") |
| 1824 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1825 | |
| 1826 | |
| 1827 | def add_c_return_converter(f, name=None): |
| 1828 | if not name: |
| 1829 | name = f.__name__ |
| 1830 | if not name.endswith('_return_converter'): |
| 1831 | return f |
| 1832 | name = name[:-len('_return_converter')] |
| 1833 | return_converters[name] = f |
| 1834 | return f |
| 1835 | |
| 1836 | |
| 1837 | class CReturnConverterAutoRegister(type): |
| 1838 | def __init__(cls, name, bases, classdict): |
| 1839 | add_c_return_converter(cls) |
| 1840 | |
| 1841 | class CReturnConverter(metaclass=CReturnConverterAutoRegister): |
| 1842 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1843 | # The C type to use for this variable. |
| 1844 | # 'type' should be a Python string specifying the type, e.g. "int". |
| 1845 | # If this is a pointer type, the type string should end with ' *'. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1846 | type = 'PyObject *' |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1847 | |
| 1848 | # The Python default value for this parameter, as a Python value. |
| 1849 | # Or the magic value "unspecified" if there is no default. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1850 | default = None |
| 1851 | |
| 1852 | def __init__(self, *, doc_default=None, **kwargs): |
| 1853 | self.doc_default = doc_default |
| 1854 | try: |
| 1855 | self.return_converter_init(**kwargs) |
| 1856 | except TypeError as e: |
| 1857 | s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) |
| 1858 | sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) |
| 1859 | |
| 1860 | def return_converter_init(self): |
| 1861 | pass |
| 1862 | |
| 1863 | def declare(self, data, name="_return_value"): |
| 1864 | line = [] |
| 1865 | add = line.append |
| 1866 | add(self.type) |
| 1867 | if not self.type.endswith('*'): |
| 1868 | add(' ') |
| 1869 | add(name + ';') |
| 1870 | data.declarations.append(''.join(line)) |
| 1871 | data.return_value = name |
| 1872 | |
| 1873 | def err_occurred_if(self, expr, data): |
| 1874 | data.return_conversion.append('if (({}) && PyErr_Occurred())\n goto exit;\n'.format(expr)) |
| 1875 | |
| 1876 | def err_occurred_if_null_pointer(self, variable, data): |
| 1877 | data.return_conversion.append('if ({} == NULL)\n goto exit;\n'.format(variable)) |
| 1878 | |
| 1879 | def render(self, function, data): |
| 1880 | """ |
| 1881 | function is a clinic.Function instance. |
| 1882 | data is a CRenderData instance. |
| 1883 | """ |
| 1884 | pass |
| 1885 | |
| 1886 | add_c_return_converter(CReturnConverter, 'object') |
| 1887 | |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 1888 | class NoneType_return_converter(CReturnConverter): |
| 1889 | def render(self, function, data): |
| 1890 | self.declare(data) |
| 1891 | data.return_conversion.append(''' |
| 1892 | if (_return_value != Py_None) |
| 1893 | goto exit; |
| 1894 | return_value = Py_None; |
| 1895 | Py_INCREF(Py_None); |
| 1896 | '''.strip()) |
| 1897 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 1898 | class int_return_converter(CReturnConverter): |
| 1899 | type = 'int' |
| 1900 | |
| 1901 | def render(self, function, data): |
| 1902 | self.declare(data) |
| 1903 | self.err_occurred_if("_return_value == -1", data) |
| 1904 | data.return_conversion.append( |
| 1905 | 'return_value = PyLong_FromLong((long)_return_value);\n') |
| 1906 | |
| 1907 | |
| 1908 | class long_return_converter(CReturnConverter): |
| 1909 | type = 'long' |
| 1910 | |
| 1911 | def render(self, function, data): |
| 1912 | self.declare(data) |
| 1913 | self.err_occurred_if("_return_value == -1", data) |
| 1914 | data.return_conversion.append( |
| 1915 | 'return_value = PyLong_FromLong(_return_value);\n') |
| 1916 | |
| 1917 | |
| 1918 | class Py_ssize_t_return_converter(CReturnConverter): |
| 1919 | type = 'Py_ssize_t' |
| 1920 | |
| 1921 | def render(self, function, data): |
| 1922 | self.declare(data) |
| 1923 | self.err_occurred_if("_return_value == -1", data) |
| 1924 | data.return_conversion.append( |
| 1925 | 'return_value = PyLong_FromSsize_t(_return_value);\n') |
| 1926 | |
| 1927 | |
| 1928 | class DecodeFSDefault_return_converter(CReturnConverter): |
| 1929 | type = 'char *' |
| 1930 | |
| 1931 | def render(self, function, data): |
| 1932 | self.declare(data) |
| 1933 | self.err_occurred_if_null_pointer("_return_value", data) |
| 1934 | data.return_conversion.append( |
| 1935 | 'return_value = PyUnicode_DecodeFSDefault(_return_value);\n') |
| 1936 | |
| 1937 | |
| 1938 | class IndentStack: |
| 1939 | def __init__(self): |
| 1940 | self.indents = [] |
| 1941 | self.margin = None |
| 1942 | |
| 1943 | def _ensure(self): |
| 1944 | if not self.indents: |
| 1945 | fail('IndentStack expected indents, but none are defined.') |
| 1946 | |
| 1947 | def measure(self, line): |
| 1948 | """ |
| 1949 | Returns the length of the line's margin. |
| 1950 | """ |
| 1951 | if '\t' in line: |
| 1952 | fail('Tab characters are illegal in the Clinic DSL.') |
| 1953 | stripped = line.lstrip() |
| 1954 | if not len(stripped): |
| 1955 | # we can't tell anything from an empty line |
| 1956 | # so just pretend it's indented like our current indent |
| 1957 | self._ensure() |
| 1958 | return self.indents[-1] |
| 1959 | return len(line) - len(stripped) |
| 1960 | |
| 1961 | def infer(self, line): |
| 1962 | """ |
| 1963 | Infer what is now the current margin based on this line. |
| 1964 | Returns: |
| 1965 | 1 if we have indented (or this is the first margin) |
| 1966 | 0 if the margin has not changed |
| 1967 | -N if we have dedented N times |
| 1968 | """ |
| 1969 | indent = self.measure(line) |
| 1970 | margin = ' ' * indent |
| 1971 | if not self.indents: |
| 1972 | self.indents.append(indent) |
| 1973 | self.margin = margin |
| 1974 | return 1 |
| 1975 | current = self.indents[-1] |
| 1976 | if indent == current: |
| 1977 | return 0 |
| 1978 | if indent > current: |
| 1979 | self.indents.append(indent) |
| 1980 | self.margin = margin |
| 1981 | return 1 |
| 1982 | # indent < current |
| 1983 | if indent not in self.indents: |
| 1984 | fail("Illegal outdent.") |
| 1985 | outdent_count = 0 |
| 1986 | while indent != current: |
| 1987 | self.indents.pop() |
| 1988 | current = self.indents[-1] |
| 1989 | outdent_count -= 1 |
| 1990 | self.margin = margin |
| 1991 | return outdent_count |
| 1992 | |
| 1993 | @property |
| 1994 | def depth(self): |
| 1995 | """ |
| 1996 | Returns how many margins are currently defined. |
| 1997 | """ |
| 1998 | return len(self.indents) |
| 1999 | |
| 2000 | def indent(self, line): |
| 2001 | """ |
| 2002 | Indents a line by the currently defined margin. |
| 2003 | """ |
| 2004 | return self.margin + line |
| 2005 | |
| 2006 | def dedent(self, line): |
| 2007 | """ |
| 2008 | Dedents a line by the currently defined margin. |
| 2009 | (The inverse of 'indent'.) |
| 2010 | """ |
| 2011 | margin = self.margin |
| 2012 | indent = self.indents[-1] |
| 2013 | if not line.startswith(margin): |
| 2014 | fail('Cannot dedent, line does not start with the previous margin:') |
| 2015 | return line[indent:] |
| 2016 | |
| 2017 | |
| 2018 | class DSLParser: |
| 2019 | def __init__(self, clinic): |
| 2020 | self.clinic = clinic |
| 2021 | |
| 2022 | self.directives = {} |
| 2023 | for name in dir(self): |
| 2024 | # functions that start with directive_ are added to directives |
| 2025 | _, s, key = name.partition("directive_") |
| 2026 | if s: |
| 2027 | self.directives[key] = getattr(self, name) |
| 2028 | |
| 2029 | # functions that start with at_ are too, with an @ in front |
| 2030 | _, s, key = name.partition("at_") |
| 2031 | if s: |
| 2032 | self.directives['@' + key] = getattr(self, name) |
| 2033 | |
| 2034 | self.reset() |
| 2035 | |
| 2036 | def reset(self): |
| 2037 | self.function = None |
| 2038 | self.state = self.state_dsl_start |
| 2039 | self.parameter_indent = None |
| 2040 | self.keyword_only = False |
| 2041 | self.group = 0 |
| 2042 | self.parameter_state = self.ps_start |
| 2043 | self.indent = IndentStack() |
| 2044 | self.kind = CALLABLE |
| 2045 | self.coexist = False |
| 2046 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 2047 | def directive_version(self, required): |
| 2048 | global version |
| 2049 | if version_comparitor(version, required) < 0: |
| 2050 | fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) |
| 2051 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2052 | def directive_module(self, name): |
| 2053 | fields = name.split('.') |
| 2054 | new = fields.pop() |
| 2055 | module, cls = self.clinic._module_and_class(fields) |
| 2056 | if cls: |
| 2057 | fail("Can't nest a module inside a class!") |
| 2058 | m = Module(name, module) |
| 2059 | module.modules[name] = m |
| 2060 | self.block.signatures.append(m) |
| 2061 | |
| 2062 | def directive_class(self, name): |
| 2063 | fields = name.split('.') |
| 2064 | in_classes = False |
| 2065 | parent = self |
| 2066 | name = fields.pop() |
| 2067 | so_far = [] |
| 2068 | module, cls = self.clinic._module_and_class(fields) |
| 2069 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2070 | c = Class(name, module, cls) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2071 | if cls: |
| 2072 | cls.classes[name] = c |
Larry Hastings | ed4a1c5 | 2013-11-18 09:32:13 -0800 | [diff] [blame] | 2073 | else: |
| 2074 | module.classes[name] = c |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2075 | self.block.signatures.append(c) |
| 2076 | |
| 2077 | def at_classmethod(self): |
| 2078 | assert self.kind is CALLABLE |
| 2079 | self.kind = CLASS_METHOD |
| 2080 | |
| 2081 | def at_staticmethod(self): |
| 2082 | assert self.kind is CALLABLE |
| 2083 | self.kind = STATIC_METHOD |
| 2084 | |
| 2085 | def at_coexist(self): |
| 2086 | assert self.coexist == False |
| 2087 | self.coexist = True |
| 2088 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 2089 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2090 | def parse(self, block): |
| 2091 | self.reset() |
| 2092 | self.block = block |
| 2093 | block_start = self.clinic.block_parser.line_number |
| 2094 | lines = block.input.split('\n') |
| 2095 | for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): |
| 2096 | if '\t' in line: |
| 2097 | fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) |
| 2098 | self.state(line) |
| 2099 | |
| 2100 | self.next(self.state_terminal) |
| 2101 | self.state(None) |
| 2102 | |
| 2103 | block.output = self.clinic.language.render(block.signatures) |
| 2104 | |
| 2105 | @staticmethod |
| 2106 | def ignore_line(line): |
| 2107 | # ignore comment-only lines |
| 2108 | if line.lstrip().startswith('#'): |
| 2109 | return True |
| 2110 | |
| 2111 | # Ignore empty lines too |
| 2112 | # (but not in docstring sections!) |
| 2113 | if not line.strip(): |
| 2114 | return True |
| 2115 | |
| 2116 | return False |
| 2117 | |
| 2118 | @staticmethod |
| 2119 | def calculate_indent(line): |
| 2120 | return len(line) - len(line.strip()) |
| 2121 | |
| 2122 | def next(self, state, line=None): |
| 2123 | # real_print(self.state.__name__, "->", state.__name__, ", line=", line) |
| 2124 | self.state = state |
| 2125 | if line is not None: |
| 2126 | self.state(line) |
| 2127 | |
| 2128 | def state_dsl_start(self, line): |
| 2129 | # self.block = self.ClinicOutputBlock(self) |
| 2130 | if self.ignore_line(line): |
| 2131 | return |
| 2132 | self.next(self.state_modulename_name, line) |
| 2133 | |
| 2134 | def state_modulename_name(self, line): |
| 2135 | # looking for declaration, which establishes the leftmost column |
| 2136 | # line should be |
| 2137 | # modulename.fnname [as c_basename] [-> return annotation] |
| 2138 | # square brackets denote optional syntax. |
| 2139 | # |
| 2140 | # (but we might find a directive first!) |
| 2141 | # |
| 2142 | # this line is permitted to start with whitespace. |
| 2143 | # we'll call this number of spaces F (for "function"). |
| 2144 | |
| 2145 | if not line.strip(): |
| 2146 | return |
| 2147 | |
| 2148 | self.indent.infer(line) |
| 2149 | |
| 2150 | # is it a directive? |
| 2151 | fields = shlex.split(line) |
| 2152 | directive_name = fields[0] |
| 2153 | directive = self.directives.get(directive_name, None) |
| 2154 | if directive: |
| 2155 | directive(*fields[1:]) |
| 2156 | return |
| 2157 | |
| 2158 | line, _, returns = line.partition('->') |
| 2159 | |
| 2160 | full_name, _, c_basename = line.partition(' as ') |
| 2161 | full_name = full_name.strip() |
| 2162 | c_basename = c_basename.strip() or None |
| 2163 | |
Larry Hastings | dfcd467 | 2013-10-27 02:49:39 -0700 | [diff] [blame] | 2164 | if not is_legal_py_identifier(full_name): |
| 2165 | fail("Illegal function name: {}".format(full_name)) |
| 2166 | if c_basename and not is_legal_c_identifier(c_basename): |
| 2167 | fail("Illegal C basename: {}".format(c_basename)) |
| 2168 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2169 | if not returns: |
| 2170 | return_converter = CReturnConverter() |
| 2171 | else: |
| 2172 | ast_input = "def x() -> {}: pass".format(returns) |
| 2173 | module = None |
| 2174 | try: |
| 2175 | module = ast.parse(ast_input) |
| 2176 | except SyntaxError: |
| 2177 | pass |
| 2178 | if not module: |
| 2179 | fail("Badly-formed annotation for " + full_name + ": " + returns) |
| 2180 | try: |
| 2181 | name, legacy, kwargs = self.parse_converter(module.body[0].returns) |
| 2182 | assert not legacy |
| 2183 | if name not in return_converters: |
| 2184 | fail("Error: No available return converter called " + repr(name)) |
| 2185 | return_converter = return_converters[name](**kwargs) |
| 2186 | except ValueError: |
| 2187 | fail("Badly-formed annotation for " + full_name + ": " + returns) |
| 2188 | |
| 2189 | fields = [x.strip() for x in full_name.split('.')] |
| 2190 | function_name = fields.pop() |
| 2191 | module, cls = self.clinic._module_and_class(fields) |
| 2192 | |
| 2193 | if not module: |
| 2194 | fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") |
| 2195 | self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, |
| 2196 | return_converter=return_converter, kind=self.kind, coexist=self.coexist) |
| 2197 | self.block.signatures.append(self.function) |
| 2198 | self.next(self.state_parameters_start) |
| 2199 | |
| 2200 | # Now entering the parameters section. The rules, formally stated: |
| 2201 | # |
| 2202 | # * All lines must be indented with spaces only. |
| 2203 | # * The first line must be a parameter declaration. |
| 2204 | # * The first line must be indented. |
| 2205 | # * This first line establishes the indent for parameters. |
| 2206 | # * We'll call this number of spaces P (for "parameter"). |
| 2207 | # * Thenceforth: |
| 2208 | # * Lines indented with P spaces specify a parameter. |
| 2209 | # * Lines indented with > P spaces are docstrings for the previous |
| 2210 | # parameter. |
| 2211 | # * We'll call this number of spaces D (for "docstring"). |
| 2212 | # * All subsequent lines indented with >= D spaces are stored as |
| 2213 | # part of the per-parameter docstring. |
| 2214 | # * All lines will have the first D spaces of the indent stripped |
| 2215 | # before they are stored. |
| 2216 | # * It's illegal to have a line starting with a number of spaces X |
| 2217 | # such that P < X < D. |
| 2218 | # * A line with < P spaces is the first line of the function |
| 2219 | # docstring, which ends processing for parameters and per-parameter |
| 2220 | # docstrings. |
| 2221 | # * The first line of the function docstring must be at the same |
| 2222 | # indent as the function declaration. |
| 2223 | # * It's illegal to have any line in the parameters section starting |
| 2224 | # with X spaces such that F < X < P. (As before, F is the indent |
| 2225 | # of the function declaration.) |
| 2226 | # |
| 2227 | ############## |
| 2228 | # |
| 2229 | # Also, currently Argument Clinic places the following restrictions on groups: |
| 2230 | # * Each group must contain at least one parameter. |
| 2231 | # * Each group may contain at most one group, which must be the furthest |
| 2232 | # thing in the group from the required parameters. (The nested group |
| 2233 | # must be the first in the group when it's before the required |
| 2234 | # parameters, and the last thing in the group when after the required |
| 2235 | # parameters.) |
| 2236 | # * There may be at most one (top-level) group to the left or right of |
| 2237 | # the required parameters. |
| 2238 | # * You must specify a slash, and it must be after all parameters. |
| 2239 | # (In other words: either all parameters are positional-only, |
| 2240 | # or none are.) |
| 2241 | # |
| 2242 | # Said another way: |
| 2243 | # * Each group must contain at least one parameter. |
| 2244 | # * All left square brackets before the required parameters must be |
| 2245 | # consecutive. (You can't have a left square bracket followed |
| 2246 | # by a parameter, then another left square bracket. You can't |
| 2247 | # have a left square bracket, a parameter, a right square bracket, |
| 2248 | # and then a left square bracket.) |
| 2249 | # * All right square brackets after the required parameters must be |
| 2250 | # consecutive. |
| 2251 | # |
| 2252 | # These rules are enforced with a single state variable: |
| 2253 | # "parameter_state". (Previously the code was a miasma of ifs and |
| 2254 | # separate boolean state variables.) The states are: |
| 2255 | # |
| 2256 | # [ [ a, b, ] c, ] d, e, f, [ g, h, [ i ] ] / <- line |
| 2257 | # 01 2 3 4 5 6 <- state transitions |
| 2258 | # |
| 2259 | # 0: ps_start. before we've seen anything. legal transitions are to 1 or 3. |
| 2260 | # 1: ps_left_square_before. left square brackets before required parameters. |
| 2261 | # 2: ps_group_before. in a group, before required parameters. |
| 2262 | # 3: ps_required. required parameters. (renumber left groups!) |
| 2263 | # 4: ps_group_after. in a group, after required parameters. |
| 2264 | # 5: ps_right_square_after. right square brackets after required parameters. |
| 2265 | # 6: ps_seen_slash. seen slash. |
| 2266 | ps_start, ps_left_square_before, ps_group_before, ps_required, \ |
| 2267 | ps_group_after, ps_right_square_after, ps_seen_slash = range(7) |
| 2268 | |
| 2269 | def state_parameters_start(self, line): |
| 2270 | if self.ignore_line(line): |
| 2271 | return |
| 2272 | |
| 2273 | # if this line is not indented, we have no parameters |
| 2274 | if not self.indent.infer(line): |
| 2275 | return self.next(self.state_function_docstring, line) |
| 2276 | |
| 2277 | return self.next(self.state_parameter, line) |
| 2278 | |
| 2279 | |
| 2280 | def to_required(self): |
| 2281 | """ |
| 2282 | Transition to the "required" parameter state. |
| 2283 | """ |
| 2284 | if self.parameter_state != self.ps_required: |
| 2285 | self.parameter_state = self.ps_required |
| 2286 | for p in self.function.parameters.values(): |
| 2287 | p.group = -p.group |
| 2288 | |
| 2289 | def state_parameter(self, line): |
| 2290 | if self.ignore_line(line): |
| 2291 | return |
| 2292 | |
| 2293 | assert self.indent.depth == 2 |
| 2294 | indent = self.indent.infer(line) |
| 2295 | if indent == -1: |
| 2296 | # we outdented, must be to definition column |
| 2297 | return self.next(self.state_function_docstring, line) |
| 2298 | |
| 2299 | if indent == 1: |
| 2300 | # we indented, must be to new parameter docstring column |
| 2301 | return self.next(self.state_parameter_docstring_start, line) |
| 2302 | |
| 2303 | line = line.lstrip() |
| 2304 | |
| 2305 | if line in ('*', '/', '[', ']'): |
| 2306 | self.parse_special_symbol(line) |
| 2307 | return |
| 2308 | |
| 2309 | if self.parameter_state in (self.ps_start, self.ps_required): |
| 2310 | self.to_required() |
| 2311 | elif self.parameter_state == self.ps_left_square_before: |
| 2312 | self.parameter_state = self.ps_group_before |
| 2313 | elif self.parameter_state == self.ps_group_before: |
| 2314 | if not self.group: |
| 2315 | self.to_required() |
| 2316 | elif self.parameter_state == self.ps_group_after: |
| 2317 | pass |
| 2318 | else: |
| 2319 | fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") |
| 2320 | |
| 2321 | ast_input = "def x({}): pass".format(line) |
| 2322 | module = None |
| 2323 | try: |
| 2324 | module = ast.parse(ast_input) |
| 2325 | except SyntaxError: |
| 2326 | pass |
| 2327 | if not module: |
Larry Hastings | ef3b1fb | 2013-10-22 23:26:23 -0700 | [diff] [blame] | 2328 | fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2329 | |
| 2330 | function_args = module.body[0].args |
| 2331 | parameter = function_args.args[0] |
| 2332 | |
Larry Hastings | 16c5191 | 2014-01-07 11:53:01 -0800 | [diff] [blame] | 2333 | py_default = None |
| 2334 | |
| 2335 | parameter_name = parameter.arg |
| 2336 | name, legacy, kwargs = self.parse_converter(parameter.annotation) |
| 2337 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2338 | if function_args.defaults: |
| 2339 | expr = function_args.defaults[0] |
| 2340 | # mild hack: explicitly support NULL as a default value |
| 2341 | if isinstance(expr, ast.Name) and expr.id == 'NULL': |
| 2342 | value = NULL |
Larry Hastings | 16c5191 | 2014-01-07 11:53:01 -0800 | [diff] [blame] | 2343 | elif isinstance(expr, ast.Attribute): |
| 2344 | a = [] |
| 2345 | n = expr |
| 2346 | while isinstance(n, ast.Attribute): |
| 2347 | a.append(n.attr) |
| 2348 | n = n.value |
| 2349 | if not isinstance(n, ast.Name): |
| 2350 | fail("Malformed default value (looked like a Python constant)") |
| 2351 | a.append(n.id) |
| 2352 | py_default = ".".join(reversed(a)) |
| 2353 | value = None |
| 2354 | c_default = kwargs.get("c_default") |
| 2355 | if not (isinstance(c_default, str) and c_default): |
| 2356 | fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") |
| 2357 | kwargs["py_default"] = py_default |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2358 | else: |
| 2359 | value = ast.literal_eval(expr) |
| 2360 | else: |
| 2361 | value = unspecified |
| 2362 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2363 | dict = legacy_converters if legacy else converters |
| 2364 | legacy_str = "legacy " if legacy else "" |
| 2365 | if name not in dict: |
| 2366 | fail('{} is not a valid {}converter'.format(name, legacy_str)) |
| 2367 | converter = dict[name](parameter_name, self.function, value, **kwargs) |
| 2368 | |
Larry Hastings | ebdcb50 | 2013-11-23 14:54:00 -0800 | [diff] [blame] | 2369 | # special case: if it's the self converter, |
| 2370 | # don't actually add it to the parameter list |
| 2371 | if isinstance(converter, self_converter): |
| 2372 | if self.function.parameters or (self.parameter_state != self.ps_required): |
| 2373 | fail("The 'self' parameter, if specified, must be the very first thing in the parameter block.") |
| 2374 | if self.function.self_converter: |
| 2375 | fail("You can't specify the 'self' parameter more than once.") |
| 2376 | self.function.self_converter = converter |
| 2377 | self.parameter_state = self.ps_start |
| 2378 | return |
| 2379 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2380 | kind = inspect.Parameter.KEYWORD_ONLY if self.keyword_only else inspect.Parameter.POSITIONAL_OR_KEYWORD |
| 2381 | p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) |
| 2382 | self.function.parameters[parameter_name] = p |
| 2383 | |
| 2384 | def parse_converter(self, annotation): |
| 2385 | if isinstance(annotation, ast.Str): |
| 2386 | return annotation.s, True, {} |
| 2387 | |
| 2388 | if isinstance(annotation, ast.Name): |
| 2389 | return annotation.id, False, {} |
| 2390 | |
| 2391 | assert isinstance(annotation, ast.Call) |
| 2392 | |
| 2393 | name = annotation.func.id |
| 2394 | kwargs = {node.arg: ast.literal_eval(node.value) for node in annotation.keywords} |
| 2395 | return name, False, kwargs |
| 2396 | |
| 2397 | def parse_special_symbol(self, symbol): |
| 2398 | if self.parameter_state == self.ps_seen_slash: |
| 2399 | fail("Function " + self.function.name + " specifies " + symbol + " after /, which is unsupported.") |
| 2400 | |
| 2401 | if symbol == '*': |
| 2402 | if self.keyword_only: |
| 2403 | fail("Function " + self.function.name + " uses '*' more than once.") |
| 2404 | self.keyword_only = True |
| 2405 | elif symbol == '[': |
| 2406 | if self.parameter_state in (self.ps_start, self.ps_left_square_before): |
| 2407 | self.parameter_state = self.ps_left_square_before |
| 2408 | elif self.parameter_state in (self.ps_required, self.ps_group_after): |
| 2409 | self.parameter_state = self.ps_group_after |
| 2410 | else: |
| 2411 | fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") |
| 2412 | self.group += 1 |
| 2413 | elif symbol == ']': |
| 2414 | if not self.group: |
| 2415 | fail("Function " + self.function.name + " has a ] without a matching [.") |
| 2416 | if not any(p.group == self.group for p in self.function.parameters.values()): |
| 2417 | fail("Function " + self.function.name + " has an empty group.\nAll groups must contain at least one parameter.") |
| 2418 | self.group -= 1 |
| 2419 | if self.parameter_state in (self.ps_left_square_before, self.ps_group_before): |
| 2420 | self.parameter_state = self.ps_group_before |
| 2421 | elif self.parameter_state in (self.ps_group_after, self.ps_right_square_after): |
| 2422 | self.parameter_state = self.ps_right_square_after |
| 2423 | else: |
| 2424 | fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") |
| 2425 | elif symbol == '/': |
| 2426 | # ps_required is allowed here, that allows positional-only without option groups |
| 2427 | # to work (and have default values!) |
| 2428 | if (self.parameter_state not in (self.ps_required, self.ps_right_square_after, self.ps_group_before)) or self.group: |
| 2429 | fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") |
| 2430 | if self.keyword_only: |
| 2431 | fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") |
| 2432 | self.parameter_state = self.ps_seen_slash |
| 2433 | # fixup preceeding parameters |
| 2434 | for p in self.function.parameters.values(): |
| 2435 | if p.kind != inspect.Parameter.POSITIONAL_OR_KEYWORD: |
| 2436 | fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") |
| 2437 | p.kind = inspect.Parameter.POSITIONAL_ONLY |
| 2438 | |
| 2439 | def state_parameter_docstring_start(self, line): |
| 2440 | self.parameter_docstring_indent = len(self.indent.margin) |
| 2441 | assert self.indent.depth == 3 |
| 2442 | return self.next(self.state_parameter_docstring, line) |
| 2443 | |
| 2444 | # every line of the docstring must start with at least F spaces, |
| 2445 | # where F > P. |
| 2446 | # these F spaces will be stripped. |
| 2447 | def state_parameter_docstring(self, line): |
| 2448 | stripped = line.strip() |
| 2449 | if stripped.startswith('#'): |
| 2450 | return |
| 2451 | |
| 2452 | indent = self.indent.measure(line) |
| 2453 | if indent < self.parameter_docstring_indent: |
| 2454 | self.indent.infer(line) |
| 2455 | assert self.indent.depth < 3 |
| 2456 | if self.indent.depth == 2: |
| 2457 | # back to a parameter |
| 2458 | return self.next(self.state_parameter, line) |
| 2459 | assert self.indent.depth == 1 |
| 2460 | return self.next(self.state_function_docstring, line) |
| 2461 | |
| 2462 | assert self.function.parameters |
| 2463 | last_parameter = next(reversed(list(self.function.parameters.values()))) |
| 2464 | |
| 2465 | new_docstring = last_parameter.docstring |
| 2466 | |
| 2467 | if new_docstring: |
| 2468 | new_docstring += '\n' |
| 2469 | if stripped: |
| 2470 | new_docstring += self.indent.dedent(line) |
| 2471 | |
| 2472 | last_parameter.docstring = new_docstring |
| 2473 | |
| 2474 | # the final stanza of the DSL is the docstring. |
| 2475 | def state_function_docstring(self, line): |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2476 | if self.group: |
| 2477 | fail("Function " + self.function.name + " has a ] without a matching [.") |
| 2478 | |
| 2479 | stripped = line.strip() |
| 2480 | if stripped.startswith('#'): |
| 2481 | return |
| 2482 | |
| 2483 | new_docstring = self.function.docstring |
| 2484 | if new_docstring: |
| 2485 | new_docstring += "\n" |
| 2486 | if stripped: |
| 2487 | line = self.indent.dedent(line).rstrip() |
| 2488 | else: |
| 2489 | line = '' |
| 2490 | new_docstring += line |
| 2491 | self.function.docstring = new_docstring |
| 2492 | |
| 2493 | def format_docstring(self): |
| 2494 | f = self.function |
| 2495 | |
| 2496 | add, output = text_accumulator() |
| 2497 | parameters = list(f.parameters.values()) |
| 2498 | |
| 2499 | ## |
| 2500 | ## docstring first line |
| 2501 | ## |
| 2502 | |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2503 | add(f.name) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2504 | add('(') |
| 2505 | |
| 2506 | # populate "right_bracket_count" field for every parameter |
| 2507 | if parameters: |
| 2508 | # for now, the only way Clinic supports positional-only parameters |
| 2509 | # is if all of them are positional-only. |
| 2510 | positional_only_parameters = [p.kind == inspect.Parameter.POSITIONAL_ONLY for p in parameters] |
| 2511 | if parameters[0].kind == inspect.Parameter.POSITIONAL_ONLY: |
| 2512 | assert all(positional_only_parameters) |
| 2513 | for p in parameters: |
| 2514 | p.right_bracket_count = abs(p.group) |
| 2515 | else: |
| 2516 | # don't put any right brackets around non-positional-only parameters, ever. |
| 2517 | for p in parameters: |
| 2518 | p.right_bracket_count = 0 |
| 2519 | |
| 2520 | right_bracket_count = 0 |
| 2521 | |
| 2522 | def fix_right_bracket_count(desired): |
| 2523 | nonlocal right_bracket_count |
| 2524 | s = '' |
| 2525 | while right_bracket_count < desired: |
| 2526 | s += '[' |
| 2527 | right_bracket_count += 1 |
| 2528 | while right_bracket_count > desired: |
| 2529 | s += ']' |
| 2530 | right_bracket_count -= 1 |
| 2531 | return s |
| 2532 | |
| 2533 | added_star = False |
| 2534 | add_comma = False |
| 2535 | |
| 2536 | for p in parameters: |
| 2537 | assert p.name |
| 2538 | |
| 2539 | if p.is_keyword_only() and not added_star: |
| 2540 | added_star = True |
| 2541 | if add_comma: |
| 2542 | add(', ') |
| 2543 | add('*') |
| 2544 | |
| 2545 | a = [p.name] |
| 2546 | if p.converter.is_optional(): |
| 2547 | a.append('=') |
| 2548 | value = p.converter.default |
| 2549 | a.append(p.converter.doc_default) |
| 2550 | s = fix_right_bracket_count(p.right_bracket_count) |
| 2551 | s += "".join(a) |
| 2552 | if add_comma: |
| 2553 | add(', ') |
| 2554 | add(s) |
| 2555 | add_comma = True |
| 2556 | |
| 2557 | add(fix_right_bracket_count(0)) |
| 2558 | add(')') |
| 2559 | |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2560 | # if f.return_converter.doc_default: |
| 2561 | # add(' -> ') |
| 2562 | # add(f.return_converter.doc_default) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2563 | |
| 2564 | docstring_first_line = output() |
| 2565 | |
| 2566 | # now fix up the places where the brackets look wrong |
| 2567 | docstring_first_line = docstring_first_line.replace(', ]', ',] ') |
| 2568 | |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2569 | # okay. now we're officially building the "parameters" section. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2570 | # create substitution text for {parameters} |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2571 | spacer_line = False |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2572 | for p in parameters: |
| 2573 | if not p.docstring.strip(): |
| 2574 | continue |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2575 | if spacer_line: |
| 2576 | add('\n') |
| 2577 | else: |
| 2578 | spacer_line = True |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2579 | add(" ") |
| 2580 | add(p.name) |
| 2581 | add('\n') |
| 2582 | add(textwrap.indent(rstrip_lines(p.docstring.rstrip()), " ")) |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2583 | parameters = output() |
| 2584 | if parameters: |
| 2585 | parameters += '\n' |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2586 | |
| 2587 | ## |
| 2588 | ## docstring body |
| 2589 | ## |
| 2590 | |
| 2591 | docstring = f.docstring.rstrip() |
| 2592 | lines = [line.rstrip() for line in docstring.split('\n')] |
| 2593 | |
| 2594 | # Enforce the summary line! |
| 2595 | # The first line of a docstring should be a summary of the function. |
| 2596 | # It should fit on one line (80 columns? 79 maybe?) and be a paragraph |
| 2597 | # by itself. |
| 2598 | # |
| 2599 | # Argument Clinic enforces the following rule: |
| 2600 | # * either the docstring is empty, |
| 2601 | # * or it must have a summary line. |
| 2602 | # |
| 2603 | # Guido said Clinic should enforce this: |
| 2604 | # http://mail.python.org/pipermail/python-dev/2013-June/127110.html |
| 2605 | |
| 2606 | if len(lines) >= 2: |
| 2607 | if lines[1]: |
| 2608 | fail("Docstring for " + f.full_name + " does not have a summary line!\n" + |
| 2609 | "Every non-blank function docstring must start with\n" + |
| 2610 | "a single line summary followed by an empty line.") |
| 2611 | elif len(lines) == 1: |
| 2612 | # the docstring is only one line right now--the summary line. |
| 2613 | # add an empty line after the summary line so we have space |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2614 | # between it and the {parameters} we're about to add. |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2615 | lines.append('') |
| 2616 | |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2617 | parameters_marker_count = len(docstring.split('{parameters}')) - 1 |
| 2618 | if parameters_marker_count > 1: |
| 2619 | fail('You may not specify {parameters} more than once in a docstring!') |
| 2620 | |
| 2621 | if not parameters_marker_count: |
| 2622 | # insert after summary line |
| 2623 | lines.insert(2, '{parameters}') |
| 2624 | |
| 2625 | # insert at front of docstring |
| 2626 | lines.insert(0, docstring_first_line) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2627 | |
| 2628 | docstring = "\n".join(lines) |
| 2629 | |
| 2630 | add(docstring) |
| 2631 | docstring = output() |
| 2632 | |
Larry Hastings | 44e2eaa | 2013-11-23 15:37:55 -0800 | [diff] [blame] | 2633 | docstring = linear_format(docstring, parameters=parameters) |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2634 | docstring = docstring.rstrip() |
| 2635 | |
| 2636 | return docstring |
| 2637 | |
| 2638 | def state_terminal(self, line): |
| 2639 | """ |
| 2640 | Called when processing the block is done. |
| 2641 | """ |
| 2642 | assert not line |
| 2643 | |
| 2644 | if not self.function: |
| 2645 | return |
| 2646 | |
Larry Hastings | 6d2ea21 | 2014-01-05 02:50:45 -0800 | [diff] [blame] | 2647 | if not self.function.self_converter: |
| 2648 | self.function.self_converter = self_converter("self", self.function) |
| 2649 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2650 | if self.keyword_only: |
| 2651 | values = self.function.parameters.values() |
| 2652 | if not values: |
| 2653 | no_parameter_after_star = True |
| 2654 | else: |
| 2655 | last_parameter = next(reversed(list(values))) |
| 2656 | no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY |
| 2657 | if no_parameter_after_star: |
| 2658 | fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") |
| 2659 | |
| 2660 | # remove trailing whitespace from all parameter docstrings |
| 2661 | for name, value in self.function.parameters.items(): |
| 2662 | if not value: |
| 2663 | continue |
| 2664 | value.docstring = value.docstring.rstrip() |
| 2665 | |
| 2666 | self.function.docstring = self.format_docstring() |
| 2667 | |
| 2668 | |
| 2669 | # maps strings to callables. |
| 2670 | # the callable should return an object |
| 2671 | # that implements the clinic parser |
| 2672 | # interface (__init__ and parse). |
| 2673 | # |
| 2674 | # example parsers: |
| 2675 | # "clinic", handles the Clinic DSL |
| 2676 | # "python", handles running Python code |
| 2677 | # |
| 2678 | parsers = {'clinic' : DSLParser, 'python': PythonParser} |
| 2679 | |
| 2680 | |
| 2681 | clinic = None |
| 2682 | |
| 2683 | |
| 2684 | def main(argv): |
| 2685 | import sys |
| 2686 | |
| 2687 | if sys.version_info.major < 3 or sys.version_info.minor < 3: |
| 2688 | sys.exit("Error: clinic.py requires Python 3.3 or greater.") |
| 2689 | |
| 2690 | import argparse |
| 2691 | cmdline = argparse.ArgumentParser() |
| 2692 | cmdline.add_argument("-f", "--force", action='store_true') |
| 2693 | cmdline.add_argument("-o", "--output", type=str) |
| 2694 | cmdline.add_argument("--converters", action='store_true') |
Larry Hastings | dcd340e | 2013-11-23 14:58:45 -0800 | [diff] [blame] | 2695 | cmdline.add_argument("--make", action='store_true') |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2696 | cmdline.add_argument("filename", type=str, nargs="*") |
| 2697 | ns = cmdline.parse_args(argv) |
| 2698 | |
| 2699 | if ns.converters: |
| 2700 | if ns.filename: |
| 2701 | print("Usage error: can't specify --converters and a filename at the same time.") |
| 2702 | print() |
| 2703 | cmdline.print_usage() |
| 2704 | sys.exit(-1) |
| 2705 | converters = [] |
| 2706 | return_converters = [] |
| 2707 | ignored = set(""" |
| 2708 | add_c_converter |
| 2709 | add_c_return_converter |
| 2710 | add_default_legacy_c_converter |
| 2711 | add_legacy_c_converter |
| 2712 | """.strip().split()) |
| 2713 | module = globals() |
| 2714 | for name in module: |
| 2715 | for suffix, ids in ( |
| 2716 | ("_return_converter", return_converters), |
| 2717 | ("_converter", converters), |
| 2718 | ): |
| 2719 | if name in ignored: |
| 2720 | continue |
| 2721 | if name.endswith(suffix): |
| 2722 | ids.append((name, name[:-len(suffix)])) |
| 2723 | break |
| 2724 | print() |
| 2725 | |
| 2726 | print("Legacy converters:") |
| 2727 | legacy = sorted(legacy_converters) |
| 2728 | print(' ' + ' '.join(c for c in legacy if c[0].isupper())) |
| 2729 | print(' ' + ' '.join(c for c in legacy if c[0].islower())) |
| 2730 | print() |
| 2731 | |
| 2732 | for title, attribute, ids in ( |
| 2733 | ("Converters", 'converter_init', converters), |
| 2734 | ("Return converters", 'return_converter_init', return_converters), |
| 2735 | ): |
| 2736 | print(title + ":") |
| 2737 | longest = -1 |
| 2738 | for name, short_name in ids: |
| 2739 | longest = max(longest, len(short_name)) |
| 2740 | for name, short_name in sorted(ids, key=lambda x: x[1].lower()): |
| 2741 | cls = module[name] |
| 2742 | callable = getattr(cls, attribute, None) |
| 2743 | if not callable: |
| 2744 | continue |
| 2745 | signature = inspect.signature(callable) |
| 2746 | parameters = [] |
| 2747 | for parameter_name, parameter in signature.parameters.items(): |
| 2748 | if parameter.kind == inspect.Parameter.KEYWORD_ONLY: |
| 2749 | if parameter.default != inspect.Parameter.empty: |
| 2750 | s = '{}={!r}'.format(parameter_name, parameter.default) |
| 2751 | else: |
| 2752 | s = parameter_name |
| 2753 | parameters.append(s) |
| 2754 | print(' {}({})'.format(short_name, ', '.join(parameters))) |
| 2755 | # add_comma = False |
| 2756 | # for parameter_name, parameter in signature.parameters.items(): |
| 2757 | # if parameter.kind == inspect.Parameter.KEYWORD_ONLY: |
| 2758 | # if add_comma: |
| 2759 | # parameters.append(', ') |
| 2760 | # else: |
| 2761 | # add_comma = True |
| 2762 | # s = parameter_name |
| 2763 | # if parameter.default != inspect.Parameter.empty: |
| 2764 | # s += '=' + repr(parameter.default) |
| 2765 | # parameters.append(s) |
| 2766 | # parameters.append(')') |
| 2767 | |
| 2768 | # print(" ", short_name + "".join(parameters)) |
| 2769 | print() |
Larry Hastings | 78cf85c | 2014-01-04 12:44:57 -0800 | [diff] [blame] | 2770 | print("All converters also accept (doc_default=None, required=False, annotation=None).") |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2771 | print("All return converters also accept (doc_default=None).") |
| 2772 | sys.exit(0) |
| 2773 | |
Larry Hastings | dcd340e | 2013-11-23 14:58:45 -0800 | [diff] [blame] | 2774 | if ns.make: |
| 2775 | if ns.output or ns.filename: |
| 2776 | print("Usage error: can't use -o or filenames with --make.") |
| 2777 | print() |
| 2778 | cmdline.print_usage() |
| 2779 | sys.exit(-1) |
| 2780 | for root, dirs, files in os.walk('.'): |
| 2781 | for rcs_dir in ('.svn', '.git', '.hg'): |
| 2782 | if rcs_dir in dirs: |
| 2783 | dirs.remove(rcs_dir) |
| 2784 | for filename in files: |
| 2785 | if not filename.endswith('.c'): |
| 2786 | continue |
| 2787 | path = os.path.join(root, filename) |
| 2788 | parse_file(path, verify=not ns.force) |
| 2789 | return |
| 2790 | |
Larry Hastings | 3182680 | 2013-10-19 00:09:25 -0700 | [diff] [blame] | 2791 | if not ns.filename: |
| 2792 | cmdline.print_usage() |
| 2793 | sys.exit(-1) |
| 2794 | |
| 2795 | if ns.output and len(ns.filename) > 1: |
| 2796 | print("Usage error: can't use -o with multiple filenames.") |
| 2797 | print() |
| 2798 | cmdline.print_usage() |
| 2799 | sys.exit(-1) |
| 2800 | |
| 2801 | for filename in ns.filename: |
| 2802 | parse_file(filename, output=ns.output, verify=not ns.force) |
| 2803 | |
| 2804 | |
| 2805 | if __name__ == "__main__": |
| 2806 | sys.exit(main(sys.argv[1:])) |