Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # A tool to parse ASTMatchers.h and update the documentation in |
| 3 | # ../LibASTMatchersReference.html automatically. Run from the |
| 4 | # directory in which this file is located to update the docs. |
| 5 | |
| 6 | import collections |
| 7 | import re |
| 8 | import urllib2 |
| 9 | |
| 10 | MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h' |
| 11 | |
| 12 | # Each matcher is documented in one row of the form: |
| 13 | # result | name | argA |
| 14 | # The subsequent row contains the documentation and is hidden by default, |
| 15 | # becoming visible via javascript when the user clicks the matcher name. |
| 16 | TD_TEMPLATE=""" |
Manuel Klimek | 67619ff | 2012-09-07 13:10:32 +0000 | [diff] [blame] | 17 | <tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr> |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 18 | <tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr> |
| 19 | """ |
| 20 | |
| 21 | # We categorize the matchers into these three categories in the reference: |
| 22 | node_matchers = {} |
| 23 | narrowing_matchers = {} |
| 24 | traversal_matchers = {} |
| 25 | |
| 26 | # We output multiple rows per matcher if the matcher can be used on multiple |
| 27 | # node types. Thus, we need a new id per row to control the documentation |
| 28 | # pop-up. ids[name] keeps track of those ids. |
| 29 | ids = collections.defaultdict(int) |
| 30 | |
| 31 | # Cache for doxygen urls we have already verified. |
| 32 | doxygen_probes = {} |
| 33 | |
| 34 | def esc(text): |
| 35 | """Escape any html in the given text.""" |
| 36 | text = re.sub(r'&', '&', text) |
| 37 | text = re.sub(r'<', '<', text) |
| 38 | text = re.sub(r'>', '>', text) |
| 39 | def link_if_exists(m): |
| 40 | name = m.group(1) |
| 41 | url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name |
| 42 | if url not in doxygen_probes: |
| 43 | try: |
| 44 | print 'Probing %s...' % url |
| 45 | urllib2.urlopen(url) |
| 46 | doxygen_probes[url] = True |
| 47 | except: |
| 48 | doxygen_probes[url] = False |
| 49 | if doxygen_probes[url]: |
| 50 | return r'Matcher<<a href="%s">%s</a>>' % (url, name) |
| 51 | else: |
| 52 | return m.group(0) |
| 53 | text = re.sub( |
| 54 | r'Matcher<([^\*&]+)>', link_if_exists, text) |
| 55 | return text |
| 56 | |
| 57 | def extract_result_types(comment): |
| 58 | """Extracts a list of result types from the given comment. |
| 59 | |
| 60 | We allow annotations in the comment of the matcher to specify what |
| 61 | nodes a matcher can match on. Those comments have the form: |
| 62 | Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]]) |
| 63 | |
| 64 | Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...]. |
| 65 | Returns the empty list if no 'Usable as' specification could be |
| 66 | parsed. |
| 67 | """ |
| 68 | result_types = [] |
| 69 | m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S) |
| 70 | if m: |
| 71 | return ['*'] |
| 72 | while True: |
| 73 | m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S) |
| 74 | if not m: |
| 75 | if re.search(r'Usable as:\s*$', comment): |
| 76 | return result_types |
| 77 | else: |
| 78 | return None |
| 79 | result_types += [m.group(2)] |
| 80 | comment = m.group(1) |
| 81 | |
| 82 | def strip_doxygen(comment): |
| 83 | """Returns the given comment without \-escaped words.""" |
| 84 | # If there is only a doxygen keyword in the line, delete the whole line. |
| 85 | comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M) |
| 86 | # Delete the doxygen command and the following whitespace. |
| 87 | comment = re.sub(r'\\[^\s]+\s+', r'', comment) |
| 88 | return comment |
| 89 | |
| 90 | def unify_arguments(args): |
| 91 | """Gets rid of anything the user doesn't care about in the argument list.""" |
| 92 | args = re.sub(r'internal::', r'', args) |
| 93 | args = re.sub(r'const\s+', r'', args) |
| 94 | args = re.sub(r'&', r' ', args) |
| 95 | args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args) |
| 96 | return args |
| 97 | |
| 98 | def add_matcher(result_type, name, args, comment, is_dyncast=False): |
| 99 | """Adds a matcher to one of our categories.""" |
| 100 | if name == 'id': |
| 101 | # FIXME: Figure out whether we want to support the 'id' matcher. |
| 102 | return |
| 103 | matcher_id = '%s%d' % (name, ids[name]) |
| 104 | ids[name] += 1 |
| 105 | args = unify_arguments(args) |
| 106 | matcher_html = TD_TEMPLATE % { |
| 107 | 'result': esc('Matcher<%s>' % result_type), |
| 108 | 'name': name, |
| 109 | 'args': esc(args), |
| 110 | 'comment': esc(strip_doxygen(comment)), |
| 111 | 'id': matcher_id, |
| 112 | } |
| 113 | if is_dyncast: |
| 114 | node_matchers[result_type + name] = matcher_html |
| 115 | # Use a heuristic to figure out whether a matcher is a narrowing or |
| 116 | # traversal matcher. By default, matchers that take other matchers as |
| 117 | # arguments (and are not node matchers) do traversal. We specifically |
| 118 | # exclude known narrowing matchers that also take other matchers as |
| 119 | # arguments. |
| 120 | elif ('Matcher<' not in args or |
| 121 | name in ['allOf', 'anyOf', 'anything', 'unless']): |
| 122 | narrowing_matchers[result_type + name] = matcher_html |
| 123 | else: |
| 124 | traversal_matchers[result_type + name] = matcher_html |
| 125 | |
| 126 | def act_on_decl(declaration, comment, allowed_types): |
| 127 | """Parse the matcher out of the given declaration and comment. |
| 128 | |
| 129 | If 'allowed_types' is set, it contains a list of node types the matcher |
| 130 | can match on, as extracted from the static type asserts in the matcher |
| 131 | definition. |
| 132 | """ |
| 133 | if declaration.strip(): |
| 134 | # Node matchers are defined by writing: |
| 135 | # VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name; |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 136 | m = re.match(r""".*Variadic(?:DynCast)?AllOfMatcher\s*< |
| 137 | \s*([^\s,]+)\s*(?:, |
| 138 | \s*([^\s>]+)\s*)?> |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 139 | \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X) |
| 140 | if m: |
| 141 | result, inner, name = m.groups() |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 142 | if not inner: |
| 143 | inner = result |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 144 | add_matcher(result, name, 'Matcher<%s>...' % inner, |
| 145 | comment, is_dyncast=True) |
| 146 | return |
| 147 | |
| 148 | # Parse the various matcher definition macros. |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 149 | m = re.match(""".*AST_TYPE_MATCHER\( |
| 150 | \s*([^\s,]+\s*), |
| 151 | \s*([^\s,]+\s*) |
| 152 | \)\s*;\s*$""", declaration, flags=re.X) |
| 153 | if m: |
| 154 | inner, name = m.groups() |
| 155 | add_matcher('Type', name, 'Matcher<%s>...' % inner, |
| 156 | comment, is_dyncast=True) |
Manuel Klimek | 2c4b2e4 | 2013-07-25 06:05:50 +0000 | [diff] [blame] | 157 | # FIXME: re-enable once we have implemented casting on the TypeLoc |
| 158 | # hierarchy. |
| 159 | # add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner, |
| 160 | # comment, is_dyncast=True) |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 161 | return |
| 162 | |
| 163 | m = re.match(""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER\( |
| 164 | \s*([^\s,]+\s*), |
Samuel Benzaquen | 3f84bb3 | 2013-07-15 19:25:06 +0000 | [diff] [blame] | 165 | \s*(?:[^\s,]+\s*), |
| 166 | \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\) |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 167 | \)\s*;\s*$""", declaration, flags=re.X) |
| 168 | if m: |
Samuel Benzaquen | 3f84bb3 | 2013-07-15 19:25:06 +0000 | [diff] [blame] | 169 | loc, name, n_results, results = m.groups()[0:4] |
| 170 | result_types = [r.strip() for r in results.split(',')] |
| 171 | |
| 172 | comment_result_types = extract_result_types(comment) |
| 173 | if (comment_result_types and |
| 174 | sorted(result_types) != sorted(comment_result_types)): |
| 175 | raise Exception('Inconsistent documentation for: %s' % name) |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 176 | for result_type in result_types: |
| 177 | add_matcher(result_type, name, 'Matcher<Type>', comment) |
| 178 | if loc: |
| 179 | add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>', |
| 180 | comment) |
| 181 | return |
| 182 | |
Samuel Benzaquen | ef7eb02 | 2013-06-21 15:51:31 +0000 | [diff] [blame] | 183 | m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\( |
| 184 | \s*([^\s,]+)\s*, |
| 185 | \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\) |
| 186 | (?:,\s*([^\s,]+)\s* |
| 187 | ,\s*([^\s,]+)\s*)? |
| 188 | (?:,\s*([^\s,]+)\s* |
| 189 | ,\s*([^\s,]+)\s*)? |
| 190 | (?:,\s*\d+\s*)? |
| 191 | \)\s*{\s*$""", declaration, flags=re.X) |
| 192 | |
| 193 | if m: |
| 194 | p, n, name, n_results, results = m.groups()[0:5] |
| 195 | args = m.groups()[5:] |
| 196 | result_types = [r.strip() for r in results.split(',')] |
| 197 | if allowed_types and allowed_types != result_types: |
| 198 | raise Exception('Inconsistent documentation for: %s' % name) |
| 199 | if n not in ['', '2']: |
| 200 | raise Exception('Cannot parse "%s"' % declaration) |
| 201 | args = ', '.join('%s %s' % (args[i], args[i+1]) |
| 202 | for i in range(0, len(args), 2) if args[i]) |
| 203 | for result_type in result_types: |
| 204 | add_matcher(result_type, name, args, comment) |
| 205 | return |
| 206 | |
| 207 | m = re.match(r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\( |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 208 | (?:\s*([^\s,]+)\s*,)? |
| 209 | \s*([^\s,]+)\s* |
| 210 | (?:,\s*([^\s,]+)\s* |
| 211 | ,\s*([^\s,]+)\s*)? |
| 212 | (?:,\s*([^\s,]+)\s* |
| 213 | ,\s*([^\s,]+)\s*)? |
Manuel Klimek | 415514d | 2013-02-06 20:36:22 +0000 | [diff] [blame] | 214 | (?:,\s*\d+\s*)? |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 215 | \)\s*{\s*$""", declaration, flags=re.X) |
| 216 | if m: |
Samuel Benzaquen | ef7eb02 | 2013-06-21 15:51:31 +0000 | [diff] [blame] | 217 | p, n, result, name = m.groups()[0:4] |
| 218 | args = m.groups()[4:] |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 219 | if not result: |
| 220 | if not allowed_types: |
| 221 | raise Exception('Did not find allowed result types for: %s' % name) |
| 222 | result_types = allowed_types |
| 223 | else: |
| 224 | result_types = [result] |
| 225 | if n not in ['', '2']: |
| 226 | raise Exception('Cannot parse "%s"' % declaration) |
| 227 | args = ', '.join('%s %s' % (args[i], args[i+1]) |
| 228 | for i in range(0, len(args), 2) if args[i]) |
| 229 | for result_type in result_types: |
| 230 | add_matcher(result_type, name, args, comment) |
| 231 | return |
| 232 | |
Samuel Benzaquen | ee0da95 | 2013-08-16 16:19:42 +0000 | [diff] [blame] | 233 | # Parse ArgumentAdapting matchers. |
| 234 | m = re.match( |
| 235 | r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*([a-zA-Z]*)\s*=\s*{};$""", |
| 236 | declaration, flags=re.X) |
| 237 | if m: |
| 238 | name = m.groups()[0] |
| 239 | add_matcher('*', name, 'Matcher<*>', comment) |
| 240 | return |
| 241 | |
Samuel Benzaquen | d36e463 | 2013-08-27 15:11:16 +0000 | [diff] [blame] | 242 | # Parse Variadic operator matchers. |
| 243 | m = re.match( |
| 244 | r"""^.*VariadicOperatorMatcherFunc\s*([a-zA-Z]*)\s*=\s*{.*};$""", |
| 245 | declaration, flags=re.X) |
| 246 | if m: |
| 247 | name = m.groups()[0] |
| 248 | add_matcher('*', name, 'Matcher<*>, ..., Matcher<*>', comment) |
| 249 | return |
| 250 | |
Samuel Benzaquen | ee0da95 | 2013-08-16 16:19:42 +0000 | [diff] [blame] | 251 | |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 252 | # Parse free standing matcher functions, like: |
| 253 | # Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) { |
| 254 | m = re.match(r"""^\s*(.*)\s+ |
| 255 | ([^\s\(]+)\s*\( |
| 256 | (.*) |
| 257 | \)\s*{""", declaration, re.X) |
| 258 | if m: |
| 259 | result, name, args = m.groups() |
| 260 | args = ', '.join(p.strip() for p in args.split(',')) |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 261 | m = re.match(r'.*\s+internal::(Bindable)?Matcher<([^>]+)>$', result) |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 262 | if m: |
Manuel Klimek | 41df16e | 2013-01-09 09:38:21 +0000 | [diff] [blame] | 263 | result_types = [m.group(2)] |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 264 | else: |
| 265 | result_types = extract_result_types(comment) |
| 266 | if not result_types: |
| 267 | if not comment: |
| 268 | # Only overloads don't have their own doxygen comments; ignore those. |
| 269 | print 'Ignoring "%s"' % name |
| 270 | else: |
| 271 | print 'Cannot determine result type for "%s"' % name |
| 272 | else: |
| 273 | for result_type in result_types: |
| 274 | add_matcher(result_type, name, args, comment) |
| 275 | else: |
| 276 | print '*** Unparsable: "' + declaration + '" ***' |
| 277 | |
| 278 | def sort_table(matcher_type, matcher_map): |
| 279 | """Returns the sorted html table for the given row map.""" |
| 280 | table = '' |
| 281 | for key in sorted(matcher_map.keys()): |
| 282 | table += matcher_map[key] + '\n' |
| 283 | return ('<!-- START_%(type)s_MATCHERS -->\n' + |
| 284 | '%(table)s' + |
| 285 | '<!--END_%(type)s_MATCHERS -->') % { |
| 286 | 'type': matcher_type, |
| 287 | 'table': table, |
| 288 | } |
| 289 | |
| 290 | # Parse the ast matchers. |
| 291 | # We alternate between two modes: |
| 292 | # body = True: We parse the definition of a matcher. We need |
| 293 | # to parse the full definition before adding a matcher, as the |
| 294 | # definition might contain static asserts that specify the result |
| 295 | # type. |
| 296 | # body = False: We parse the comments and declaration of the matcher. |
| 297 | comment = '' |
| 298 | declaration = '' |
| 299 | allowed_types = [] |
| 300 | body = False |
| 301 | for line in open(MATCHERS_FILE).read().splitlines(): |
| 302 | if body: |
| 303 | if line.strip() and line[0] == '}': |
| 304 | if declaration: |
| 305 | act_on_decl(declaration, comment, allowed_types) |
| 306 | comment = '' |
| 307 | declaration = '' |
| 308 | allowed_types = [] |
| 309 | body = False |
| 310 | else: |
| 311 | m = re.search(r'is_base_of<([^,]+), NodeType>', line) |
| 312 | if m and m.group(1): |
| 313 | allowed_types += [m.group(1)] |
| 314 | continue |
| 315 | if line.strip() and line.lstrip()[0] == '/': |
| 316 | comment += re.sub(r'/+\s?', '', line) + '\n' |
| 317 | else: |
| 318 | declaration += ' ' + line |
| 319 | if ((not line.strip()) or |
| 320 | line.rstrip()[-1] == ';' or |
Samuel Benzaquen | d36e463 | 2013-08-27 15:11:16 +0000 | [diff] [blame] | 321 | (line.rstrip()[-1] == '{' and line.rstrip()[-3:] != '= {')): |
Manuel Klimek | 95a58d2 | 2012-08-27 18:49:12 +0000 | [diff] [blame] | 322 | if line.strip() and line.rstrip()[-1] == '{': |
| 323 | body = True |
| 324 | else: |
| 325 | act_on_decl(declaration, comment, allowed_types) |
| 326 | comment = '' |
| 327 | declaration = '' |
| 328 | allowed_types = [] |
| 329 | |
| 330 | node_matcher_table = sort_table('DECL', node_matchers) |
| 331 | narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers) |
| 332 | traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers) |
| 333 | |
| 334 | reference = open('../LibASTMatchersReference.html').read() |
| 335 | reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->', |
| 336 | '%s', reference, flags=re.S) % node_matcher_table |
| 337 | reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->', |
| 338 | '%s', reference, flags=re.S) % narrowing_matcher_table |
| 339 | reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->', |
| 340 | '%s', reference, flags=re.S) % traversal_matcher_table |
| 341 | |
| 342 | with open('../LibASTMatchersReference.html', 'w') as output: |
| 343 | output.write(reference) |
| 344 | |