bpo-36876: Re-organize the c-analyzer tool code. (gh-16841)
This is partly a cleanup of the code. It also is preparation for getting the variables from the source (cross-platform) rather than from the symbols.
The change only touches the tool (and its tests).
diff --git a/Tools/c-analyzer/c-globals.py b/Tools/c-analyzer/c-globals.py
index 9afe059..b36b791 100644
--- a/Tools/c-analyzer/c-globals.py
+++ b/Tools/c-analyzer/c-globals.py
@@ -1,6 +1,6 @@
# This is a script equivalent of running "python -m test.test_c_globals.cg".
-from c_globals.__main__ import parse_args, main
+from cpython.__main__ import parse_args, main
# This is effectively copied from cg/__main__.py:
diff --git a/Tools/c-analyzer/c_parser/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py
similarity index 100%
rename from Tools/c-analyzer/c_parser/__init__.py
rename to Tools/c-analyzer/c_analyzer/__init__.py
diff --git a/Tools/c-analyzer/c_globals/__init__.py b/Tools/c-analyzer/c_analyzer/common/__init__.py
similarity index 100%
rename from Tools/c-analyzer/c_globals/__init__.py
rename to Tools/c-analyzer/c_analyzer/common/__init__.py
diff --git a/Tools/c-analyzer/c_analyzer_common/files.py b/Tools/c-analyzer/c_analyzer/common/files.py
similarity index 82%
rename from Tools/c-analyzer/c_analyzer_common/files.py
rename to Tools/c-analyzer/c_analyzer/common/files.py
index b3cd16c..ab551a8 100644
--- a/Tools/c-analyzer/c_analyzer_common/files.py
+++ b/Tools/c-analyzer/c_analyzer/common/files.py
@@ -2,7 +2,10 @@
import os
import os.path
-from . import SOURCE_DIRS, REPO_ROOT
+# XXX need tests:
+# * walk_tree()
+# * glob_tree()
+# * iter_files_by_suffix()
C_SOURCE_SUFFIXES = ('.c', '.h')
@@ -115,24 +118,3 @@
# XXX Ignore repeated suffixes?
for suffix in suffixes:
yield from _iter_files(root, suffix, relparent)
-
-
-def iter_cpython_files(*,
- walk=walk_tree,
- _files=iter_files_by_suffix,
- ):
- """Yield each file in the tree for each of the given directory names."""
- excludedtrees = [
- os.path.join('Include', 'cpython', ''),
- ]
- def is_excluded(filename):
- for root in excludedtrees:
- if filename.startswith(root):
- return True
- return False
- for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT,
- walk=walk,
- ):
- if is_excluded(filename):
- continue
- yield filename
diff --git a/Tools/c-analyzer/c_analyzer/common/info.py b/Tools/c-analyzer/c_analyzer/common/info.py
new file mode 100644
index 0000000..3f3f8c5
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/common/info.py
@@ -0,0 +1,138 @@
+from collections import namedtuple
+import re
+
+from .util import classonly, _NTBase
+
+# XXX need tests:
+# * ID.match()
+
+
+UNKNOWN = '???'
+
+NAME_RE = re.compile(r'^([a-zA-Z]|_\w*[a-zA-Z]\w*|[a-zA-Z]\w*)$')
+
+
+class ID(_NTBase, namedtuple('ID', 'filename funcname name')):
+ """A unique ID for a single symbol or declaration."""
+
+ __slots__ = ()
+ # XXX Add optional conditions (tuple of strings) field.
+ #conditions = Slot()
+
+ @classonly
+ def from_raw(cls, raw):
+ if not raw:
+ return None
+ if isinstance(raw, str):
+ return cls(None, None, raw)
+ try:
+ name, = raw
+ filename = None
+ except ValueError:
+ try:
+ filename, name = raw
+ except ValueError:
+ return super().from_raw(raw)
+ return cls(filename, None, name)
+
+ def __new__(cls, filename, funcname, name):
+ self = super().__new__(
+ cls,
+ filename=str(filename) if filename else None,
+ funcname=str(funcname) if funcname else None,
+ name=str(name) if name else None,
+ )
+ #cls.conditions.set(self, tuple(str(s) if s else None
+ # for s in conditions or ()))
+ return self
+
+ def validate(self):
+ """Fail if the object is invalid (i.e. init with bad data)."""
+ if not self.name:
+ raise TypeError('missing name')
+ else:
+ if not NAME_RE.match(self.name):
+ raise ValueError(
+ f'name must be an identifier, got {self.name!r}')
+
+ # Symbols from a binary might not have filename/funcname info.
+
+ if self.funcname:
+ if not self.filename:
+ raise TypeError('missing filename')
+ if not NAME_RE.match(self.funcname) and self.funcname != UNKNOWN:
+ raise ValueError(
+ f'name must be an identifier, got {self.funcname!r}')
+
+ # XXX Require the filename (at least UNKONWN)?
+ # XXX Check the filename?
+
+ @property
+ def islocal(self):
+ return self.funcname is not None
+
+ def match(self, other, *,
+ match_files=(lambda f1, f2: f1 == f2),
+ ):
+ """Return True if the two match.
+
+ At least one of the two must be completely valid (no UNKNOWN
+ anywhere). Otherwise False is returned. The remaining one
+ *may* have UNKNOWN for both funcname and filename. It must
+ have a valid name though.
+
+ The caller is responsible for knowing which of the two is valid
+ (and which to use if both are valid).
+ """
+ # First check the name.
+ if self.name is None:
+ return False
+ if other.name != self.name:
+ return False
+
+ # Then check the filename.
+ if self.filename is None:
+ return False
+ if other.filename is None:
+ return False
+ if self.filename == UNKNOWN:
+ # "other" must be the valid one.
+ if other.funcname == UNKNOWN:
+ return False
+ elif self.funcname != UNKNOWN:
+ # XXX Try matching funcname even though we don't
+ # know the filename?
+ raise NotImplementedError
+ else:
+ return True
+ elif other.filename == UNKNOWN:
+ # "self" must be the valid one.
+ if self.funcname == UNKNOWN:
+ return False
+ elif other.funcname != UNKNOWN:
+ # XXX Try matching funcname even though we don't
+ # know the filename?
+ raise NotImplementedError
+ else:
+ return True
+ elif not match_files(self.filename, other.filename):
+ return False
+
+ # Finally, check the funcname.
+ if self.funcname == UNKNOWN:
+ # "other" must be the valid one.
+ if other.funcname == UNKNOWN:
+ return False
+ else:
+ return other.funcname is not None
+ elif other.funcname == UNKNOWN:
+ # "self" must be the valid one.
+ if self.funcname == UNKNOWN:
+ return False
+ else:
+ return self.funcname is not None
+ elif self.funcname == other.funcname:
+ # Both are valid.
+ return True
+
+ return False
diff --git a/Tools/c-analyzer/c_analyzer/common/show.py b/Tools/c-analyzer/c_analyzer/common/show.py
new file mode 100644
index 0000000..5f3cb1c
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/common/show.py
@@ -0,0 +1,11 @@
+
+def basic(variables, *,
+ _print=print):
+ """Print each row simply."""
+ for var in variables:
+ if var.funcname:
+ line = f'{var.filename}:{var.funcname}():{var.name}'
+ else:
+ line = f'{var.filename}:{var.name}'
+ line = f'{line:<64} {var.vartype}'
+ _print(line)
diff --git a/Tools/c-analyzer/c_analyzer_common/util.py b/Tools/c-analyzer/c_analyzer/common/util.py
similarity index 100%
rename from Tools/c-analyzer/c_analyzer_common/util.py
rename to Tools/c-analyzer/c_analyzer/common/util.py
diff --git a/Tools/c-analyzer/c_parser/__init__.py b/Tools/c-analyzer/c_analyzer/parser/__init__.py
similarity index 100%
copy from Tools/c-analyzer/c_parser/__init__.py
copy to Tools/c-analyzer/c_analyzer/parser/__init__.py
diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_analyzer/parser/declarations.py
similarity index 83%
rename from Tools/c-analyzer/c_parser/declarations.py
rename to Tools/c-analyzer/c_analyzer/parser/declarations.py
index 19fa3ff..f37072c 100644
--- a/Tools/c-analyzer/c_parser/declarations.py
+++ b/Tools/c-analyzer/c_analyzer/parser/declarations.py
@@ -2,6 +2,8 @@
import shlex
import subprocess
+from ..common.info import UNKNOWN
+
from . import source
@@ -194,7 +196,28 @@
return name, signature
-def parse_var(stmt):
+#TYPE_SPEC = rf'''(?:
+# )'''
+#VAR_DECLARATOR = rf'''(?:
+# )'''
+#VAR_DECL = rf'''(?:
+# {TYPE_SPEC}+
+# {VAR_DECLARATOR}
+# \s*
+# )'''
+#VAR_DECLARATION = rf'''(?:
+# {VAR_DECL}
+# (?: = [^=] [^;]* )?
+# ;
+# )'''
+#
+#
+#def parse_variable(decl, *, inFunc=False):
+# """Return [(name, storage, vartype)] for the given variable declaration."""
+# ...
+
+
+def _parse_var(stmt):
"""Return (name, vartype) for the given variable declaration."""
stmt = stmt.rstrip(';')
m = LOCAL_STMT_START_RE.match(stmt)
@@ -220,6 +243,27 @@
return name, vartype
+def extract_storage(decl, *, infunc=None):
+ """Return (storage, vartype) based on the given declaration.
+
+ The default storage is "implicit" (or "local" if infunc is True).
+ """
+ if decl == UNKNOWN:
+ return decl
+ if decl.startswith('static '):
+ return 'static'
+ #return 'static', decl.partition(' ')[2].strip()
+ elif decl.startswith('extern '):
+ return 'extern'
+ #return 'extern', decl.partition(' ')[2].strip()
+ elif re.match('.*\b(static|extern)\b', decl):
+ raise NotImplementedError
+ elif infunc:
+ return 'local'
+ else:
+ return 'implicit'
+
+
def parse_compound(stmt, blocks):
"""Return (headers, bodies) for the given compound statement."""
# XXX Identify declarations inside compound statements
@@ -228,14 +272,17 @@
def iter_variables(filename, *,
+ preprocessed=False,
_iter_source_lines=source.iter_lines,
_iter_global=iter_global_declarations,
_iter_local=iter_local_statements,
_parse_func=parse_func,
- _parse_var=parse_var,
+ _parse_var=_parse_var,
_parse_compound=parse_compound,
):
"""Yield (funcname, name, vartype) for every variable in the given file."""
+ if preprocessed:
+ raise NotImplementedError
lines = _iter_source_lines(filename)
for stmt, body in _iter_global(lines):
# At the file top-level we only have to worry about vars & funcs.
@@ -256,7 +303,7 @@
def _iter_locals(lines, *,
_iter_statements=iter_local_statements,
- _parse_var=parse_var,
+ _parse_var=_parse_var,
_parse_compound=parse_compound,
):
compound = [lines]
@@ -278,18 +325,15 @@
compound.extend(bodies)
-def iter_all(dirnames):
+def iter_all(filename, *,
+ preprocessed=False,
+ ):
"""Yield a Declaration for each one found.
If there are duplicates, due to preprocessor conditionals, then
they are checked to make sure they are the same.
"""
- raise NotImplementedError
-
-
-def iter_preprocessed(dirnames):
- """Yield a Declaration for each one found.
-
- All source files are run through the preprocessor first.
- """
- raise NotImplementedError
+ # XXX For the moment we cheat.
+ for funcname, name, decl in iter_variables(filename,
+ preprocessed=preprocessed):
+ yield 'variable', funcname, name, decl
diff --git a/Tools/c-analyzer/c_analyzer/parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py
new file mode 100644
index 0000000..3860d3d
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/parser/find.py
@@ -0,0 +1,107 @@
+from ..common.info import UNKNOWN, ID
+
+from . import declarations
+
+# XXX need tests:
+# * variables
+# * variable
+# * variable_from_id
+
+
+def _iter_vars(filenames, preprocessed, *,
+ handle_id=None,
+ _iter_decls=declarations.iter_all,
+ ):
+ if handle_id is None:
+ handle_id = ID
+
+ for filename in filenames or ():
+ for kind, funcname, name, decl in _iter_decls(filename,
+ preprocessed=preprocessed,
+ ):
+ if kind != 'variable':
+ continue
+ varid = handle_id(filename, funcname, name)
+ yield varid, decl
+
+
+# XXX Add a "handle_var" arg like we did for get_resolver()?
+
+def variables(*filenames,
+ perfilecache=None,
+ preprocessed=False,
+ known=None, # for types
+ handle_id=None,
+ _iter_vars=_iter_vars,
+ ):
+ """Yield (varid, decl) for each variable found in the given files.
+
+ If "preprocessed" is provided (and not False/None) then it is used
+ to decide which tool to use to parse the source code after it runs
+ through the C preprocessor. Otherwise the raw
+ """
+ if len(filenames) == 1 and not (filenames[0], str):
+ filenames, = filenames
+
+ if perfilecache is None:
+ yield from _iter_vars(filenames, preprocessed)
+ else:
+ # XXX Cache per-file variables (e.g. `{filename: [(varid, decl)]}`).
+ raise NotImplementedError
+
+
+def variable(name, filenames, *,
+ local=False,
+ perfilecache=None,
+ preprocessed=False,
+ handle_id=None,
+ _iter_vars=variables,
+ ):
+ """Return (varid, decl) for the first found variable that matches.
+
+ If "local" is True then the first matching local variable in the
+ file will always be returned. To avoid that, pass perfilecache and
+ pop each variable from the cache after using it.
+ """
+ for varid, decl in _iter_vars(filenames,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ ):
+ if varid.name != name:
+ continue
+ if local:
+ if varid.funcname:
+ if varid.funcname == UNKNOWN:
+ raise NotImplementedError
+ return varid, decl
+ elif not varid.funcname:
+ return varid, decl
+ else:
+ return None, None # No matching variable was found.
+
+
+def variable_from_id(id, filenames, *,
+ perfilecache=None,
+ preprocessed=False,
+ handle_id=None,
+ _get_var=variable,
+ ):
+ """Return (varid, decl) for the first found variable that matches."""
+ local = False
+ if isinstance(id, str):
+ name = id
+ else:
+ if id.funcname == UNKNOWN:
+ local = True
+ elif id.funcname:
+ raise NotImplementedError
+
+ name = id.name
+ if id.filename and id.filename != UNKNOWN:
+ filenames = [id.filename]
+ return _get_var(name, filenames,
+ local=local,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ handle_id=handle_id,
+ )
diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_analyzer/parser/naive.py
similarity index 80%
rename from Tools/c-analyzer/c_parser/naive.py
rename to Tools/c-analyzer/c_analyzer/parser/naive.py
index 160f96c..4a4822d 100644
--- a/Tools/c-analyzer/c_parser/naive.py
+++ b/Tools/c-analyzer/c_analyzer/parser/naive.py
@@ -1,8 +1,7 @@
import re
-from c_analyzer_common.info import UNKNOWN
+from ..common.info import UNKNOWN, ID
-from .info import Variable
from .preprocessor import _iter_clean_lines
@@ -55,7 +54,7 @@
def parse_variable(srcline, funcname=None):
- """Return a Variable for the variable declared on the line (or None)."""
+ """Return (varid, decl) for the variable declared on the line (or None)."""
line = srcline.strip()
# XXX Handle more than just static variables.
@@ -74,7 +73,7 @@
_get_srclines=get_srclines,
_default_parse_variable=parse_variable,
):
- """Yield a Variable for each in the given source file."""
+ """Yield (varid, decl) for each variable in the given source file."""
if parse_variable is None:
parse_variable = _default_parse_variable
@@ -99,13 +98,13 @@
info = parse_variable(line, funcname)
if isinstance(info, list):
for name, _funcname, decl in info:
- yield Variable.from_parts(filename, _funcname, name, decl)
+ yield ID(filename, _funcname, name), decl
continue
name, decl = info
if name is None:
continue
- yield Variable.from_parts(filename, funcname, name, decl)
+ yield ID(filename, funcname, name), decl
def _match_varid(variable, name, funcname, ignored=None):
@@ -134,12 +133,12 @@
Return None if the variable is not found.
"""
- for variable in _iter_variables(filename,
+ for varid, decl in _iter_variables(filename,
srccache=srccache,
parse_variable=parse_variable,
):
- if _match_varid(variable, name, funcname, ignored):
- return variable
+ if _match_varid(varid, name, funcname, ignored):
+ return varid, decl
else:
return None
@@ -149,10 +148,10 @@
parse_variable=None,
_find_symbol=find_variable,
):
- """Yield a Variable for each ID.
+ """Yield (varid, decl) for each ID.
If the variable is not found then its decl will be UNKNOWN. That
- way there will be one resulting Variable per given ID.
+ way there will be one resulting variable per given ID.
"""
if srccache is _NOT_SET:
srccache = {}
@@ -163,18 +162,18 @@
srcfiles = [varid.filename]
else:
if not filenames:
- yield Variable(varid, UNKNOWN, UNKNOWN)
+ yield varid, UNKNOWN
continue
srcfiles = filenames
for filename in srcfiles:
- found = _find_varid(filename, varid.funcname, varid.name,
- ignored=used,
- srccache=srccache,
- parse_variable=parse_variable,
- )
- if found:
- yield found
- used.add(found)
+ varid, decl = _find_varid(filename, varid.funcname, varid.name,
+ ignored=used,
+ srccache=srccache,
+ parse_variable=parse_variable,
+ )
+ if varid:
+ yield varid, decl
+ used.add(varid)
break
else:
- yield Variable(varid, UNKNOWN, UNKNOWN)
+ yield varid, UNKNOWN
diff --git a/Tools/c-analyzer/c_parser/preprocessor.py b/Tools/c-analyzer/c_analyzer/parser/preprocessor.py
similarity index 99%
rename from Tools/c-analyzer/c_parser/preprocessor.py
rename to Tools/c-analyzer/c_analyzer/parser/preprocessor.py
index 0e2866e..41f306e 100644
--- a/Tools/c-analyzer/c_parser/preprocessor.py
+++ b/Tools/c-analyzer/c_analyzer/parser/preprocessor.py
@@ -3,8 +3,7 @@
import os
import re
-from c_analyzer_common import util
-from . import info
+from ..common import util, info
CONTINUATION = '\\' + os.linesep
diff --git a/Tools/c-analyzer/c_parser/source.py b/Tools/c-analyzer/c_analyzer/parser/source.py
similarity index 100%
rename from Tools/c-analyzer/c_parser/source.py
rename to Tools/c-analyzer/c_analyzer/parser/source.py
diff --git a/Tools/c-analyzer/c_symbols/__init__.py b/Tools/c-analyzer/c_analyzer/symbols/__init__.py
similarity index 100%
rename from Tools/c-analyzer/c_symbols/__init__.py
rename to Tools/c-analyzer/c_analyzer/symbols/__init__.py
diff --git a/Tools/c-analyzer/c_analyzer/symbols/_nm.py b/Tools/c-analyzer/c_analyzer/symbols/_nm.py
new file mode 100644
index 0000000..f3a75a6
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/symbols/_nm.py
@@ -0,0 +1,117 @@
+import os.path
+import shutil
+
+from c_analyzer.common import util, info
+
+from .info import Symbol
+
+
+# XXX need tests:
+# * iter_symbols
+
+NM_KINDS = {
+ 'b': Symbol.KIND.VARIABLE, # uninitialized
+ 'd': Symbol.KIND.VARIABLE, # initialized
+ #'g': Symbol.KIND.VARIABLE, # uninitialized
+ #'s': Symbol.KIND.VARIABLE, # initialized
+ 't': Symbol.KIND.FUNCTION,
+ }
+
+SPECIAL_SYMBOLS = {
+ # binary format (e.g. ELF)
+ '__bss_start',
+ '__data_start',
+ '__dso_handle',
+ '_DYNAMIC',
+ '_edata',
+ '_end',
+ '__environ@@GLIBC_2.2.5',
+ '_GLOBAL_OFFSET_TABLE_',
+ '__JCR_END__',
+ '__JCR_LIST__',
+ '__TMC_END__',
+ }
+
+
+def _is_special_symbol(name):
+ if name in SPECIAL_SYMBOLS:
+ return True
+ if '@@GLIBC' in name:
+ return True
+ return False
+
+
+def iter_symbols(binfile, *,
+ nm=None,
+ handle_id=None,
+ _which=shutil.which,
+ _run=util.run_cmd,
+ ):
+ """Yield a Symbol for each relevant entry reported by the "nm" command."""
+ if nm is None:
+ nm = _which('nm')
+ if not nm:
+ raise NotImplementedError
+ if handle_id is None:
+ handle_id = info.ID
+
+ argv = [nm,
+ '--line-numbers',
+ binfile,
+ ]
+ try:
+ output = _run(argv)
+ except Exception:
+ if nm is None:
+ # XXX Use dumpbin.exe /SYMBOLS on Windows.
+ raise NotImplementedError
+ raise
+ for line in output.splitlines():
+ (name, kind, external, filename, funcname,
+ ) = _parse_nm_line(line)
+ if kind != Symbol.KIND.VARIABLE:
+ continue
+ elif _is_special_symbol(name):
+ continue
+ yield Symbol(
+ id=handle_id(filename, funcname, name),
+ kind=kind,
+ external=external,
+ )
+
+
+def _parse_nm_line(line):
+ _origline = line
+ _, _, line = line.partition(' ') # strip off the address
+ line = line.strip()
+
+ kind, _, line = line.partition(' ')
+ line = line.strip()
+ external = kind.isupper()
+ kind = NM_KINDS.get(kind.lower(), Symbol.KIND.OTHER)
+
+ name, _, filename = line.partition('\t')
+ name = name.strip()
+ if filename:
+ filename = os.path.relpath(filename.partition(':')[0])
+ else:
+ filename = info.UNKNOWN
+
+ name, islocal = _parse_nm_name(name, kind)
+ funcname = info.UNKNOWN if islocal else None
+ return name, kind, external, filename, funcname
+
+
+def _parse_nm_name(name, kind):
+ if kind != Symbol.KIND.VARIABLE:
+ return name, None
+ if _is_special_symbol(name):
+ return name, None
+
+ actual, sep, digits = name.partition('.')
+ if not sep:
+ return name, False
+
+ if not digits.isdigit():
+ raise Exception(f'got bogus name {name}')
+ return actual, True
diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py
new file mode 100644
index 0000000..8564652
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/symbols/find.py
@@ -0,0 +1,175 @@
+import os
+import os.path
+import shutil
+
+from ..common import files
+from ..common.info import UNKNOWN, ID
+from ..parser import find as p_find
+
+from . import _nm
+from .info import Symbol
+
+# XXX need tests:
+# * get_resolver()
+# * get_resolver_from_dirs()
+# * symbol()
+# * symbols()
+# * variables()
+
+
+def _resolve_known(symbol, knownvars):
+ for varid in knownvars:
+ if symbol.match(varid):
+ break
+ else:
+ return None
+ return knownvars.pop(varid)
+
+
+def get_resolver(filenames=None, known=None, *,
+ handle_var,
+ check_filename=None,
+ perfilecache=None,
+ preprocessed=False,
+ _from_source=p_find.variable_from_id,
+ ):
+ """Return a "resolver" func for the given known vars/types and filenames.
+
+ "handle_var" is a callable that takes (ID, decl) and returns a
+ Variable. Variable.from_id is a suitable callable.
+
+ The returned func takes a single Symbol and returns a corresponding
+ Variable. If the symbol was located then the variable will be
+ valid, populated with the corresponding information. Otherwise None
+ is returned.
+ """
+ knownvars = (known or {}).get('variables')
+ if knownvars:
+ knownvars = dict(knownvars) # a copy
+ if filenames:
+ if check_filename is None:
+ filenames = list(filenames)
+ def check_filename(filename):
+ return filename in filenames
+ def resolve(symbol):
+ # XXX Check "found" instead?
+ if not check_filename(symbol.filename):
+ return None
+ found = _resolve_known(symbol, knownvars)
+ if found is None:
+ #return None
+ varid, decl = _from_source(symbol, filenames,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ )
+ found = handle_var(varid, decl)
+ return found
+ else:
+ def resolve(symbol):
+ return _resolve_known(symbol, knownvars)
+ elif filenames:
+ def resolve(symbol):
+ varid, decl = _from_source(symbol, filenames,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ )
+ return handle_var(varid, decl)
+ else:
+ def resolve(symbol):
+ return None
+ return resolve
+
+
+def get_resolver_from_dirs(dirnames, known=None, *,
+ handle_var,
+ suffixes=('.c',),
+ perfilecache=None,
+ preprocessed=False,
+ _iter_files=files.iter_files_by_suffix,
+ _get_resolver=get_resolver,
+ ):
+ """Return a "resolver" func for the given known vars/types and filenames.
+
+ "dirnames" should be absolute paths. If not then they will be
+ resolved relative to CWD.
+
+ See get_resolver().
+ """
+ dirnames = [d if d.endswith(os.path.sep) else d + os.path.sep
+ for d in dirnames]
+ filenames = _iter_files(dirnames, suffixes)
+ def check_filename(filename):
+ for dirname in dirnames:
+ if filename.startswith(dirname):
+ return True
+ else:
+ return False
+ return _get_resolver(filenames, known,
+ handle_var=handle_var,
+ check_filename=check_filename,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ )
+
+
+def symbol(symbol, filenames, known=None, *,
+ perfilecache=None,
+ preprocessed=False,
+ handle_id=None,
+ _get_resolver=get_resolver,
+ ):
+ """Return a Variable for the one matching the given symbol.
+
+ "symbol" can be one of several objects:
+
+ * Symbol - use the contained info
+ * name (str) - look for a global variable with that name
+ * (filename, name) - look for named global in file
+ * (filename, funcname, name) - look for named local in file
+
+ A name is always required. If the filename is None, "", or
+ "UNKNOWN" then all files will be searched. If the funcname is
+ "" or "UNKNOWN" then only local variables will be searched for.
+ """
+ resolve = _get_resolver(known, filenames,
+ handle_id=handle_id,
+ perfilecache=perfilecache,
+ preprocessed=preprocessed,
+ )
+ return resolve(symbol)
+
+
+def _get_platform_tool():
+ if os.name == 'nt':
+ # XXX Support this.
+ raise NotImplementedError
+ elif nm := shutil.which('nm'):
+ return lambda b, hi: _nm.iter_symbols(b, nm=nm, handle_id=hi)
+ else:
+ raise NotImplementedError
+
+
+def symbols(binfile, *,
+ handle_id=None,
+ _file_exists=os.path.exists,
+ _get_platform_tool=_get_platform_tool,
+ ):
+ """Yield a Symbol for each one found in the binary."""
+ if not _file_exists(binfile):
+ raise Exception('executable missing (need to build it first?)')
+
+ _iter_symbols = _get_platform_tool()
+ yield from _iter_symbols(binfile, handle_id)
+
+
+def variables(binfile, *,
+ resolve,
+ handle_id=None,
+ _iter_symbols=symbols,
+ ):
+ """Yield (Variable, Symbol) for each found symbol."""
+ for symbol in _iter_symbols(binfile, handle_id=handle_id):
+ if symbol.kind != Symbol.KIND.VARIABLE:
+ continue
+ var = resolve(symbol) or None
+ yield var, symbol
diff --git a/Tools/c-analyzer/c_symbols/info.py b/Tools/c-analyzer/c_analyzer/symbols/info.py
similarity index 93%
rename from Tools/c-analyzer/c_symbols/info.py
rename to Tools/c-analyzer/c_analyzer/symbols/info.py
index f6ed52c..96a251a 100644
--- a/Tools/c-analyzer/c_symbols/info.py
+++ b/Tools/c-analyzer/c_analyzer/symbols/info.py
@@ -1,7 +1,7 @@
from collections import namedtuple
-from c_analyzer_common.info import ID
-from c_analyzer_common.util import classonly, _NTBase
+from c_analyzer.common.info import ID
+from c_analyzer.common.util import classonly, _NTBase
class Symbol(_NTBase, namedtuple('Symbol', 'id kind external')):
diff --git a/Tools/c-analyzer/c_globals/__init__.py b/Tools/c-analyzer/c_analyzer/variables/__init__.py
similarity index 100%
copy from Tools/c-analyzer/c_globals/__init__.py
copy to Tools/c-analyzer/c_analyzer/variables/__init__.py
diff --git a/Tools/c-analyzer/c_analyzer/variables/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py
new file mode 100644
index 0000000..3fe7284
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/variables/find.py
@@ -0,0 +1,75 @@
+from ..common import files
+from ..common.info import UNKNOWN
+from ..parser import (
+ find as p_find,
+ )
+from ..symbols import (
+ info as s_info,
+ find as s_find,
+ )
+from .info import Variable
+
+# XXX need tests:
+# * vars_from_source
+
+
+def _remove_cached(cache, var):
+ if not cache:
+ return
+ try:
+ cached = cache[var.filename]
+ cached.remove(var)
+ except (KeyError, IndexError):
+ pass
+
+
+def vars_from_binary(binfile, *,
+ known=None,
+ filenames=None,
+ handle_id=None,
+ check_filename=None,
+ handle_var=Variable.from_id,
+ _iter_vars=s_find.variables,
+ _get_symbol_resolver=s_find.get_resolver,
+ ):
+ """Yield a Variable for each found Symbol.
+
+ Details are filled in from the given "known" variables and types.
+ """
+ cache = {}
+ resolve = _get_symbol_resolver(filenames, known,
+ handle_var=handle_var,
+ check_filename=check_filename,
+ perfilecache=cache,
+ )
+ for var, symbol in _iter_vars(binfile,
+ resolve=resolve,
+ handle_id=handle_id,
+ ):
+ if var is None:
+ var = Variable(symbol.id, UNKNOWN, UNKNOWN)
+ yield var
+ _remove_cached(cache, var)
+
+
+def vars_from_source(filenames, *,
+ preprocessed=None,
+ known=None,
+ handle_id=None,
+ handle_var=Variable.from_id,
+ iter_vars=p_find.variables,
+ ):
+ """Yield a Variable for each declaration in the raw source code.
+
+ Details are filled in from the given "known" variables and types.
+ """
+ cache = {}
+ for varid, decl in iter_vars(filenames or (),
+ perfilecache=cache,
+ preprocessed=preprocessed,
+ known=known,
+ handle_id=handle_id,
+ ):
+ var = handle_var(varid, decl)
+ yield var
+ _remove_cached(cache, var)
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_analyzer/variables/info.py
similarity index 61%
rename from Tools/c-analyzer/c_parser/info.py
rename to Tools/c-analyzer/c_analyzer/variables/info.py
index a4e32d7..336a523 100644
--- a/Tools/c-analyzer/c_parser/info.py
+++ b/Tools/c-analyzer/c_analyzer/variables/info.py
@@ -1,8 +1,7 @@
from collections import namedtuple
-import re
-from c_analyzer_common import info, util
-from c_analyzer_common.util import classonly, _NTBase
+from ..common.info import ID, UNKNOWN
+from ..common.util import classonly, _NTBase
def normalize_vartype(vartype):
@@ -16,26 +15,7 @@
return str(vartype)
-def extract_storage(decl, *, isfunc=False):
- """Return (storage, vartype) based on the given declaration.
-
- The default storage is "implicit" or "local".
- """
- if decl == info.UNKNOWN:
- return decl, decl
- if decl.startswith('static '):
- return 'static', decl
- #return 'static', decl.partition(' ')[2].strip()
- elif decl.startswith('extern '):
- return 'extern', decl
- #return 'extern', decl.partition(' ')[2].strip()
- elif re.match('.*\b(static|extern)\b', decl):
- raise NotImplementedError
- elif isfunc:
- return 'local', decl
- else:
- return 'implicit', decl
-
+# XXX Variable.vartype -> decl (Declaration).
class Variable(_NTBase,
namedtuple('Variable', 'id storage vartype')):
@@ -52,16 +32,23 @@
@classonly
def from_parts(cls, filename, funcname, name, decl, storage=None):
+ varid = ID(filename, funcname, name)
if storage is None:
- storage, decl = extract_storage(decl, isfunc=funcname)
- id = info.ID(filename, funcname, name)
- self = cls(id, storage, decl)
+ self = cls.from_id(varid, decl)
+ else:
+ self = cls(varid, storage, decl)
return self
+ @classonly
+ def from_id(cls, varid, decl):
+ from ..parser.declarations import extract_storage
+ storage = extract_storage(decl, infunc=varid.funcname)
+ return cls(varid, storage, decl)
+
def __new__(cls, id, storage, vartype):
self = super().__new__(
cls,
- id=info.ID.from_raw(id),
+ id=ID.from_raw(id),
storage=str(storage) if storage else None,
vartype=normalize_vartype(vartype) if vartype else None,
)
@@ -77,10 +64,10 @@
if not self.id:
raise TypeError('missing id')
- if not self.filename or self.filename == info.UNKNOWN:
+ if not self.filename or self.filename == UNKNOWN:
raise TypeError(f'id missing filename ({self.id})')
- if self.funcname and self.funcname == info.UNKNOWN:
+ if self.funcname and self.funcname == UNKNOWN:
raise TypeError(f'id missing funcname ({self.id})')
self.id.validate()
@@ -89,12 +76,12 @@
"""Fail if the object is invalid (i.e. init with bad data)."""
self._validate_id()
- if self.storage is None or self.storage == info.UNKNOWN:
+ if self.storage is None or self.storage == UNKNOWN:
raise TypeError('missing storage')
elif self.storage not in self.STORAGE:
raise ValueError(f'unsupported storage {self.storage:r}')
- if self.vartype is None or self.vartype == info.UNKNOWN:
+ if self.vartype is None or self.vartype == UNKNOWN:
raise TypeError('missing vartype')
@property
diff --git a/Tools/c-analyzer/c_analyzer/variables/known.py b/Tools/c-analyzer/c_analyzer/variables/known.py
new file mode 100644
index 0000000..aa2934a
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer/variables/known.py
@@ -0,0 +1,91 @@
+import csv
+
+from ..common.info import ID, UNKNOWN
+from ..common.util import read_tsv
+from .info import Variable
+
+
+# XXX need tests:
+# * read_file()
+# * look_up_variable()
+
+
+COLUMNS = ('filename', 'funcname', 'name', 'kind', 'declaration')
+HEADER = '\t'.join(COLUMNS)
+
+
+def read_file(infile, *,
+ _read_tsv=read_tsv,
+ ):
+ """Yield (kind, id, decl) for each row in the data file.
+
+ The caller is responsible for validating each row.
+ """
+ for row in _read_tsv(infile, HEADER):
+ filename, funcname, name, kind, declaration = row
+ if not funcname or funcname == '-':
+ funcname = None
+ id = ID(filename, funcname, name)
+ yield kind, id, declaration
+
+
+def from_file(infile, *,
+ handle_var=Variable.from_id,
+ _read_file=read_file,
+ ):
+ """Return the info for known declarations in the given file."""
+ known = {
+ 'variables': {},
+ #'types': {},
+ #'constants': {},
+ #'macros': {},
+ }
+ for kind, id, decl in _read_file(infile):
+ if kind == 'variable':
+ values = known['variables']
+ value = handle_var(id, decl)
+ else:
+ raise ValueError(f'unsupported kind in row {row}')
+ value.validate()
+ values[id] = value
+ return known
+
+
+def look_up_variable(varid, knownvars, *,
+ match_files=(lambda f1, f2: f1 == f2),
+ ):
+ """Return the known Variable matching the given ID.
+
+ "knownvars" is a mapping of ID to Variable.
+
+ "match_files" is used to verify if two filenames point to
+ the same file.
+
+ If no match is found then None is returned.
+ """
+ if not knownvars:
+ return None
+
+ if varid.funcname == UNKNOWN:
+ if not varid.filename or varid.filename == UNKNOWN:
+ for varid in knownvars:
+ if not varid.funcname:
+ continue
+ if varid.name == varid.name:
+ return knownvars[varid]
+ else:
+ return None
+ else:
+ for varid in knownvars:
+ if not varid.funcname:
+ continue
+ if not match_files(varid.filename, varid.filename):
+ continue
+ if varid.name == varid.name:
+ return knownvars[varid]
+ else:
+ return None
+ elif not varid.filename or varid.filename == UNKNOWN:
+ raise NotImplementedError
+ else:
+ return knownvars.get(varid.id)
diff --git a/Tools/c-analyzer/c_analyzer_common/__init__.py b/Tools/c-analyzer/c_analyzer_common/__init__.py
deleted file mode 100644
index 888b16f..0000000
--- a/Tools/c-analyzer/c_analyzer_common/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os.path
-
-
-PKG_ROOT = os.path.dirname(__file__)
-DATA_DIR = os.path.dirname(PKG_ROOT)
-REPO_ROOT = os.path.dirname(
- os.path.dirname(DATA_DIR))
-
-SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [
- 'Include',
- 'Python',
- 'Parser',
- 'Objects',
- 'Modules',
- ]]
-
-
-# Clean up the namespace.
-del os
diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer_common/info.py
deleted file mode 100644
index e217380..0000000
--- a/Tools/c-analyzer/c_analyzer_common/info.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from collections import namedtuple
-import re
-
-from .util import classonly, _NTBase
-
-
-UNKNOWN = '???'
-
-NAME_RE = re.compile(r'^([a-zA-Z]|_\w*[a-zA-Z]\w*|[a-zA-Z]\w*)$')
-
-
-class ID(_NTBase, namedtuple('ID', 'filename funcname name')):
- """A unique ID for a single symbol or declaration."""
-
- __slots__ = ()
- # XXX Add optional conditions (tuple of strings) field.
- #conditions = Slot()
-
- @classonly
- def from_raw(cls, raw):
- if not raw:
- return None
- if isinstance(raw, str):
- return cls(None, None, raw)
- try:
- name, = raw
- filename = None
- except ValueError:
- try:
- filename, name = raw
- except ValueError:
- return super().from_raw(raw)
- return cls(filename, None, name)
-
- def __new__(cls, filename, funcname, name):
- self = super().__new__(
- cls,
- filename=str(filename) if filename else None,
- funcname=str(funcname) if funcname else None,
- name=str(name) if name else None,
- )
- #cls.conditions.set(self, tuple(str(s) if s else None
- # for s in conditions or ()))
- return self
-
- def validate(self):
- """Fail if the object is invalid (i.e. init with bad data)."""
- if not self.name:
- raise TypeError('missing name')
- else:
- if not NAME_RE.match(self.name):
- raise ValueError(
- f'name must be an identifier, got {self.name!r}')
-
- # Symbols from a binary might not have filename/funcname info.
-
- if self.funcname:
- if not self.filename:
- raise TypeError('missing filename')
- if not NAME_RE.match(self.funcname) and self.funcname != UNKNOWN:
- raise ValueError(
- f'name must be an identifier, got {self.funcname!r}')
-
- # XXX Require the filename (at least UNKONWN)?
- # XXX Check the filename?
-
- @property
- def islocal(self):
- return self.funcname is not None
diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py
deleted file mode 100644
index dec1e1d..0000000
--- a/Tools/c-analyzer/c_analyzer_common/known.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import csv
-import os.path
-
-from c_parser.info import Variable
-
-from . import DATA_DIR
-from .info import ID, UNKNOWN
-from .util import read_tsv
-
-
-DATA_FILE = os.path.join(DATA_DIR, 'known.tsv')
-
-COLUMNS = ('filename', 'funcname', 'name', 'kind', 'declaration')
-HEADER = '\t'.join(COLUMNS)
-
-
-# XXX need tests:
-# * from_file()
-
-def from_file(infile, *,
- _read_tsv=read_tsv,
- ):
- """Return the info for known declarations in the given file."""
- known = {
- 'variables': {},
- #'types': {},
- #'constants': {},
- #'macros': {},
- }
- for row in _read_tsv(infile, HEADER):
- filename, funcname, name, kind, declaration = row
- if not funcname or funcname == '-':
- funcname = None
- id = ID(filename, funcname, name)
- if kind == 'variable':
- values = known['variables']
- if funcname:
- storage = _get_storage(declaration) or 'local'
- else:
- storage = _get_storage(declaration) or 'implicit'
- value = Variable(id, storage, declaration)
- else:
- raise ValueError(f'unsupported kind in row {row}')
- value.validate()
-# if value.name == 'id' and declaration == UNKNOWN:
-# # None of these are variables.
-# declaration = 'int id';
-# else:
-# value.validate()
- values[id] = value
- return known
-
-
-def _get_storage(decl):
- # statics
- if decl.startswith('static '):
- return 'static'
- if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
- return 'static'
- if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')):
- return 'static'
- if decl.startswith('PyDoc_VAR('):
- return 'static'
- if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
- return 'static'
- if decl.startswith('WRAP_METHOD('):
- return 'static'
- # public extern
- if decl.startswith('extern '):
- return 'extern'
- if decl.startswith('PyAPI_DATA('):
- return 'extern'
- # implicit or local
- return None
diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py
deleted file mode 100644
index a51b947..0000000
--- a/Tools/c-analyzer/c_globals/find.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from c_analyzer_common import SOURCE_DIRS
-from c_analyzer_common.info import UNKNOWN
-from c_symbols import (
- info as s_info,
- binary as b_symbols,
- source as s_symbols,
- resolve,
- )
-from c_parser import info, declarations
-
-
-# XXX needs tests:
-# * iter_variables
-
-def globals_from_binary(binfile=b_symbols.PYTHON, *,
- knownvars=None,
- dirnames=None,
- _iter_symbols=b_symbols.iter_symbols,
- _resolve=resolve.symbols_to_variables,
- _get_symbol_resolver=resolve.get_resolver,
- ):
- """Yield a Variable for each found Symbol.
-
- Details are filled in from the given "known" variables and types.
- """
- symbols = _iter_symbols(binfile, find_local_symbol=None)
- #symbols = list(symbols)
- for variable in _resolve(symbols,
- resolve=_get_symbol_resolver(knownvars, dirnames),
- ):
- # Skip each non-global variable (unless we couldn't find it).
- # XXX Drop the "UNKNOWN" condition?
- if not variable.isglobal and variable.vartype != UNKNOWN:
- continue
- yield variable
-
-
-def globals_from_declarations(dirnames=SOURCE_DIRS, *,
- known=None,
- ):
- """Yield a Variable for each found declaration.
-
- Details are filled in from the given "known" variables and types.
- """
- raise NotImplementedError
-
-
-def iter_variables(kind='platform', *,
- known=None,
- dirnames=None,
- _resolve_symbols=resolve.symbols_to_variables,
- _get_symbol_resolver=resolve.get_resolver,
- _symbols_from_binary=b_symbols.iter_symbols,
- _symbols_from_source=s_symbols.iter_symbols,
- _iter_raw=declarations.iter_all,
- _iter_preprocessed=declarations.iter_preprocessed,
- ):
- """Yield a Variable for each one found (e.g. in files)."""
- kind = kind or 'platform'
-
- if kind == 'symbols':
- knownvars = (known or {}).get('variables')
- yield from _resolve_symbols(
- _symbols_from_source(dirnames, known),
- resolve=_get_symbol_resolver(knownvars, dirnames),
- )
- elif kind == 'platform':
- knownvars = (known or {}).get('variables')
- yield from _resolve_symbols(
- _symbols_from_binary(find_local_symbol=None),
- resolve=_get_symbol_resolver(knownvars, dirnames),
- )
- elif kind == 'declarations':
- for decl in _iter_raw(dirnames):
- if not isinstance(decl, info.Variable):
- continue
- yield decl
- elif kind == 'preprocessed':
- for decl in _iter_preprocessed(dirnames):
- if not isinstance(decl, info.Variable):
- continue
- yield decl
- else:
- raise ValueError(f'unsupported kind {kind!r}')
-
-
-def globals(dirnames, known, *,
- kind=None, # Use the default.
- _iter_variables=iter_variables,
- ):
- """Return a list of (StaticVar, <supported>) for each found global var."""
- for found in _iter_variables(kind, known=known, dirnames=dirnames):
- if not found.isglobal:
- continue
- yield found
diff --git a/Tools/c-analyzer/c_globals/show.py b/Tools/c-analyzer/c_globals/show.py
deleted file mode 100644
index f4298b1..0000000
--- a/Tools/c-analyzer/c_globals/show.py
+++ /dev/null
@@ -1,16 +0,0 @@
-
-def basic(globals, *,
- _print=print):
- """Print each row simply."""
- for variable in globals:
- if variable.funcname:
- line = f'{variable.filename}:{variable.funcname}():{variable.name}'
- else:
- line = f'{variable.filename}:{variable.name}'
- vartype = variable.vartype
- #if vartype.startswith('static '):
- # vartype = vartype.partition(' ')[2]
- #else:
- # vartype = '=' + vartype
- line = f'{line:<64} {vartype}'
- _print(line)
diff --git a/Tools/c-analyzer/c_symbols/binary.py b/Tools/c-analyzer/c_symbols/binary.py
deleted file mode 100644
index e125dbd5..0000000
--- a/Tools/c-analyzer/c_symbols/binary.py
+++ /dev/null
@@ -1,157 +0,0 @@
-import os
-import os.path
-import shutil
-import sys
-
-from c_analyzer_common import util, info
-from . import source
-from .info import Symbol
-
-
-#PYTHON = os.path.join(REPO_ROOT, 'python')
-PYTHON = sys.executable
-
-
-def iter_symbols(binary=PYTHON, dirnames=None, *,
- # Alternately, use look_up_known_symbol()
- # from c_globals.supported.
- find_local_symbol=source.find_symbol,
- _file_exists=os.path.exists,
- _iter_symbols_nm=(lambda b, *a: _iter_symbols_nm(b, *a)),
- ):
- """Yield a Symbol for each symbol found in the binary."""
- if not _file_exists(binary):
- raise Exception('executable missing (need to build it first?)')
-
- if find_local_symbol:
- cache = {}
- def find_local_symbol(name, *, _find=find_local_symbol):
- return _find(name, dirnames, _perfilecache=cache)
- else:
- find_local_symbol = None
-
- if os.name == 'nt':
- # XXX Support this.
- raise NotImplementedError
- else:
- yield from _iter_symbols_nm(binary, find_local_symbol)
-
-
-#############################
-# binary format (e.g. ELF)
-
-SPECIAL_SYMBOLS = {
- '__bss_start',
- '__data_start',
- '__dso_handle',
- '_DYNAMIC',
- '_edata',
- '_end',
- '__environ@@GLIBC_2.2.5',
- '_GLOBAL_OFFSET_TABLE_',
- '__JCR_END__',
- '__JCR_LIST__',
- '__TMC_END__',
- }
-
-
-def _is_special_symbol(name):
- if name in SPECIAL_SYMBOLS:
- return True
- if '@@GLIBC' in name:
- return True
- return False
-
-
-#############################
-# "nm"
-
-NM_KINDS = {
- 'b': Symbol.KIND.VARIABLE, # uninitialized
- 'd': Symbol.KIND.VARIABLE, # initialized
- #'g': Symbol.KIND.VARIABLE, # uninitialized
- #'s': Symbol.KIND.VARIABLE, # initialized
- 't': Symbol.KIND.FUNCTION,
- }
-
-
-def _iter_symbols_nm(binary, find_local_symbol=None,
- *,
- _which=shutil.which,
- _run=util.run_cmd,
- ):
- nm = _which('nm')
- if not nm:
- raise NotImplementedError
- argv = [nm,
- '--line-numbers',
- binary,
- ]
- try:
- output = _run(argv)
- except Exception:
- if nm is None:
- # XXX Use dumpbin.exe /SYMBOLS on Windows.
- raise NotImplementedError
- raise
- for line in output.splitlines():
- (name, kind, external, filename, funcname, vartype,
- ) = _parse_nm_line(line,
- _find_local_symbol=find_local_symbol,
- )
- if kind != Symbol.KIND.VARIABLE:
- continue
- elif _is_special_symbol(name):
- continue
- assert vartype is None
- yield Symbol(
- id=(filename, funcname, name),
- kind=kind,
- external=external,
- )
-
-
-def _parse_nm_line(line, *, _find_local_symbol=None):
- _origline = line
- _, _, line = line.partition(' ') # strip off the address
- line = line.strip()
-
- kind, _, line = line.partition(' ')
- line = line.strip()
- external = kind.isupper()
- kind = NM_KINDS.get(kind.lower(), Symbol.KIND.OTHER)
-
- name, _, filename = line.partition('\t')
- name = name.strip()
- if filename:
- filename = os.path.relpath(filename.partition(':')[0])
- else:
- filename = info.UNKNOWN
-
- vartype = None
- name, islocal = _parse_nm_name(name, kind)
- if islocal:
- funcname = info.UNKNOWN
- if _find_local_symbol is not None:
- filename, funcname, vartype = _find_local_symbol(name)
- filename = filename or info.UNKNOWN
- funcname = funcname or info.UNKNOWN
- else:
- funcname = None
- # XXX fine filename and vartype?
- return name, kind, external, filename, funcname, vartype
-
-
-def _parse_nm_name(name, kind):
- if kind != Symbol.KIND.VARIABLE:
- return name, None
- if _is_special_symbol(name):
- return name, None
-
- actual, sep, digits = name.partition('.')
- if not sep:
- return name, False
-
- if not digits.isdigit():
- raise Exception(f'got bogus name {name}')
- return actual, True
diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py
deleted file mode 100644
index 56210ce..0000000
--- a/Tools/c-analyzer/c_symbols/resolve.py
+++ /dev/null
@@ -1,147 +0,0 @@
-import os.path
-
-from c_analyzer_common import files
-from c_analyzer_common.info import UNKNOWN
-from c_parser import declarations, info
-from .info import Symbol
-from .source import _find_symbol
-
-
-# XXX need tests:
-# * look_up_known_symbol()
-# * symbol_from_source()
-# * get_resolver()
-# * symbols_to_variables()
-
-def look_up_known_symbol(symbol, knownvars, *,
- match_files=(lambda f1, f2: f1 == f2),
- ):
- """Return the known variable matching the given symbol.
-
- "knownvars" is a mapping of common.ID to parser.Variable.
-
- "match_files" is used to verify if two filenames point to
- the same file.
- """
- if not knownvars:
- return None
-
- if symbol.funcname == UNKNOWN:
- if not symbol.filename or symbol.filename == UNKNOWN:
- for varid in knownvars:
- if not varid.funcname:
- continue
- if varid.name == symbol.name:
- return knownvars[varid]
- else:
- return None
- else:
- for varid in knownvars:
- if not varid.funcname:
- continue
- if not match_files(varid.filename, symbol.filename):
- continue
- if varid.name == symbol.name:
- return knownvars[varid]
- else:
- return None
- elif not symbol.filename or symbol.filename == UNKNOWN:
- raise NotImplementedError
- else:
- return knownvars.get(symbol.id)
-
-
-def find_in_source(symbol, dirnames, *,
- _perfilecache={},
- _find_symbol=_find_symbol,
- _iter_files=files.iter_files_by_suffix,
- ):
- """Return the Variable matching the given Symbol.
-
- If there is no match then return None.
- """
- if symbol.filename and symbol.filename != UNKNOWN:
- filenames = [symbol.filename]
- else:
- filenames = _iter_files(dirnames, ('.c', '.h'))
-
- if symbol.funcname and symbol.funcname != UNKNOWN:
- raise NotImplementedError
-
- (filename, funcname, decl
- ) = _find_symbol(symbol.name, filenames, _perfilecache)
- if filename == UNKNOWN:
- return None
- return info.Variable.from_parts(filename, funcname, symbol.name, decl)
-
-
-def get_resolver(knownvars=None, dirnames=None, *,
- _look_up_known=look_up_known_symbol,
- _from_source=find_in_source,
- ):
- """Return a "resolver" func for the given known vars and dirnames.
-
- The func takes a single Symbol and returns a corresponding Variable.
- If the symbol was located then the variable will be valid, populated
- with the corresponding information. Otherwise None is returned.
- """
- if knownvars:
- knownvars = dict(knownvars) # a copy
- def resolve_known(symbol):
- found = _look_up_known(symbol, knownvars)
- if found is None:
- return None
- elif symbol.funcname == UNKNOWN:
- knownvars.pop(found.id)
- elif not symbol.filename or symbol.filename == UNKNOWN:
- knownvars.pop(found.id)
- return found
- if dirnames:
- def resolve(symbol):
- found = resolve_known(symbol)
- if found is None:
- return None
- #return _from_source(symbol, dirnames)
- else:
- for dirname in dirnames:
- if not dirname.endswith(os.path.sep):
- dirname += os.path.sep
- if found.filename.startswith(dirname):
- break
- else:
- return None
- return found
- else:
- resolve = resolve_known
- elif dirnames:
- def resolve(symbol):
- return _from_source(symbol, dirnames)
- else:
- def resolve(symbol):
- return None
- return resolve
-
-
-def symbols_to_variables(symbols, *,
- resolve=(lambda s: look_up_known_symbol(s, None)),
- ):
- """Yield the variable the matches each given symbol.
-
- Use get_resolver() for a "resolve" func to use.
- """
- for symbol in symbols:
- if isinstance(symbol, info.Variable):
- # XXX validate?
- yield symbol
- continue
- if symbol.kind != Symbol.KIND.VARIABLE:
- continue
- resolved = resolve(symbol)
- if resolved is None:
- #raise NotImplementedError(symbol)
- resolved = info.Variable(
- id=symbol.id,
- storage=UNKNOWN,
- vartype=UNKNOWN,
- )
- yield resolved
diff --git a/Tools/c-analyzer/c_symbols/source.py b/Tools/c-analyzer/c_symbols/source.py
deleted file mode 100644
index a724810..0000000
--- a/Tools/c-analyzer/c_symbols/source.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from c_analyzer_common import files
-from c_analyzer_common.info import UNKNOWN
-from c_parser import declarations
-
-
-# XXX need tests:
-# * find_symbol()
-
-def find_symbol(name, dirnames, *,
- _perfilecache,
- _iter_files=files.iter_files_by_suffix,
- **kwargs
- ):
- """Return (filename, funcname, vartype) for the matching Symbol."""
- filenames = _iter_files(dirnames, ('.c', '.h'))
- return _find_symbol(name, filenames, _perfilecache, **kwargs)
-
-
-def _get_symbols(filename, *,
- _iter_variables=declarations.iter_variables,
- ):
- """Return the list of Symbols found in the given file."""
- symbols = {}
- for funcname, name, vartype in _iter_variables(filename):
- if not funcname:
- continue
- try:
- instances = symbols[name]
- except KeyError:
- instances = symbols[name] = []
- instances.append((funcname, vartype))
- return symbols
-
-
-def _find_symbol(name, filenames, _perfilecache, *,
- _get_local_symbols=_get_symbols,
- ):
- for filename in filenames:
- try:
- symbols = _perfilecache[filename]
- except KeyError:
- symbols = _perfilecache[filename] = _get_local_symbols(filename)
-
- try:
- instances = symbols[name]
- except KeyError:
- continue
-
- funcname, vartype = instances.pop(0)
- if not instances:
- symbols.pop(name)
- return filename, funcname, vartype
- else:
- return UNKNOWN, UNKNOWN, UNKNOWN
-
-
-def iter_symbols():
- raise NotImplementedError
diff --git a/Tools/c-analyzer/c_globals/README b/Tools/c-analyzer/cpython/README
similarity index 100%
rename from Tools/c-analyzer/c_globals/README
rename to Tools/c-analyzer/cpython/README
diff --git a/Tools/c-analyzer/cpython/__init__.py b/Tools/c-analyzer/cpython/__init__.py
new file mode 100644
index 0000000..ae45b42
--- /dev/null
+++ b/Tools/c-analyzer/cpython/__init__.py
@@ -0,0 +1,29 @@
+import os.path
+import sys
+
+
+TOOL_ROOT = os.path.abspath(
+ os.path.dirname( # c-analyzer/
+ os.path.dirname(__file__))) # cpython/
+DATA_DIR = TOOL_ROOT
+REPO_ROOT = (
+ os.path.dirname( # ..
+ os.path.dirname(TOOL_ROOT))) # Tools/
+
+INCLUDE_DIRS = [os.path.join(REPO_ROOT, name) for name in [
+ 'Include',
+ ]]
+SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [
+ 'Python',
+ 'Parser',
+ 'Objects',
+ 'Modules',
+ ]]
+
+#PYTHON = os.path.join(REPO_ROOT, 'python')
+PYTHON = sys.executable
+
+
+# Clean up the namespace.
+del sys
+del os
diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/cpython/__main__.py
similarity index 68%
rename from Tools/c-analyzer/c_globals/__main__.py
rename to Tools/c-analyzer/cpython/__main__.py
index 9570fb6..6b0f9bc 100644
--- a/Tools/c-analyzer/c_globals/__main__.py
+++ b/Tools/c-analyzer/cpython/__main__.py
@@ -1,42 +1,42 @@
import argparse
-import os.path
import re
import sys
-from c_analyzer_common import SOURCE_DIRS, REPO_ROOT
-from c_analyzer_common.info import UNKNOWN
-from c_analyzer_common.known import (
+from c_analyzer.common import show
+from c_analyzer.common.info import UNKNOWN
+
+from . import SOURCE_DIRS
+from .find import supported_vars
+from .known import (
from_file as known_from_file,
DATA_FILE as KNOWN_FILE,
)
-from . import find, show
-from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object
-
-
-def _match_unused_global(variable, knownvars, used):
- found = []
- for varid in knownvars:
- if varid in used:
- continue
- if varid.funcname is not None:
- continue
- if varid.name != variable.name:
- continue
- if variable.filename and variable.filename != UNKNOWN:
- if variable.filename == varid.filename:
- found.append(varid)
- else:
- found.append(varid)
- return found
+from .supported import IGNORED_FILE
def _check_results(unknown, knownvars, used):
+ def _match_unused_global(variable):
+ found = []
+ for varid in knownvars:
+ if varid in used:
+ continue
+ if varid.funcname is not None:
+ continue
+ if varid.name != variable.name:
+ continue
+ if variable.filename and variable.filename != UNKNOWN:
+ if variable.filename == varid.filename:
+ found.append(varid)
+ else:
+ found.append(varid)
+ return found
+
badknown = set()
for variable in sorted(unknown):
msg = None
if variable.funcname != UNKNOWN:
msg = f'could not find global symbol {variable.id}'
- elif m := _match_unused_global(variable, knownvars, used):
+ elif m := _match_unused_global(variable):
assert isinstance(m, list)
badknown.update(m)
elif variable.name in ('completed', 'id'): # XXX Figure out where these variables are.
@@ -65,32 +65,29 @@
raise Exception('could not find all symbols')
-def _find_globals(dirnames, known, ignored):
- if dirnames == SOURCE_DIRS:
- dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames]
-
- ignored = ignored_from_file(ignored)
- known = known_from_file(known)
+# XXX Move this check to its own command.
+def cmd_check_cache(cmd, *,
+ known=KNOWN_FILE,
+ ignored=IGNORED_FILE,
+ _known_from_file=known_from_file,
+ _find=supported_vars,
+ ):
+ known = _known_from_file(known)
used = set()
unknown = set()
- knownvars = (known or {}).get('variables')
- for variable in find.globals_from_binary(knownvars=knownvars,
- dirnames=dirnames):
- #for variable in find.globals(dirnames, known, kind='platform'):
- if variable.vartype == UNKNOWN:
- unknown.add(variable)
+ for var, supported in _find(known=known, ignored=ignored):
+ if supported is None:
+ unknown.add(var)
continue
- yield variable, is_supported(variable, ignored, known)
- used.add(variable.id)
-
- #_check_results(unknown, knownvars, used)
+ used.add(var.id)
+ _check_results(unknown, known['variables'], used)
-def cmd_check(cmd, dirs=SOURCE_DIRS, *,
- ignored=IGNORED_FILE,
+def cmd_check(cmd, *,
known=KNOWN_FILE,
- _find=_find_globals,
+ ignored=IGNORED_FILE,
+ _find=supported_vars,
_show=show.basic,
_print=print,
):
@@ -100,7 +97,11 @@
In the failure case, the list of unsupported variables
will be printed out.
"""
- unsupported = [v for v, s in _find(dirs, known, ignored) if not s]
+ unsupported = []
+ for var, supported in _find(known=known, ignored=ignored):
+ if not supported:
+ unsupported.append(var)
+
if not unsupported:
#_print('okay')
return
@@ -112,11 +113,11 @@
sys.exit(1)
-def cmd_show(cmd, dirs=SOURCE_DIRS, *,
- ignored=IGNORED_FILE,
+def cmd_show(cmd, *,
known=KNOWN_FILE,
+ ignored=IGNORED_FILE,
skip_objects=False,
- _find=_find_globals,
+ _find=supported_vars,
_show=show.basic,
_print=print,
):
@@ -127,10 +128,12 @@
"""
allsupported = []
allunsupported = []
- for found, supported in _find(dirs, known, ignored):
- if skip_objects: # XXX Support proper filters instead.
- if _is_object(found.vartype):
- continue
+ for found, supported in _find(known=known,
+ ignored=ignored,
+ skip_objects=skip_objects,
+ ):
+ if supported is None:
+ continue
(allsupported if supported else allunsupported
).append(found)
@@ -165,9 +168,9 @@
common.add_argument('--known', metavar='FILE',
default=KNOWN_FILE,
help='path to file that lists known types')
- common.add_argument('dirs', metavar='DIR', nargs='*',
- default=SOURCE_DIRS,
- help='a directory to check')
+ #common.add_argument('dirs', metavar='DIR', nargs='*',
+ # default=SOURCE_DIRS,
+ # help='a directory to check')
parser = argparse.ArgumentParser(
prog=prog,
diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/cpython/_generate.py
similarity index 97%
rename from Tools/c-analyzer/c_analyzer_common/_generate.py
rename to Tools/c-analyzer/cpython/_generate.py
index 9b2fc9e..4c340ac 100644
--- a/Tools/c-analyzer/c_analyzer_common/_generate.py
+++ b/Tools/c-analyzer/cpython/_generate.py
@@ -1,15 +1,16 @@
# The code here consists of hacks for pre-populating the known.tsv file.
-from c_parser.preprocessor import _iter_clean_lines
-from c_parser.naive import (
+from c_analyzer.parser.preprocessor import _iter_clean_lines
+from c_analyzer.parser.naive import (
iter_variables, parse_variable_declaration, find_variables,
)
-from c_parser.info import Variable
+from c_analyzer.common.known import HEADER as KNOWN_HEADER
+from c_analyzer.common.info import UNKNOWN, ID
+from c_analyzer.variables import Variable
+from c_analyzer.util import write_tsv
from . import SOURCE_DIRS, REPO_ROOT
-from .known import DATA_FILE as KNOWN_FILE, HEADER as KNOWN_HEADER
-from .info import UNKNOWN, ID
-from .util import write_tsv
+from .known import DATA_FILE as KNOWN_FILE
from .files import iter_cpython_files
diff --git a/Tools/c-analyzer/cpython/files.py b/Tools/c-analyzer/cpython/files.py
new file mode 100644
index 0000000..543097a
--- /dev/null
+++ b/Tools/c-analyzer/cpython/files.py
@@ -0,0 +1,29 @@
+from c_analyzer.common.files import (
+ C_SOURCE_SUFFIXES, walk_tree, iter_files_by_suffix,
+ )
+
+from . import SOURCE_DIRS, REPO_ROOT
+
+# XXX need tests:
+# * iter_files()
+
+
+def iter_files(*,
+ walk=walk_tree,
+ _files=iter_files_by_suffix,
+ ):
+ """Yield each file in the tree for each of the given directory names."""
+ excludedtrees = [
+ os.path.join('Include', 'cpython', ''),
+ ]
+ def is_excluded(filename):
+ for root in excludedtrees:
+ if filename.startswith(root):
+ return True
+ return False
+ for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT,
+ walk=walk,
+ ):
+ if is_excluded(filename):
+ continue
+ yield filename
diff --git a/Tools/c-analyzer/cpython/find.py b/Tools/c-analyzer/cpython/find.py
new file mode 100644
index 0000000..a7bc0b4
--- /dev/null
+++ b/Tools/c-analyzer/cpython/find.py
@@ -0,0 +1,101 @@
+import os.path
+
+from c_analyzer.common import files
+from c_analyzer.common.info import UNKNOWN, ID
+from c_analyzer.variables import find as _common
+
+from . import SOURCE_DIRS, PYTHON, REPO_ROOT
+from .known import (
+ from_file as known_from_file,
+ DATA_FILE as KNOWN_FILE,
+ )
+from .supported import (
+ ignored_from_file, IGNORED_FILE, is_supported, _is_object,
+ )
+
+# XXX need tests:
+# * vars_from_binary()
+# * vars_from_source()
+# * supported_vars()
+
+
+def _handle_id(filename, funcname, name, *,
+ _relpath=os.path.relpath,
+ ):
+ filename = _relpath(filename, REPO_ROOT)
+ return ID(filename, funcname, name)
+
+
+def vars_from_binary(*,
+ known=KNOWN_FILE,
+ _known_from_file=known_from_file,
+ _iter_files=files.iter_files_by_suffix,
+ _iter_vars=_common.vars_from_binary,
+ ):
+ """Yield a Variable for each found Symbol.
+
+ Details are filled in from the given "known" variables and types.
+ """
+ if isinstance(known, str):
+ known = _known_from_file(known)
+ dirnames = SOURCE_DIRS
+ suffixes = ('.c',)
+ filenames = _iter_files(dirnames, suffixes)
+ # XXX For now we only use known variables (no source lookup).
+ filenames = None
+ yield from _iter_vars(PYTHON,
+ known=known,
+ filenames=filenames,
+ handle_id=_handle_id,
+ check_filename=(lambda n: True),
+ )
+
+
+def vars_from_source(*,
+ preprocessed=None,
+ known=KNOWN_FILE,
+ _known_from_file=known_from_file,
+ _iter_files=files.iter_files_by_suffix,
+ _iter_vars=_common.vars_from_source,
+ ):
+ """Yield a Variable for each declaration in the raw source code.
+
+ Details are filled in from the given "known" variables and types.
+ """
+ if isinstance(known, str):
+ known = _known_from_file(known)
+ dirnames = SOURCE_DIRS
+ suffixes = ('.c',)
+ filenames = _iter_files(dirnames, suffixes)
+ yield from _iter_vars(filenames,
+ preprocessed=preprocessed,
+ known=known,
+ handle_id=_handle_id,
+ )
+
+
+def supported_vars(*,
+ known=KNOWN_FILE,
+ ignored=IGNORED_FILE,
+ skip_objects=False,
+ _known_from_file=known_from_file,
+ _ignored_from_file=ignored_from_file,
+ _iter_vars=vars_from_binary,
+ _is_supported=is_supported,
+ ):
+ """Yield (var, is supported) for each found variable."""
+ if isinstance(known, str):
+ known = _known_from_file(known)
+ if isinstance(ignored, str):
+ ignored = _ignored_from_file(ignored)
+
+ for var in _iter_vars(known=known):
+ if not var.isglobal:
+ continue
+ elif var.vartype == UNKNOWN:
+ yield var, None
+ # XXX Support proper filters instead.
+ elif skip_objects and _is_object(found.vartype):
+ continue
+ else:
+ yield var, _is_supported(var, ignored, known)
diff --git a/Tools/c-analyzer/cpython/known.py b/Tools/c-analyzer/cpython/known.py
new file mode 100644
index 0000000..c3cc2c0
--- /dev/null
+++ b/Tools/c-analyzer/cpython/known.py
@@ -0,0 +1,66 @@
+import csv
+import os.path
+
+from c_analyzer.parser.declarations import extract_storage
+from c_analyzer.variables import known as _common
+from c_analyzer.variables.info import Variable
+
+from . import DATA_DIR
+
+
+# XXX need tests:
+# * from_file()
+# * look_up_variable()
+
+
+DATA_FILE = os.path.join(DATA_DIR, 'known.tsv')
+
+
+def _get_storage(decl, infunc):
+ # statics
+ if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
+ return 'static'
+ if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')):
+ return 'static'
+ if decl.startswith('PyDoc_VAR('):
+ return 'static'
+ if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
+ return 'static'
+ if decl.startswith('WRAP_METHOD('):
+ return 'static'
+ # public extern
+ if decl.startswith('PyAPI_DATA('):
+ return 'extern'
+ # Fall back to the normal handler.
+ return extract_storage(decl, infunc=infunc)
+
+
+def _handle_var(varid, decl):
+# if varid.name == 'id' and decl == UNKNOWN:
+# # None of these are variables.
+# decl = 'int id';
+ storage = _get_storage(decl, varid.funcname)
+ return Variable(varid, storage, decl)
+
+
+def from_file(infile=DATA_FILE, *,
+ _from_file=_common.from_file,
+ _handle_var=_handle_var,
+ ):
+ """Return the info for known declarations in the given file."""
+ return _from_file(infile, handle_var=_handle_var)
+
+
+def look_up_variable(varid, knownvars, *,
+ _lookup=_common.look_up_variable,
+ ):
+ """Return the known variable matching the given ID.
+
+ "knownvars" is a mapping of ID to Variable.
+
+ "match_files" is used to verify if two filenames point to
+ the same file.
+
+ If no match is found then None is returned.
+ """
+ return _lookup(varid, knownvars)
diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/cpython/supported.py
similarity index 97%
rename from Tools/c-analyzer/c_globals/supported.py
rename to Tools/c-analyzer/cpython/supported.py
index d185daa..18786ee 100644
--- a/Tools/c-analyzer/c_globals/supported.py
+++ b/Tools/c-analyzer/cpython/supported.py
@@ -1,9 +1,13 @@
import os.path
import re
-from c_analyzer_common import DATA_DIR
-from c_analyzer_common.info import ID
-from c_analyzer_common.util import read_tsv, write_tsv
+from c_analyzer.common.info import ID
+from c_analyzer.common.util import read_tsv, write_tsv
+
+from . import DATA_DIR
+
+# XXX need tests:
+# * generate / script
IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv')
@@ -379,11 +383,12 @@
if __name__ == '__main__':
- from c_analyzer_common import SOURCE_DIRS
- from c_analyzer_common.known import (
+ from cpython import SOURCE_DIRS
+ from cpython.known import (
from_file as known_from_file,
DATA_FILE as KNOWN_FILE,
)
+ # XXX This is wrong!
from . import find
known = known_from_file(KNOWN_FILE)
knownvars = (known or {}).get('variables')