bpo-36876: [c-analyzer tool] Tighten up the results and output. (GH-23431)

We also update the "ignored" file with a temporary list of all known globals.

Author: ericsnowcurrently

Tools/c-analyzer/c_analyzer/__main__.py

@@ -5,6 +5,7 @@
 import re
 import sys
 
+from c_common import fsutil
 from c_common.logging import VERBOSITY, Printer
 from c_common.scriptutil import (
     add_verbosity_cli,
@@ -298,9 +299,9 @@ def cmd_check(filenames, *,
               checks=None,
               ignored=None,
               fmt=None,
-              relroot=None,
               failfast=False,
               iter_filenames=None,
+              relroot=fsutil.USE_CWD,
               track_progress=None,
               verbosity=VERBOSITY,
               _analyze=_analyze,
@@ -317,14 +318,14 @@ def cmd_check(filenames, *,
     (handle_failure, handle_after, div
      ) = _get_check_handlers(fmt, printer, verbosity)
 
-    filenames = filter_filenames(filenames, iter_filenames)
+    filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
+    filenames = filter_filenames(filenames, iter_filenames, relroot)
     if track_progress:
         filenames = track_progress(filenames)
 
     logger.info('analyzing files...')
     analyzed = _analyze(filenames, **kwargs)
-    if relroot:
-        analyzed.fix_filenames(relroot)
+    analyzed.fix_filenames(relroot, normalize=False)
     decls = filter_forward(analyzed, markpublic=True)
 
     logger.info('checking analysis results...')
@@ -374,6 +375,7 @@ def _cli_analyze(parser, **kwargs):
 def cmd_analyze(filenames, *,
                 fmt=None,
                 iter_filenames=None,
+                relroot=fsutil.USE_CWD,
                 track_progress=None,
                 verbosity=None,
                 _analyze=_analyze,
@@ -387,12 +389,14 @@ def cmd_analyze(filenames, *,
     except KeyError:
         raise ValueError(f'unsupported fmt {fmt!r}')
 
-    filenames = filter_filenames(filenames, iter_filenames)
+    filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
+    filenames = filter_filenames(filenames, iter_filenames, relroot)
     if track_progress:
         filenames = track_progress(filenames)
 
     logger.info('analyzing files...')
     analyzed = _analyze(filenames, **kwargs)
+    analyzed.fix_filenames(relroot, normalize=False)
     decls = filter_forward(analyzed, markpublic=True)
 
     for line in do_fmt(decls):
@@ -434,7 +438,7 @@ def cmd_data(datacmd, filenames, known=None, *,
              _analyze=_analyze,
              formats=FORMATS,
              extracolumns=None,
-             relroot=None,
+             relroot=fsutil.USE_CWD,
              track_progress=None,
              **kwargs
              ):
@@ -447,9 +451,11 @@ def cmd_data(datacmd, filenames, known=None, *,
         for line in do_fmt(known):
             print(line)
     elif datacmd == 'dump':
+        filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
         if track_progress:
             filenames = track_progress(filenames)
         analyzed = _analyze(filenames, **kwargs)
+        analyzed.fix_filenames(relroot, normalize=False)
         if known is None or usestdout:
             outfile = io.StringIO()
             _datafiles.write_known(analyzed, outfile, extracolumns,

Tools/c-analyzer/c_analyzer/datafiles.py

@@ -1,3 +1,6 @@
+import os.path
+
+from c_common import fsutil
 import c_common.tables as _tables
 import c_parser.info as _info
 import c_parser.match as _match
@@ -13,31 +16,10 @@
 ]
 
 
-def analyze_known(known, *,
-                  analyze_resolved=None,
-                  handle_unresolved=True,
-                  ):
-    knowntypes = knowntypespecs = {}
-    collated = _match.group_by_kinds(known)
-    types = {decl: None for decl in collated['type']}
-    typespecs = _analyze.get_typespecs(types)
-    def analyze_decl(decl):
-        return _analyze.analyze_decl(
-            decl,
-            typespecs,
-            knowntypespecs,
-            types,
-            knowntypes,
-            analyze_resolved=analyze_resolved,
-        )
-    _analyze.analyze_type_decls(types, analyze_decl, handle_unresolved)
-    return types, typespecs
-
-
 def get_known(known, extracolumns=None, *,
               analyze_resolved=None,
               handle_unresolved=True,
-              relroot=None,
+              relroot=fsutil.USE_CWD,
               ):
     if isinstance(known, str):
         known = read_known(known, extracolumns, relroot)
@@ -48,7 +30,7 @@ def get_known(known, extracolumns=None, *,
     )
 
 
-def read_known(infile, extracolumns=None, relroot=None):
+def read_known(infile, extracolumns=None, relroot=fsutil.USE_CWD):
     extracolumns = EXTRA_COLUMNS + (
         list(extracolumns) if extracolumns else []
     )
@@ -58,8 +40,29 @@ def read_known(infile, extracolumns=None, relroot=None):
     return known
 
 
+def analyze_known(known, *,
+                  analyze_resolved=None,
+                  handle_unresolved=True,
+                  ):
+    knowntypes = knowntypespecs = {}
+    collated = _match.group_by_kinds(known)
+    types = {decl: None for decl in collated['type']}
+    typespecs = _analyze.get_typespecs(types)
+    def analyze_decl(decl):
+        return _analyze.analyze_decl(
+            decl,
+            typespecs,
+            knowntypespecs,
+            types,
+            knowntypes,
+            analyze_resolved=analyze_resolved,
+        )
+    _analyze.analyze_type_decls(types, analyze_decl, handle_unresolved)
+    return types, typespecs
+
+
 def write_known(rows, outfile, extracolumns=None, *,
-                relroot=None,
+                relroot=fsutil.USE_CWD,
                 backup=True,
                 ):
     extracolumns = EXTRA_COLUMNS + (
@@ -86,22 +89,34 @@ def write_known(rows, outfile, extracolumns=None, *,
 IGNORED_HEADER = '\t'.join(IGNORED_COLUMNS)
 
 
-def read_ignored(infile):
-    return dict(_iter_ignored(infile))
+def read_ignored(infile, relroot=fsutil.USE_CWD):
+    return dict(_iter_ignored(infile, relroot))
 
 
-def _iter_ignored(infile):
+def _iter_ignored(infile, relroot):
+    if relroot and relroot is not fsutil.USE_CWD:
+        relroot = os.path.abspath(relroot)
+    bogus = {_tables.EMPTY, _tables.UNKNOWN}
     for row in _tables.read_table(infile, IGNORED_HEADER, sep='\t'):
         *varidinfo, reason = row
+        if _tables.EMPTY in varidinfo or _tables.UNKNOWN in varidinfo:
+            varidinfo = tuple(None if v in bogus else v
+                              for v in varidinfo)
+        if reason in bogus:
+            reason = None
         varid = _info.DeclID.from_row(varidinfo)
+        varid = varid.fix_filename(relroot, formatted=False, fixroot=False)
         yield varid, reason
 
 
-def write_ignored(variables, outfile):
+def write_ignored(variables, outfile, relroot=fsutil.USE_CWD):
     raise NotImplementedError
+    if relroot and relroot is not fsutil.USE_CWD:
+        relroot = os.path.abspath(relroot)
     reason = '???'
     #if not isinstance(varid, DeclID):
     #    varid = getattr(varid, 'parsed', varid).id
+    decls = (d.fix_filename(relroot, fixroot=False) for d in decls)
     _tables.write_table(
         outfile,
         IGNORED_HEADER,

Tools/c-analyzer/c_analyzer/info.py

@@ -1,5 +1,7 @@
 from collections import namedtuple
+import os.path
 
+from c_common import fsutil
 from c_common.clsutil import classonly
 import c_common.misc as _misc
 from c_parser.info import (
@@ -223,8 +225,9 @@ def is_known(self):
         else:
             return UNKNOWN not in self.typedecl
 
-    def fix_filename(self, relroot):
-        self.item.fix_filename(relroot)
+    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
+        self.item.fix_filename(relroot, **kwargs)
+        return self
 
     def as_rowdata(self, columns=None):
         # XXX finsih!
@@ -309,9 +312,11 @@ def __getitem__(self, key):
         else:
             return self._analyzed[key]
 
-    def fix_filenames(self, relroot):
+    def fix_filenames(self, relroot=fsutil.USE_CWD, **kwargs):
+        if relroot and relroot is not fsutil.USE_CWD:
+            relroot = os.path.abspath(relroot)
         for item in self._analyzed:
-            item.fix_filename(relroot)
+            item.fix_filename(relroot, fixroot=False, **kwargs)
 
     def _add_result(self, info, resolved):
         analyzed = type(self).build_item(info, resolved)

Tools/c-analyzer/c_common/fsutil.py

@@ -8,6 +8,9 @@
 from .iterutil import iter_many
 
 
+USE_CWD = object()
+
+
 C_SOURCE_SUFFIXES = ('.c', '.h')
 
 
@@ -29,6 +32,78 @@ def create_backup(old, backup=None):
     return backup
 
 
+##################################
+# filenames
+
+def fix_filename(filename, relroot=USE_CWD, *,
+                 fixroot=True,
+                 _badprefix=f'..{os.path.sep}',
+                 ):
+    """Return a normalized, absolute-path copy of the given filename."""
+    if not relroot or relroot is USE_CWD:
+        return os.path.abspath(filename)
+    if fixroot:
+        relroot = os.path.abspath(relroot)
+    return _fix_filename(filename, relroot)
+
+
+def _fix_filename(filename, relroot, *,
+                  _badprefix=f'..{os.path.sep}',
+                  ):
+    orig = filename
+
+    # First we normalize.
+    filename = os.path.normpath(filename)
+    if filename.startswith(_badprefix):
+        raise ValueError(f'bad filename {orig!r} (resolves beyond relative root')
+
+    # Now make sure it is absolute (relative to relroot).
+    if not os.path.isabs(filename):
+        filename = os.path.join(relroot, filename)
+    else:
+        relpath = os.path.relpath(filename, relroot)
+        if os.path.join(relroot, relpath) != filename:
+            raise ValueError(f'expected {relroot!r} as lroot, got {orig!r}')
+
+    return filename
+
+
+def fix_filenames(filenames, relroot=USE_CWD):
+    if not relroot or relroot is USE_CWD:
+        filenames = (os.path.abspath(v) for v in filenames)
+    else:
+        relroot = os.path.abspath(relroot)
+        filenames = (_fix_filename(v, relroot) for v in filenames)
+    return filenames, relroot
+
+
+def format_filename(filename, relroot=USE_CWD, *,
+                    fixroot=True,
+                    normalize=True,
+                    _badprefix=f'..{os.path.sep}',
+                    ):
+    """Return a consistent relative-path representation of the filename."""
+    orig = filename
+    if normalize:
+        filename = os.path.normpath(filename)
+    if relroot is None:
+        # Otherwise leave it as-is.
+        return filename
+    elif relroot is USE_CWD:
+        # Make it relative to CWD.
+        filename = os.path.relpath(filename)
+    else:
+        # Make it relative to "relroot".
+        if fixroot:
+            relroot = os.path.abspath(relroot)
+        elif not relroot:
+            raise ValueError('missing relroot')
+        filename = os.path.relpath(filename, relroot)
+    if filename.startswith(_badprefix):
+        raise ValueError(f'bad filename {orig!r} (resolves beyond relative root')
+    return filename
+
+
 ##################################
 # find files
 
@@ -54,34 +129,29 @@ def match_glob(filename, pattern):
     return fnmatch.fnmatch(filename, pattern.replace('**/', '', 1))
 
 
-def iter_filenames(filenames, *,
-                   start=None,
-                   include=None,
-                   exclude=None,
-                   ):
+def process_filenames(filenames, *,
+                      start=None,
+                      include=None,
+                      exclude=None,
+                      relroot=USE_CWD,
+                      ):
+    if relroot and relroot is not USE_CWD:
+        relroot = os.path.abspath(relroot)
+    if start:
+        start = fix_filename(start, relroot, fixroot=False)
+    if include:
+        include = set(fix_filename(v, relroot, fixroot=False)
+                      for v in include)
+    if exclude:
+        exclude = set(fix_filename(v, relroot, fixroot=False)
+                      for v in exclude)
+
     onempty = Exception('no filenames provided')
     for filename, solo in iter_many(filenames, onempty):
+        filename = fix_filename(filename, relroot, fixroot=False)
+        relfile = format_filename(filename, relroot, fixroot=False, normalize=False)
         check, start = _get_check(filename, start, include, exclude)
-        yield filename, check, solo
-#    filenames = iter(filenames or ())
-#    try:
-#        first = next(filenames)
-#    except StopIteration:
-#        raise Exception('no filenames provided')
-#    try:
-#        second = next(filenames)
-#    except StopIteration:
-#        check, _ = _get_check(first, start, include, exclude)
-#        yield first, check, False
-#        return
-#
-#    check, start = _get_check(first, start, include, exclude)
-#    yield first, check, True
-#    check, start = _get_check(second, start, include, exclude)
-#    yield second, check, True
-#    for filename in filenames:
-#        check, start = _get_check(filename, start, include, exclude)
-#        yield filename, check, True
+        yield filename, relfile, check, solo
 
 
 def expand_filenames(filenames):