bpo-36876: Small adjustments to the C-analyzer tool. (GH-23045)

This is a little bit of clean-up, small fixes, and additional helpers prior to building an updated & accurate list of globals to eliminate.

Author: ericsnowcurrently

Tools/c-analyzer/c_analyzer/__init__.py

@@ -4,10 +4,12 @@
 from c_parser.info import (
     KIND,
     TypeDeclaration,
-    filter_by_kind,
-    collate_by_kind_group,
     resolve_parsed,
 )
+from c_parser.match import (
+    filter_by_kind,
+    group_by_kinds,
+)
 from . import (
     analyze as _analyze,
     datafiles as _datafiles,
@@ -55,7 +57,7 @@ def analyze_decls(decls, known, *,
     )
 
     decls = list(decls)
-    collated = collate_by_kind_group(decls)
+    collated = group_by_kinds(decls)
 
     types = {decl: None for decl in collated['type']}
     typespecs = _analyze.get_typespecs(types)

Tools/c-analyzer/c_analyzer/__main__.py

@@ -1,5 +1,6 @@
 import io
 import logging
+import os
 import os.path
 import re
 import sys
@@ -9,6 +10,7 @@
     add_verbosity_cli,
     add_traceback_cli,
     add_sepval_cli,
+    add_progress_cli,
     add_files_cli,
     add_commands_cli,
     process_args_by_key,
@@ -17,11 +19,13 @@
     filter_filenames,
     iter_marks,
 )
-from c_parser.info import KIND, is_type_decl
+from c_parser.info import KIND
+from c_parser.match import is_type_decl
+from .match import filter_forward
 from . import (
     analyze as _analyze,
-    check_all as _check_all,
     datafiles as _datafiles,
+    check_all as _check_all,
 )
 
 
@@ -44,7 +48,7 @@
 TABLE_SECTIONS = {
     'types': (
         ['kind', 'name', 'data', 'file'],
-        is_type_decl,
+        KIND.is_type_decl,
         (lambda v: (v.kind.value, v.filename or '', v.name)),
     ),
     'typedefs': 'types',
@@ -167,9 +171,7 @@ def handle_failure(failure, data):
             print(f'{data.filename}:{name} - {failure}')
     elif fmt == 'summary':
         def handle_failure(failure, data):
-            parent = data.parent or ''
-            funcname = parent if isinstance(parent, str) else parent.name
-            print(f'{data.filename:35}\t{funcname or "-":35}\t{data.name:40}\t{failure}')
+            print(_fmt_one_summary(data, failure))
     elif fmt == 'full':
         div = ''
         def handle_failure(failure, data):
@@ -230,6 +232,15 @@ def section(name):
     yield f'grand total: {total}'
 
 
+def _fmt_one_summary(item, extra=None):
+    parent = item.parent or ''
+    funcname = parent if isinstance(parent, str) else parent.name
+    if extra:
+        return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
+    else:
+        return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'
+
+
 def fmt_full(analysis):
     # XXX Support sorting.
     items = sorted(analysis, key=lambda v: v.key)
@@ -272,10 +283,12 @@ def process_checks(args):
             args.checks = [check]
     else:
         process_checks = add_checks_cli(parser, checks=checks)
+    process_progress = add_progress_cli(parser)
     process_output = add_output_cli(parser, default=None)
     process_files = add_files_cli(parser, **kwargs)
     return [
         process_checks,
+        process_progress,
         process_output,
         process_files,
     ]
@@ -288,6 +301,7 @@ def cmd_check(filenames, *,
               relroot=None,
               failfast=False,
               iter_filenames=None,
+              track_progress=None,
               verbosity=VERBOSITY,
               _analyze=_analyze,
               _CHECKS=CHECKS,
@@ -304,36 +318,53 @@ def cmd_check(filenames, *,
      ) = _get_check_handlers(fmt, printer, verbosity)
 
     filenames = filter_filenames(filenames, iter_filenames)
+    if track_progress:
+        filenames = track_progress(filenames)
 
-    logger.info('analyzing...')
+    logger.info('analyzing files...')
     analyzed = _analyze(filenames, **kwargs)
     if relroot:
         analyzed.fix_filenames(relroot)
+    decls = filter_forward(analyzed, markpublic=True)
 
-    logger.info('checking...')
-    numfailed = 0
-    for data, failure in _check_all(analyzed, checks, failfast=failfast):
+    logger.info('checking analysis results...')
+    failed = []
+    for data, failure in _check_all(decls, checks, failfast=failfast):
         if data is None:
             printer.info('stopping after one failure')
             break
-        if div is not None and numfailed > 0:
+        if div is not None and len(failed) > 0:
             printer.info(div)
-        numfailed += 1
+        failed.append(data)
         handle_failure(failure, data)
     handle_after()
 
     printer.info('-------------------------')
-    logger.info(f'total failures: {numfailed}')
+    logger.info(f'total failures: {len(failed)}')
     logger.info('done checking')
 
-    if numfailed > 0:
-        sys.exit(numfailed)
+    if fmt == 'summary':
+        print('Categorized by storage:')
+        print()
+        from .match import group_by_storage
+        grouped = group_by_storage(failed, ignore_non_match=False)
+        for group, decls in grouped.items():
+            print()
+            print(group)
+            for decl in decls:
+                print(' ', _fmt_one_summary(decl))
+            print(f'subtotal: {len(decls)}')
+
+    if len(failed) > 0:
+        sys.exit(len(failed))
 
 
 def _cli_analyze(parser, **kwargs):
+    process_progress = add_progress_cli(parser)
     process_output = add_output_cli(parser)
     process_files = add_files_cli(parser, **kwargs)
     return [
+        process_progress,
         process_output,
         process_files,
     ]
@@ -343,6 +374,7 @@ def _cli_analyze(parser, **kwargs):
 def cmd_analyze(filenames, *,
                 fmt=None,
                 iter_filenames=None,
+                track_progress=None,
                 verbosity=None,
                 _analyze=_analyze,
                 formats=FORMATS,
@@ -356,56 +388,54 @@ def cmd_analyze(filenames, *,
         raise ValueError(f'unsupported fmt {fmt!r}')
 
     filenames = filter_filenames(filenames, iter_filenames)
-    if verbosity == 2:
-        def iter_filenames(filenames=filenames):
-            marks = iter_marks()
-            for filename in filenames:
-                print(next(marks), end='')
-                yield filename
-        filenames = iter_filenames()
-    elif verbosity > 2:
-        def iter_filenames(filenames=filenames):
-            for filename in filenames:
-                print(f'<{filename}>')
-                yield filename
-        filenames = iter_filenames()
-
-    logger.info('analyzing...')
+    if track_progress:
+        filenames = track_progress(filenames)
+
+    logger.info('analyzing files...')
     analyzed = _analyze(filenames, **kwargs)
+    decls = filter_forward(analyzed, markpublic=True)
 
-    for line in do_fmt(analyzed):
+    for line in do_fmt(decls):
         print(line)
 
 
 def _cli_data(parser, filenames=None, known=None):
     ArgumentParser = type(parser)
     common = ArgumentParser(add_help=False)
-    if filenames is None:
-        common.add_argument('filenames', metavar='FILE', nargs='+')
+    # These flags will get processed by the top-level parse_args().
+    add_verbosity_cli(common)
+    add_traceback_cli(common)
 
     subs = parser.add_subparsers(dest='datacmd')
 
     sub = subs.add_parser('show', parents=[common])
     if known is None:
         sub.add_argument('--known', required=True)
+    if filenames is None:
+        sub.add_argument('filenames', metavar='FILE', nargs='+')
 
-    sub = subs.add_parser('dump')
+    sub = subs.add_parser('dump', parents=[common])
     if known is None:
         sub.add_argument('--known')
     sub.add_argument('--show', action='store_true')
+    process_progress = add_progress_cli(sub)
 
-    sub = subs.add_parser('check')
+    sub = subs.add_parser('check', parents=[common])
     if known is None:
         sub.add_argument('--known', required=True)
 
-    return None
+    def process_args(args):
+        if args.datacmd == 'dump':
+            process_progress(args)
+    return process_args
 
 
 def cmd_data(datacmd, filenames, known=None, *,
              _analyze=_analyze,
              formats=FORMATS,
              extracolumns=None,
              relroot=None,
+             track_progress=None,
              **kwargs
              ):
     kwargs.pop('verbosity', None)
@@ -417,6 +447,8 @@ def cmd_data(datacmd, filenames, known=None, *,
         for line in do_fmt(known):
             print(line)
     elif datacmd == 'dump':
+        if track_progress:
+            filenames = track_progress(filenames)
         analyzed = _analyze(filenames, **kwargs)
         if known is None or usestdout:
             outfile = io.StringIO()

Tools/c-analyzer/c_analyzer/analyze.py

@@ -3,15 +3,19 @@
     TypeDeclaration,
     POTSType,
     FuncPtr,
+)
+from c_parser.match import (
     is_pots,
     is_funcptr,
 )
 from .info import (
     IGNORED,
     UNKNOWN,
-    is_system_type,
     SystemType,
 )
+from .match import (
+    is_system_type,
+)
 
 
 def get_typespecs(typedecls):

Tools/c-analyzer/c_analyzer/datafiles.py

@@ -1,5 +1,6 @@
 import c_common.tables as _tables
 import c_parser.info as _info
+import c_parser.match as _match
 import c_parser.datafiles as _parser
 from . import analyze as _analyze
 
@@ -17,7 +18,7 @@ def analyze_known(known, *,
                   handle_unresolved=True,
                   ):
     knowntypes = knowntypespecs = {}
-    collated = _info.collate_by_kind_group(known)
+    collated = _match.group_by_kinds(known)
     types = {decl: None for decl in collated['type']}
     typespecs = _analyze.get_typespecs(types)
     def analyze_decl(decl):

Tools/c-analyzer/c_analyzer/info.py

@@ -7,7 +7,11 @@
     HighlevelParsedItem,
     Declaration,
     TypeDeclaration,
+)
+from c_parser.match import (
     is_type_decl,
+)
+from .match import (
     is_process_global,
 )
 
@@ -16,44 +20,6 @@
 UNKNOWN = _misc.Labeled('UNKNOWN')
 
 
-# XXX Use known.tsv for these?
-SYSTEM_TYPES = {
-    'int8_t',
-    'uint8_t',
-    'int16_t',
-    'uint16_t',
-    'int32_t',
-    'uint32_t',
-    'int64_t',
-    'uint64_t',
-    'size_t',
-    'ssize_t',
-    'intptr_t',
-    'uintptr_t',
-    'wchar_t',
-    '',
-    # OS-specific
-    'pthread_cond_t',
-    'pthread_mutex_t',
-    'pthread_key_t',
-    'atomic_int',
-    'atomic_uintptr_t',
-    '',
-    # lib-specific
-    'WINDOW',  # curses
-    'XML_LChar',
-    'XML_Size',
-    'XML_Parser',
-    'enum XML_Error',
-    'enum XML_Status',
-    '',
-}
-
-
-def is_system_type(typespec):
-    return typespec in SYSTEM_TYPES
-
-
 class SystemType(TypeDeclaration):
 
     def __init__(self, name):