From 77d1750471e5f3cebc8f251ebca997162feff4d2 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 15:04:37 -0500 Subject: [PATCH 01/13] Add a tool for finding undocumented C API --- Tools/c-api-docs-check/ignored_c_api.txt | 100 +++++++++++++++++ Tools/c-api-docs-check/main.py | 137 +++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 Tools/c-api-docs-check/ignored_c_api.txt create mode 100644 Tools/c-api-docs-check/main.py diff --git a/Tools/c-api-docs-check/ignored_c_api.txt b/Tools/c-api-docs-check/ignored_c_api.txt new file mode 100644 index 00000000000000..24a25626dcf797 --- /dev/null +++ b/Tools/c-api-docs-check/ignored_c_api.txt @@ -0,0 +1,100 @@ +# descrobject.h +PyClassMethodDescr_Type +PyDictProxy_Type +PyGetSetDescr_Type +PyMemberDescr_Type +PyMethodDescr_Type +PyWrapperDescr_Type +# pydtrace_probes.h +PyDTrace_AUDIT +PyDTrace_FUNCTION_ENTRY +PyDTrace_FUNCTION_RETURN +PyDTrace_GC_DONE +PyDTrace_GC_START +PyDTrace_IMPORT_FIND_LOAD_DONE +PyDTrace_IMPORT_FIND_LOAD_START +PyDTrace_INSTANCE_DELETE_DONE +PyDTrace_INSTANCE_DELETE_START +PyDTrace_INSTANCE_NEW_DONE +PyDTrace_INSTANCE_NEW_START +PyDTrace_LINE +# fileobject.h +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_HasFileSystemDefaultEncoding +Py_UTF8Mode +# pyhash.h +Py_HASH_EXTERNAL +# exports.h +PyAPI_DATA +Py_EXPORTED_SYMBOL +Py_IMPORTED_SYMBOL +Py_LOCAL_SYMBOL +# modsupport.h +PyABIInfo_FREETHREADING_AGNOSTIC +# moduleobject.h +PyModuleDef_Type +# object.h +Py_INVALID_SIZE +Py_TPFLAGS_HAVE_VERSION_TAG +Py_TPFLAGS_INLINE_VALUES +Py_TPFLAGS_IS_ABSTRACT +# pyexpat.h +PyExpat_CAPI_MAGIC +PyExpat_CAPSULE_NAME +# pyport.h +Py_ALIGNED +Py_ARITHMETIC_RIGHT_SHIFT +Py_CAN_START_THREADS +Py_FORCE_EXPANSION +Py_GCC_ATTRIBUTE +Py_LL +Py_SAFE_DOWNCAST +Py_ULL +Py_VA_COPY +# unicodeobject.h +Py_UNICODE_SIZE +# cpython/methodobject.h +PyCFunction_GET_CLASS +# cpython/compile.h +PyCF_ALLOW_INCOMPLETE_INPUT +PyCF_COMPILE_MASK +PyCF_DONT_IMPLY_DEDENT +PyCF_IGNORE_COOKIE +PyCF_MASK +PyCF_MASK_OBSOLETE +PyCF_SOURCE_IS_UTF8 +# cpython/descrobject.h +PyDescr_COMMON +PyDescr_NAME +PyDescr_TYPE +PyWrapperFlag_KEYWORDS +# cpython/fileobject.h +PyFile_NewStdPrinter +PyStdPrinter_Type +Py_UniversalNewlineFgets +# cpython/setobject.h +PySet_MINSIZE +# cpython/ceval.h +PyUnstable_CopyPerfMapFile +PyUnstable_PerfTrampoline_CompileCode +PyUnstable_PerfTrampoline_SetPersistAfterFork +# cpython/genobject.h +PyAsyncGenASend_CheckExact +# cpython/longintrepr.h +PyLong_BASE +PyLong_MASK +PyLong_SHIFT +# cpython/pyerrors.h +PyException_HEAD +# cpython/pyframe.h +PyUnstable_EXECUTABLE_KINDS +PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION +PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR +PyUnstable_EXECUTABLE_KIND_PY_FUNCTION +PyUnstable_EXECUTABLE_KIND_SKIP +# cpython/pylifecycle.h +Py_FrozenMain +# cpython/unicodeobject.h +PyUnicode_IS_COMPACT +PyUnicode_IS_COMPACT_ASCII diff --git a/Tools/c-api-docs-check/main.py b/Tools/c-api-docs-check/main.py new file mode 100644 index 00000000000000..4ef638ee76c551 --- /dev/null +++ b/Tools/c-api-docs-check/main.py @@ -0,0 +1,137 @@ +import re +from pathlib import Path +import sys +import _colorize + +SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") +SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") +SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") +SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") + +_CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = _CPYTHON / "Include" +C_API_DOCS = _CPYTHON / "Doc" / "c-api" +IGNORED = (_CPYTHON / "Tools" / "c-api-docs-check" / "ignored_c_api.txt").read_text().split("\n") + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + + +def is_documented(name: str) -> bool: + """ + Is a name present in the C API documentation? + """ + if name in IGNORED: + return True + + for path in C_API_DOCS.iterdir(): + if path.is_dir(): + continue + if path.suffix != ".rst": + continue + + text = path.read_text(encoding="utf-8") + if name in text: + return True + + return False + + +def scan_file_for_missing_docs(filename: str, text: str) -> list[str]: + """ + Scan a header file for undocumented C API functions. + """ + undocumented: list[str] = [] + colors = _colorize.get_colors() + + for function in SIMPLE_FUNCTION_REGEX.finditer(text): + name = function.group(2) + if not name.startswith("Py"): + continue + + if not is_documented(name): + undocumented.append(name) + + for macro in SIMPLE_MACRO_REGEX.finditer(text): + name = macro.group(1) + if not name.startswith("Py"): + continue + + if "(" in name: + name = name[: name.index("(")] + + if not is_documented(name): + undocumented.append(name) + + for inline in SIMPLE_INLINE_REGEX.finditer(text): + name = inline.group(2) + if not name.startswith("Py"): + continue + + if not is_documented(name): + undocumented.append(name) + + for data in SIMPLE_DATA_REGEX.finditer(text): + name = data.group(1) + if not name.startswith("Py"): + continue + + if not is_documented(name): + undocumented.append(name) + + # Remove duplicates and sort alphabetically to keep the output non-deterministic + undocumented = list(set(undocumented)) + undocumented.sort() + + if undocumented: + print(f"{filename} {colors.RED}BAD{colors.RESET}") + for name in undocumented: + print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") + + return undocumented + else: + print(f"{filename} {colors.GREEN}OK{colors.RESET}") + + return [] + + +def main() -> None: + print("Scanning for undocumented C API functions...") + files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] + all_missing: list[str] = [] + for file in files: + if file.is_dir(): + continue + assert file.exists() + text = file.read_text(encoding="utf-8") + missing = scan_file_for_missing_docs(str(file.relative_to(INCLUDE)), text) + all_missing += missing + + if all_missing != []: + s = "s" if len(all_missing) != 1 else "" + print(f"-- {len(all_missing)} missing function{s} --") + for name in all_missing: + print(f" - {name}") + print() + print( + "Found some undocumented C API!", + "Python requires documentation on all public C API functions.", + "If these function(s) were not meant to be public, please prefix " + "them with a leading underscore (_PySomething_API) or move them to " + "the internal C API (pycore_*.h files).", + "", + "In exceptional cases, certain functions can be ignored by adding " + "them to Tools/c-api-docs-check/ignored_c_api.txt", + "If this is a mistake and this script should not be failing, please " + "create an issue and tag Peter (@ZeroIntensity) on it.", + sep="\n", + ) + sys.exit(1) + else: + print("Nothing found :)") + sys.exit(0) + + +if __name__ == "__main__": + main() From e2191364f36ff1e35b61eaae7d2df2003d493351 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 18:52:07 -0500 Subject: [PATCH 02/13] Add a CI job. --- .github/workflows/build.yml | 4 ++++ Makefile.pre.in | 5 +++++ .../{c-api-docs-check => check-c-api-docs}/ignored_c_api.txt | 0 Tools/{c-api-docs-check => check-c-api-docs}/main.py | 0 4 files changed, 9 insertions(+) rename Tools/{c-api-docs-check => check-c-api-docs}/ignored_c_api.txt (100%) rename Tools/{c-api-docs-check => check-c-api-docs}/main.py (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e15400e4978eb..d4ac8c53f71282 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,6 +142,10 @@ jobs: - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + - name: Check for undocumented C APIs + if: github.event_name == 'pull_request' + run: make check-c-api-docs + build-windows: name: >- diff --git a/Makefile.pre.in b/Makefile.pre.in index 7b8e7ec0965180..f19ddfe966df9a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3313,6 +3313,11 @@ check-c-globals: --format summary \ --traceback +# Check for undocumented C APIs. +.PHONY: check-c-globals +check-c-api-docs: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py + # Find files with funny names .PHONY: funny funny: diff --git a/Tools/c-api-docs-check/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt similarity index 100% rename from Tools/c-api-docs-check/ignored_c_api.txt rename to Tools/check-c-api-docs/ignored_c_api.txt diff --git a/Tools/c-api-docs-check/main.py b/Tools/check-c-api-docs/main.py similarity index 100% rename from Tools/c-api-docs-check/main.py rename to Tools/check-c-api-docs/main.py From 8c8840281bb7af758c9a5bfebe9bd3f76e4cd91a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 19:17:30 -0500 Subject: [PATCH 03/13] General improvements. --- Tools/check-c-api-docs/main.py | 102 +++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index 4ef638ee76c551..51e73466a2516d 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -8,10 +8,37 @@ SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") +MISTAKE = """\ +If this is a mistake and this script should not be failing, please create an +issue and tag Peter (@ZeroIntensity) on it.\ +""" + +FOUND_UNDOCUMENTED = f"""\ +Found some undocumented C API! + +Python requires documentation on all public C API functions. +If these API(s) were not meant to be public, please prefix them with a +leading underscore (_PySomething_API) or move them to the internal C API +(pycore_*.h files). + +In exceptional cases, certain functions can be ignored by adding them to +Tools/c-api-docs-check/ignored_c_api.txt + +{MISTAKE}\ +""" + +FOUND_IGNORED_DOCUMENTED = f"""\ +Some C API(s) were listed in Tools/c-api-docs-check/ignored_c_api.txt, but +they were found in the documentation. To fix this, simply update ignored_c_api.txt +accordingly. + +{MISTAKE}\ +""" + _CPYTHON = Path(__file__).parent.parent.parent INCLUDE = _CPYTHON / "Include" C_API_DOCS = _CPYTHON / "Doc" / "c-api" -IGNORED = (_CPYTHON / "Tools" / "c-api-docs-check" / "ignored_c_api.txt").read_text().split("\n") +IGNORED = (_CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt").read_text().split("\n") for index, line in enumerate(IGNORED): if line.startswith("#"): @@ -22,9 +49,6 @@ def is_documented(name: str) -> bool: """ Is a name present in the C API documentation? """ - if name in IGNORED: - return True - for path in C_API_DOCS.iterdir(): if path.is_dir(): continue @@ -38,20 +62,27 @@ def is_documented(name: str) -> bool: return False -def scan_file_for_missing_docs(filename: str, text: str) -> list[str]: +def scan_file_for_docs(filename: str, text: str) -> tuple[list[str], list[str]]: """ - Scan a header file for undocumented C API functions. + Scan a header file for C API functions. """ undocumented: list[str] = [] + documented_ignored: list[str] = [] colors = _colorize.get_colors() + def check_for_name(name: str) -> None: + documented = is_documented(name) + if documented and (name in IGNORED): + documented_ignored.append(name) + elif not documented and (name not in IGNORED): + undocumented.append(name) + for function in SIMPLE_FUNCTION_REGEX.finditer(text): name = function.group(2) if not name.startswith("Py"): continue - if not is_documented(name): - undocumented.append(name) + check_for_name(name) for macro in SIMPLE_MACRO_REGEX.finditer(text): name = macro.group(1) @@ -61,76 +92,71 @@ def scan_file_for_missing_docs(filename: str, text: str) -> list[str]: if "(" in name: name = name[: name.index("(")] - if not is_documented(name): - undocumented.append(name) + check_for_name(name) for inline in SIMPLE_INLINE_REGEX.finditer(text): name = inline.group(2) if not name.startswith("Py"): continue - if not is_documented(name): - undocumented.append(name) + check_for_name(name) for data in SIMPLE_DATA_REGEX.finditer(text): name = data.group(1) if not name.startswith("Py"): continue - if not is_documented(name): - undocumented.append(name) + check_for_name(name) # Remove duplicates and sort alphabetically to keep the output non-deterministic undocumented = list(set(undocumented)) undocumented.sort() - if undocumented: + if undocumented or documented_ignored: print(f"{filename} {colors.RED}BAD{colors.RESET}") for name in undocumented: print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") - - return undocumented + for name in documented_ignored: + print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") else: print(f"{filename} {colors.GREEN}OK{colors.RESET}") - return [] + return undocumented, documented_ignored def main() -> None: print("Scanning for undocumented C API functions...") files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] all_missing: list[str] = [] + all_found_ignored: list[str] = [] + for file in files: if file.is_dir(): continue assert file.exists() text = file.read_text(encoding="utf-8") - missing = scan_file_for_missing_docs(str(file.relative_to(INCLUDE)), text) + missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text) + all_found_ignored += ignored all_missing += missing + fail = False if all_missing != []: s = "s" if len(all_missing) != 1 else "" - print(f"-- {len(all_missing)} missing function{s} --") + print(f"-- {len(all_missing)} missing C API{s} --") for name in all_missing: print(f" - {name}") - print() - print( - "Found some undocumented C API!", - "Python requires documentation on all public C API functions.", - "If these function(s) were not meant to be public, please prefix " - "them with a leading underscore (_PySomething_API) or move them to " - "the internal C API (pycore_*.h files).", - "", - "In exceptional cases, certain functions can be ignored by adding " - "them to Tools/c-api-docs-check/ignored_c_api.txt", - "If this is a mistake and this script should not be failing, please " - "create an issue and tag Peter (@ZeroIntensity) on it.", - sep="\n", - ) - sys.exit(1) - else: - print("Nothing found :)") - sys.exit(0) + print(FOUND_UNDOCUMENTED) + fail = True + + if all_found_ignored != []: + s = "s" if len(all_found_ignored) != 1 else "" + print(f"-- Found {len(all_found_ignored)} documented but ignored C API{s} --") + for name in all_found_ignored: + print(f" - {name}") + print(FOUND_IGNORED_DOCUMENTED) + fail = True + + sys.exit(1 if fail else 0) if __name__ == "__main__": From 0f687cf8c9649ae5e976f71c7de3999c5d5c2701 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 19:19:24 -0500 Subject: [PATCH 04/13] Add myself as a codeowner. --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1086b42620479d..d1a914fe516a80 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -140,6 +140,9 @@ Misc/externals.spdx.json @sethmlarson Misc/sbom.spdx.json @sethmlarson Tools/build/generate_sbom.py @sethmlarson +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # ---------------------------------------------------------------------------- # Platform Support From ec03845902a3c54d3672e92dd4921fb572d2aa04 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 19:19:37 -0500 Subject: [PATCH 05/13] Run black. --- Tools/check-c-api-docs/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index 51e73466a2516d..b05ae04a28af2f 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -38,7 +38,11 @@ _CPYTHON = Path(__file__).parent.parent.parent INCLUDE = _CPYTHON / "Include" C_API_DOCS = _CPYTHON / "Doc" / "c-api" -IGNORED = (_CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt").read_text().split("\n") +IGNORED = ( + (_CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) for index, line in enumerate(IGNORED): if line.startswith("#"): From 25104766a21fd4ddd9053af06b5e0f3dd3e5c0ed Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 29 Nov 2025 19:30:34 -0500 Subject: [PATCH 06/13] Eliminate some duplication. --- Tools/check-c-api-docs/main.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index b05ae04a28af2f..e142aa0ab1523a 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -14,9 +14,9 @@ """ FOUND_UNDOCUMENTED = f"""\ -Found some undocumented C API! +Found some undocumented C API(s)! -Python requires documentation on all public C API functions. +Python requires documentation on all public C API symbols, macros, and types. If these API(s) were not meant to be public, please prefix them with a leading underscore (_PySomething_API) or move them to the internal C API (pycore_*.h files). @@ -144,20 +144,19 @@ def main() -> None: all_missing += missing fail = False - if all_missing != []: - s = "s" if len(all_missing) != 1 else "" - print(f"-- {len(all_missing)} missing C API{s} --") - for name in all_missing: - print(f" - {name}") - print(FOUND_UNDOCUMENTED) - fail = True + to_check = [ + (all_missing, "missing", FOUND_UNDOCUMENTED), + (all_found_ignored, "documented but ignored", FOUND_IGNORED_DOCUMENTED), + ] + for name_list, what, message in to_check: + if not name_list: + continue - if all_found_ignored != []: - s = "s" if len(all_found_ignored) != 1 else "" - print(f"-- Found {len(all_found_ignored)} documented but ignored C API{s} --") - for name in all_found_ignored: + s = "s" if len(name_list) != 1 else "" + print(f"-- {len(name_list)} {what} C API{s} --") + for name in all_missing: print(f" - {name}") - print(FOUND_IGNORED_DOCUMENTED) + print(message) fail = True sys.exit(1 if fail else 0) From 51b2747d160c294b8d1582eaf49be8a7f9bc1167 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 11:15:16 -0500 Subject: [PATCH 07/13] Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Makefile.pre.in | 2 +- Tools/check-c-api-docs/main.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index f19ddfe966df9a..b6237418eb8bcf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3314,7 +3314,7 @@ check-c-globals: --traceback # Check for undocumented C APIs. -.PHONY: check-c-globals +.PHONY: check-c-api-docs check-c-api-docs: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index e142aa0ab1523a..7dad80270ef55c 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -9,7 +9,7 @@ SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") MISTAKE = """\ -If this is a mistake and this script should not be failing, please create an +If this is a mistake and this script should not be failing, create an issue and tag Peter (@ZeroIntensity) on it.\ """ @@ -17,7 +17,7 @@ Found some undocumented C API(s)! Python requires documentation on all public C API symbols, macros, and types. -If these API(s) were not meant to be public, please prefix them with a +If these API(s) were not meant to be public, prefix them with a leading underscore (_PySomething_API) or move them to the internal C API (pycore_*.h files). @@ -29,8 +29,8 @@ FOUND_IGNORED_DOCUMENTED = f"""\ Some C API(s) were listed in Tools/c-api-docs-check/ignored_c_api.txt, but -they were found in the documentation. To fix this, simply update ignored_c_api.txt -accordingly. +they were found in the documentation. To fix this, remove them from +ignored_c_api.txt. {MISTAKE}\ """ @@ -112,7 +112,7 @@ def check_for_name(name: str) -> None: check_for_name(name) - # Remove duplicates and sort alphabetically to keep the output non-deterministic + # Remove duplicates and sort alphabetically to keep the output deterministic undocumented = list(set(undocumented)) undocumented.sort() From cb9a68dd14ba925968da4bde891cb7dfea693b49 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 11:16:41 -0500 Subject: [PATCH 08/13] Fix alphabetical order. --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d1a914fe516a80..6acc156ebff713 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,6 +126,9 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner # C Analyser Tools/c-analyzer/ @ericsnowcurrently +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # Fuzzing Modules/_xxtestfuzz/ @ammaraskar @@ -140,9 +143,6 @@ Misc/externals.spdx.json @sethmlarson Misc/sbom.spdx.json @sethmlarson Tools/build/generate_sbom.py @sethmlarson -# C API Documentation Checks -Tools/check-c-api-docs/ @ZeroIntensity - # ---------------------------------------------------------------------------- # Platform Support From 681962d1572f05cd1b6fbf4fc9ef065ceb5cf86d Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 11:28:54 -0500 Subject: [PATCH 09/13] Change plurality in the message depending on the size of the list. --- Tools/check-c-api-docs/main.py | 86 ++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index 7dad80270ef55c..879a7f6c1b789a 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -2,51 +2,72 @@ from pathlib import Path import sys import _colorize +import textwrap SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") +CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = CPYTHON / "Include" +C_API_DOCS = CPYTHON / "Doc" / "c-api" +IGNORED = ( + (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + MISTAKE = """\ If this is a mistake and this script should not be failing, create an issue and tag Peter (@ZeroIntensity) on it.\ """ -FOUND_UNDOCUMENTED = f"""\ -Found some undocumented C API(s)! -Python requires documentation on all public C API symbols, macros, and types. -If these API(s) were not meant to be public, prefix them with a -leading underscore (_PySomething_API) or move them to the internal C API -(pycore_*.h files). +def found_undocumented(singular: bool): + some = "an" if singular else "some" + s = "" if singular else "s" + these = "this" if singular else "these" + them = "it" if singular else "them" + were = "was" if singular else "were" -In exceptional cases, certain functions can be ignored by adding them to -Tools/c-api-docs-check/ignored_c_api.txt + return textwrap.dedent( + f"""\ + Found {some} undocumented C API{s}! -{MISTAKE}\ -""" + Python requires documentation on all public C API symbols, macros, and types. + If {these} API{s} {were} not meant to be public, prefix {them} with a + leading underscore (_PySomething_API) or move {them} to the internal C API + (pycore_*.h files). -FOUND_IGNORED_DOCUMENTED = f"""\ -Some C API(s) were listed in Tools/c-api-docs-check/ignored_c_api.txt, but -they were found in the documentation. To fix this, remove them from -ignored_c_api.txt. + In exceptional cases, certain APIs can be ignored by adding them to + Tools/c-api-docs-check/ignored_c_api.txt -{MISTAKE}\ -""" + {MISTAKE}\ + """ + ) -_CPYTHON = Path(__file__).parent.parent.parent -INCLUDE = _CPYTHON / "Include" -C_API_DOCS = _CPYTHON / "Doc" / "c-api" -IGNORED = ( - (_CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") - .read_text() - .split("\n") -) -for index, line in enumerate(IGNORED): - if line.startswith("#"): - IGNORED.pop(index) +def found_ignored_documented(singular: bool) -> str: + some = "a" if singular else "some" + s = "" if singular else "s" + them = "it" if singular else "them" + were = "was" if singular else "were" + they = "it" if singular else "they" + + return textwrap.dedent( + f"""\ + Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but + {they} {were} found in the documentation. To fix this, remove {them} from + ignored_c_api.txt. + + {MISTAKE}\ + """ + ) def is_documented(name: str) -> bool: @@ -145,8 +166,12 @@ def main() -> None: fail = False to_check = [ - (all_missing, "missing", FOUND_UNDOCUMENTED), - (all_found_ignored, "documented but ignored", FOUND_IGNORED_DOCUMENTED), + (all_missing, "missing", found_undocumented(len(all_missing) == 1)), + ( + all_found_ignored, + "documented but ignored", + found_ignored_documented(len(all_found_ignored) == 1), + ), ] for name_list, what, message in to_check: if not name_list: @@ -154,8 +179,9 @@ def main() -> None: s = "s" if len(name_list) != 1 else "" print(f"-- {len(name_list)} {what} C API{s} --") - for name in all_missing: + for name in name_list: print(f" - {name}") + print() print(message) fail = True From 1d0c8f681ea19bb8c9037c0983588ea91716ab6f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 11:36:13 -0500 Subject: [PATCH 10/13] Fix indentation on the output. --- Tools/check-c-api-docs/main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index 879a7f6c1b789a..914eaee1f5a611 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -22,7 +22,7 @@ if line.startswith("#"): IGNORED.pop(index) -MISTAKE = """\ +MISTAKE = """ If this is a mistake and this script should not be failing, create an issue and tag Peter (@ZeroIntensity) on it.\ """ @@ -35,8 +35,9 @@ def found_undocumented(singular: bool): them = "it" if singular else "them" were = "was" if singular else "were" - return textwrap.dedent( - f"""\ + return ( + textwrap.dedent( + f""" Found {some} undocumented C API{s}! Python requires documentation on all public C API symbols, macros, and types. @@ -46,9 +47,9 @@ def found_undocumented(singular: bool): In exceptional cases, certain APIs can be ignored by adding them to Tools/c-api-docs-check/ignored_c_api.txt - - {MISTAKE}\ """ + ) + + MISTAKE ) @@ -59,14 +60,15 @@ def found_ignored_documented(singular: bool) -> str: were = "was" if singular else "were" they = "it" if singular else "they" - return textwrap.dedent( - f"""\ + return ( + textwrap.dedent( + f""" Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but {they} {were} found in the documentation. To fix this, remove {them} from ignored_c_api.txt. - - {MISTAKE}\ """ + ) + + MISTAKE ) @@ -181,7 +183,6 @@ def main() -> None: print(f"-- {len(name_list)} {what} C API{s} --") for name in name_list: print(f" - {name}") - print() print(message) fail = True From f487d6e384723bc16237d58c08207ae9270db560 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 19:26:00 -0500 Subject: [PATCH 11/13] Update .github/workflows/build.yml Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4ac8c53f71282..3d889fa128e261 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,7 +143,6 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals - name: Check for undocumented C APIs - if: github.event_name == 'pull_request' run: make check-c-api-docs From 02d2f078af82d164bcb2aff9cb9d987c072d056b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 30 Nov 2025 19:26:17 -0500 Subject: [PATCH 12/13] Update Tools/check-c-api-docs/main.py Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Tools/check-c-api-docs/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index 914eaee1f5a611..e6b55b6ad29da4 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -28,7 +28,7 @@ """ -def found_undocumented(singular: bool): +def found_undocumented(singular: bool) -> str: some = "an" if singular else "some" s = "" if singular else "s" these = "this" if singular else "these" From 147d710c0a00155fefa2c7f79c278b92429b4e21 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 3 Dec 2025 21:47:40 -0500 Subject: [PATCH 13/13] Remove newly documented C APIs from ignored_c_api.txt --- Tools/check-c-api-docs/ignored_c_api.txt | 7 ------- Tools/check-c-api-docs/main.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index 24a25626dcf797..e81ffd51e193b2 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -1,10 +1,3 @@ -# descrobject.h -PyClassMethodDescr_Type -PyDictProxy_Type -PyGetSetDescr_Type -PyMemberDescr_Type -PyMethodDescr_Type -PyWrapperDescr_Type # pydtrace_probes.h PyDTrace_AUDIT PyDTrace_FUNCTION_ENTRY diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py index e6b55b6ad29da4..6bdf80a9ae8985 100644 --- a/Tools/check-c-api-docs/main.py +++ b/Tools/check-c-api-docs/main.py @@ -46,7 +46,7 @@ def found_undocumented(singular: bool) -> str: (pycore_*.h files). In exceptional cases, certain APIs can be ignored by adding them to - Tools/c-api-docs-check/ignored_c_api.txt + Tools/check-c-api-docs/ignored_c_api.txt """ ) + MISTAKE