3
3
Check the output of running Sphinx in nit-picky mode (missing references).
4
4
"""
5
5
import argparse
6
- import csv
6
+ import itertools
7
7
import os
8
8
import re
9
+ import subprocess
9
10
import sys
10
11
from pathlib import Path
12
+ from typing import TextIO
11
13
12
14
# Exclude these whether they're dirty or clean,
13
15
# because they trigger a rebuild of dirty files.
24
26
"venv" ,
25
27
}
26
28
27
- PATTERN = re .compile (r"(?P<file>[^:]+):(?P<line>\d+): WARNING: (?P<msg>.+)" )
29
+ # Regex pattern to match the parts of a Sphinx warning
30
+ WARNING_PATTERN = re .compile (
31
+ r"(?P<file>([A-Za-z]:[\\/])?[^:]+):(?P<line>\d+): WARNING: (?P<msg>.+)"
32
+ )
28
33
34
+ # Regex pattern to match the line numbers in a Git unified diff
35
+ DIFF_PATTERN = re .compile (
36
+ r"^@@ -(?P<linea>\d+),(?P<removed>\d+) \+(?P<lineb>\d+),(?P<added>\d+) @@" ,
37
+ flags = re .MULTILINE ,
38
+ )
29
39
30
- def check_and_annotate (warnings : list [str ], files_to_check : str ) -> None :
40
+
41
+ def get_diff_files (ref_a : str , ref_b : str , filter_mode : str = "" ) -> set [Path ]:
42
+ """List the files changed between two Gif refs, filtered by change type."""
43
+ added_files_result = subprocess .run (
44
+ [
45
+ "git" ,
46
+ "diff" ,
47
+ f"--diff-filter={ filter_mode } " ,
48
+ "--name-only" ,
49
+ f"{ ref_a } ...{ ref_b } " ,
50
+ ],
51
+ stdout = subprocess .PIPE ,
52
+ check = True ,
53
+ text = True ,
54
+ )
55
+
56
+ added_files = added_files_result .stdout .strip ().split ("\n " )
57
+ return {Path (file .strip ()) for file in added_files if file .strip ()}
58
+
59
+
60
+ def get_diff_lines (ref_a : str , ref_b : str , file : os .PathLike ) -> list [int ]:
61
+ """List the lines changed between two Gif refs for a specific file."""
62
+ diff_output = subprocess .run (
63
+ [
64
+ "git" ,
65
+ "diff" ,
66
+ "--unified=0" ,
67
+ "--diff-filter=M" ,
68
+ f"{ ref_a } ...{ ref_b } " ,
69
+ "--" ,
70
+ str (file ),
71
+ ],
72
+ stdout = subprocess .PIPE ,
73
+ check = True ,
74
+ text = True ,
75
+ )
76
+
77
+ line_matches = DIFF_PATTERN .finditer (diff_output .stdout )
78
+ line_ints = [
79
+ (int (line_match ["lineb" ]), int (line_match ["added" ]))
80
+ for line_match in line_matches
81
+ ]
82
+ line_ranges = [
83
+ range (line_b , line_b + added ) for line_b , added in line_ints
84
+ ]
85
+ line_numbers = list (itertools .chain (* line_ranges ))
86
+ return line_numbers
87
+
88
+
89
+ def get_para_line_numbers (file_obj : TextIO ) -> list [list [int ]]:
90
+ """Get the line numbers of text in a file object, grouped by paragraph."""
91
+ paragraphs = []
92
+ prev_line = None
93
+ for lineno , line in enumerate (file_obj ):
94
+ lineno = lineno + 1
95
+ if prev_line is None or (line .strip () and not prev_line .strip ()):
96
+ paragraph = [lineno - 1 ]
97
+ paragraphs .append (paragraph )
98
+ paragraph .append (lineno )
99
+ prev_line = line
100
+ return paragraphs
101
+
102
+
103
+ def process_touched_warnings (
104
+ warnings : list [str ], ref_a : str , ref_b : str
105
+ ) -> list [re .Match ]:
106
+ """Filter a list of Sphinx warnings to those affecting touched lines."""
107
+ added_files , modified_files = tuple (
108
+ get_diff_files (ref_a , ref_b , filter_mode = mode ) for mode in ("A" , "M" )
109
+ )
110
+
111
+ warnings_added , warnings_modified = tuple (
112
+ [
113
+ WARNING_PATTERN .fullmatch (warning .strip ())
114
+ for warning in warnings
115
+ if any (str (file ) in warning for file in files )
116
+ ]
117
+ for files in (added_files , modified_files )
118
+ )
119
+ warnings_added , warnings_modified = tuple (
120
+ [warning for warning in warning_matches if warning ]
121
+ for warning_matches in (warnings_added , warnings_modified )
122
+ )
123
+
124
+ modified_files_warned = {
125
+ file
126
+ for file in modified_files
127
+ if any (str (file ) in warning ["file" ] for warning in warnings_modified )
128
+ }
129
+
130
+ warnings_touched = warnings_added .copy ()
131
+ for file in modified_files_warned :
132
+ diff_lines = get_diff_lines (ref_a , ref_b , file )
133
+ with file .open (encoding = "UTF-8" ) as file_obj :
134
+ paragraphs = get_para_line_numbers (file_obj )
135
+ touched_paras = [
136
+ para_lines
137
+ for para_lines in paragraphs
138
+ if set (diff_lines ) & set (para_lines )
139
+ ]
140
+ touched_para_lines = set (itertools .chain (* touched_paras ))
141
+ warnings_infile = [
142
+ warning
143
+ for warning in warnings_modified
144
+ if str (file ) in warning ["file" ]
145
+ ]
146
+ warnings_touched += [
147
+ warning
148
+ for warning in warnings_infile
149
+ if int (warning ["line" ]) in touched_para_lines
150
+ ]
151
+
152
+ return warnings_touched
153
+
154
+
155
+ def check_and_annotate_changed (
156
+ warnings : list [str ], ref_a : str = "main" , ref_b : str = "HEAD"
157
+ ) -> None :
31
158
"""
32
- Convert Sphinx warning messages to GitHub Actions.
159
+ Convert Sphinx warning messages to GitHub Actions for changed paragraphs .
33
160
34
161
Converts lines like:
35
162
.../Doc/library/cgi.rst:98: WARNING: reference target not found
36
163
to:
37
164
::warning file=.../Doc/library/cgi.rst,line=98::reference target not found
38
165
39
- Non-matching lines are echoed unchanged.
40
-
41
- see:
166
+ See:
42
167
https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message
43
168
"""
44
- files_to_check = next (csv .reader ([files_to_check ]))
45
- for warning in warnings :
46
- if any (filename in warning for filename in files_to_check ):
47
- if match := PATTERN .fullmatch (warning ):
48
- print ("::warning file={file},line={line}::{msg}" .format_map (match ))
169
+ warnings_touched = process_touched_warnings (warnings , ref_a , ref_b )
170
+ for warning in warnings_touched :
171
+ print ("::warning file={file},line={line}::{msg}" .format_map (warning ))
49
172
50
173
51
174
def fail_if_regression (
@@ -68,7 +191,7 @@ def fail_if_regression(
68
191
print (filename )
69
192
for warning in warnings :
70
193
if filename in warning :
71
- if match := PATTERN .fullmatch (warning ):
194
+ if match := WARNING_PATTERN .fullmatch (warning ):
72
195
print (" {line}: {msg}" .format_map (match ))
73
196
return - 1
74
197
return 0
@@ -94,9 +217,10 @@ def fail_if_improved(
94
217
def main () -> int :
95
218
parser = argparse .ArgumentParser ()
96
219
parser .add_argument (
97
- "--check-and-annotate" ,
98
- help = "Comma-separated list of files to check, "
99
- "and annotate those with warnings on GitHub Actions" ,
220
+ "--check-and-annotate-changed" ,
221
+ nargs = 2 ,
222
+ help = "Annotate lines changed between two refs "
223
+ "with warnings on GitHub Actions" ,
100
224
)
101
225
parser .add_argument (
102
226
"--fail-if-regression" ,
@@ -114,7 +238,7 @@ def main() -> int:
114
238
wrong_directory_msg = "Must run this script from the repo root"
115
239
assert Path ("Doc" ).exists () and Path ("Doc" ).is_dir (), wrong_directory_msg
116
240
117
- with Path ("Doc/sphinx-warnings.txt" ).open () as f :
241
+ with Path ("Doc/sphinx-warnings.txt" ).open (encoding = "UTF-8" ) as f :
118
242
warnings = f .read ().splitlines ()
119
243
120
244
cwd = str (Path .cwd ()) + os .path .sep
@@ -131,8 +255,8 @@ def main() -> int:
131
255
if filename .strip () and not filename .startswith ("#" )
132
256
}
133
257
134
- if args .check_and_annotate :
135
- check_and_annotate (warnings , args .check_and_annotate )
258
+ if args .check_and_annotate_changed :
259
+ check_and_annotate_changed (warnings , * args .check_and_annotate_changed )
136
260
137
261
if args .fail_if_regression :
138
262
exit_code += fail_if_regression (
0 commit comments