From 016f241cba524762cb834af54321996af2f7c5e6 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 24 Nov 2023 10:58:29 -0800 Subject: [PATCH 1/6] Use tokenize to replace convinience variables --- Lib/pdb.py | 33 ++++++++++++++++++++++++++++++++- Lib/test/test_pdb.py | 6 ++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index ed78d749a47fa8..38871c22335af6 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -76,6 +76,7 @@ import dis import code import glob +import token import codeop import pprint import signal @@ -590,6 +591,36 @@ def default(self, line): except: self._error_exc() + def _replace_convenience_variable(self, line): + """Replace the convenience variable in line""" + last_token_is_dollar = False + dollar_start = None + dollar_end = None + replace_variables = [] + try: + for t in tokenize.generate_tokens(io.StringIO(line).readline): + token_type, token_string, start, end, _ = t + if token_type == token.OP and token_string == '$': + last_token_is_dollar = True + dollar_end = end + dollar_start = start + else: + if (last_token_is_dollar and + token_type == token.NAME and + start == dollar_end): + # line is a one line command so we only care about column + replace_variables.append((dollar_start[1], end[1], token_string)) + last_token_is_dollar = False + except tokenize.TokenError: + return line + last_end = 0 + new_line = '' + for start, end, name in replace_variables: + new_line += line[last_end:start] + f'__pdb_convenience_variables["{name}"]' + last_end = end + new_line += line[last_end:] + return new_line + def precmd(self, line): """Handle alias expansion and ';;' separator.""" if not line.strip(): @@ -624,7 +655,7 @@ def precmd(self, line): line = line[:marker].rstrip() # Replace all the convenience variables - line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) + line = self._replace_convenience_variable(line) return line diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 67a4053a2ac8bc..1255c2d484e79b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -794,9 +794,11 @@ def test_convenience_variables(): >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... '$_frame.f_lineno', # Check frame convenience variable + ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable ... '$a', # Print its value ... 'p $a + 2', # Do some calculation + ... 'p f"$a = {$a}"', # Make sure $ in string is not converted and f-string works ... 'u', # Switch frame ... '$_frame.f_lineno', # Make sure the frame changed ... '$a', # Make sure the value persists @@ -816,11 +818,15 @@ def test_convenience_variables(): -> try: (Pdb) $_frame.f_lineno 3 + (Pdb) $ _frame + *** SyntaxError: invalid syntax (Pdb) $a = 10 (Pdb) $a 10 (Pdb) p $a + 2 12 + (Pdb) p f"$a = {$a}" + '$a = 10' (Pdb) u > (2)test_function() -> util_function() From bb53b6cb3575665336383bbed7a1688f713e66cb Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 19:08:52 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst new file mode 100644 index 00000000000000..7f3c9d43b57033 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst @@ -0,0 +1 @@ +Switch to :mod:`tokenize` for convenience variables to avoid replacing strings. From e8ebbe624ec109336e1545e3066608e1dee5b4a3 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 24 Nov 2023 11:58:58 -0900 Subject: [PATCH 3/6] Update Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst Co-authored-by: Ned Batchelder --- .../next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst index 7f3c9d43b57033..aaa50fce3ac962 100644 --- a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst +++ b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst @@ -1 +1 @@ -Switch to :mod:`tokenize` for convenience variables to avoid replacing strings. +Improve handling of pdb convenience variables to avoid replacing string contents. From f7ece907f91ee1bc05054f477ec60a2a2f81cf4c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 26 Nov 2023 16:03:26 -0800 Subject: [PATCH 4/6] Add early break and rename the function --- Lib/pdb.py | 15 ++++++++++++--- Lib/test/test_pdb.py | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 38871c22335af6..29748fec78b4f8 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -591,8 +591,12 @@ def default(self, line): except: self._error_exc() - def _replace_convenience_variable(self, line): - """Replace the convenience variable in line""" + def _replace_convenience_variables(self, line): + """Replace the convenience variables in line""" + + if "$" not in line: + return line + last_token_is_dollar = False dollar_start = None dollar_end = None @@ -613,12 +617,17 @@ def _replace_convenience_variable(self, line): last_token_is_dollar = False except tokenize.TokenError: return line + + if not replace_variables: + return line + last_end = 0 new_line = '' for start, end, name in replace_variables: new_line += line[last_end:start] + f'__pdb_convenience_variables["{name}"]' last_end = end new_line += line[last_end:] + return new_line def precmd(self, line): @@ -655,7 +664,7 @@ def precmd(self, line): line = line[:marker].rstrip() # Replace all the convenience variables - line = self._replace_convenience_variable(line) + line = self._replace_convenience_variables(line) return line diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 1255c2d484e79b..20434b2abce4b7 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -797,6 +797,7 @@ def test_convenience_variables(): ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable ... '$a', # Print its value + ... 'p "$a"', # Print the string $a ... 'p $a + 2', # Do some calculation ... 'p f"$a = {$a}"', # Make sure $ in string is not converted and f-string works ... 'u', # Switch frame @@ -823,6 +824,8 @@ def test_convenience_variables(): (Pdb) $a = 10 (Pdb) $a 10 + (Pdb) p "$a" + '$a' (Pdb) p $a + 2 12 (Pdb) p f"$a = {$a}" From 7072c146cf4e85d48f32fc5ec69ef47d3f5fbbe1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 16 Jan 2024 22:44:10 -0800 Subject: [PATCH 5/6] Update comment in Lib/pdb.py Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/pdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 29748fec78b4f8..88c54cfd49edc9 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -612,7 +612,7 @@ def _replace_convenience_variables(self, line): if (last_token_is_dollar and token_type == token.NAME and start == dollar_end): - # line is a one line command so we only care about column + # line is a one-line command so we only care about column replace_variables.append((dollar_start[1], end[1], token_string)) last_token_is_dollar = False except tokenize.TokenError: From 808dbbdd26c200dd9753816522b8f3a0dc734a72 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 16 Jan 2024 22:58:16 -0800 Subject: [PATCH 6/6] Clean up code --- Lib/pdb.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 88c54cfd49edc9..c4483c3022c31b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -592,29 +592,23 @@ def default(self, line): self._error_exc() def _replace_convenience_variables(self, line): - """Replace the convenience variables in line""" + """Replace the convenience variables in 'line' with their values. + e.g. $foo is replaced by __pdb_convenience_variables["foo"]. + Note: such pattern in string literals will be skipped""" if "$" not in line: return line - last_token_is_dollar = False - dollar_start = None - dollar_end = None + dollar_start = dollar_end = -1 replace_variables = [] try: for t in tokenize.generate_tokens(io.StringIO(line).readline): token_type, token_string, start, end, _ = t if token_type == token.OP and token_string == '$': - last_token_is_dollar = True - dollar_end = end - dollar_start = start - else: - if (last_token_is_dollar and - token_type == token.NAME and - start == dollar_end): - # line is a one-line command so we only care about column - replace_variables.append((dollar_start[1], end[1], token_string)) - last_token_is_dollar = False + dollar_start, dollar_end = start, end + elif start == dollar_end and token_type == token.NAME: + # line is a one-line command so we only care about column + replace_variables.append((dollar_start[1], end[1], token_string)) except tokenize.TokenError: return line @@ -622,13 +616,13 @@ def _replace_convenience_variables(self, line): return line last_end = 0 - new_line = '' + line_pieces = [] for start, end, name in replace_variables: - new_line += line[last_end:start] + f'__pdb_convenience_variables["{name}"]' + line_pieces.append(line[last_end:start] + f'__pdb_convenience_variables["{name}"]') last_end = end - new_line += line[last_end:] + line_pieces.append(line[last_end:]) - return new_line + return ''.join(line_pieces) def precmd(self, line): """Handle alias expansion and ';;' separator."""