Adjust inspect.getsource to properly extract source for decorated functions

When the arguments to a decorator contain any additional parentheses,
``inspect.BlockFinder`` was misinterpreting the closing paren as the closing
paren of the decorator, leading to a clipped result of the extracted source
of a function. Instead of assuming the closing paren closes the decorator
call itself, we keep track of all the parentheses in a new stack, making
sure to unset the decorator context once we consumed all of them.

Author: PCManticore

Lib/inspect.py

@@ -899,6 +899,7 @@ def __init__(self):
         self.indecorator = False
         self.decoratorhasargs = False
         self.last = 1
+        self.decoratorparens = []
 
     def tokeneater(self, type, token, srowcol, erowcol, line):
         if not self.started and not self.indecorator:
@@ -913,11 +914,18 @@ def tokeneater(self, type, token, srowcol, erowcol, line):
             self.passline = True    # skip to the end of the line
         elif token == "(":
             if self.indecorator:
+                self.decoratorparens.append(token)
                 self.decoratorhasargs = True
         elif token == ")":
             if self.indecorator:
-                self.indecorator = False
-                self.decoratorhasargs = False
+                if self.decoratorparens:
+                    if self.decoratorparens[-1] == "(":
+                        self.decoratorparens = self.decoratorparens[:-1]
+                    if not self.decoratorparens:
+                        self.indecorator = False
+                        self.decoratorhasargs = False
+                else:
+                    self.decoratorparens.append(token)
         elif type == tokenize.NEWLINE:
             self.passline = False   # stop skipping when a NEWLINE is seen
             self.last = srowcol[0]

Lib/test/inspect_fodder.py

@@ -1,7 +1,7 @@
 # line 1
 'A module docstring.'
 
-import sys, inspect
+import sys, collections, functools, inspect
 # line 5
 
 # line 7
@@ -91,3 +91,32 @@ def as_method_of(self, obj):
 
 custom_method = Callable().as_method_of(42)
 del Callable
+
+
+def decorator(*args):
+    def inner(func):
+        @functools.wraps(func)
+        def wrapper():
+            pass
+        return wrapper
+    return inner
+
+
+@decorator(dict(), 24)
+@decorator(dict(), 1)
+@decorator(dict(), collections.defaultdict(lambda: 1))
+@decorator("string containing )", "other string ()")
+@decorator(
+    (()),
+    collections.defaultdict(lambda: 1),
+    [(i, j) for i, j in enumerate(range(5))],
+)
+def decorated():
+    local_var = 2
+    return local_var + 42
+
+
+@decorator()
+def other_decorated():
+    local_var = 2
+    return local_var + 42

Lib/test/test_inspect.py

@@ -419,8 +419,11 @@ def test_getclasses(self):
 
     def test_getfunctions(self):
         functions = inspect.getmembers(mod, inspect.isfunction)
-        self.assertEqual(functions, [('eggs', mod.eggs),
+        self.assertEqual(functions, [('decorated', mod.decorated),
+                                     ('decorator', mod.decorator),
+                                     ('eggs', mod.eggs),
                                      ('lobbest', mod.lobbest),
+                                     ('other_decorated', mod.other_decorated),
                                      ('spam', mod.spam)])
 
     @unittest.skipIf(sys.flags.optimize >= 2,
@@ -493,6 +496,8 @@ def test_getsource(self):
         self.assertSourceEqual(git.abuse, 29, 39)
         self.assertSourceEqual(mod.StupidGit, 21, 51)
         self.assertSourceEqual(mod.lobbest, 75, 76)
+        self.assertSourceEqual(mod.decorated, 105, 116)
+        self.assertSourceEqual(mod.other_decorated, 119, 122)
 
     def test_getsourcefile(self):
         self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile)

Misc/NEWS.d/next/Library/2019-11-25-09-37-50.bpo-38854.o-4kLU.rst

@@ -0,0 +1,3 @@
+Properly extract source with ``inspect.getsource`` for decorated functions.
+
+Patch by Claudiu Popa