bpo-37961: tracemalloc: store the actual length of traceback

Doc/library/tracemalloc.rst

@@ -313,6 +313,9 @@ Functions
    frames. By default, a trace of a memory block only stores the most recent
    frame: the limit is ``1``. *nframe* must be greater or equal to ``1``.
 
+   You can still read the original number of total frames that composed the
+   traceback by looking at the :attr:`Traceback.total_nframe` attribute.
+
    Storing more than ``1`` frame is only useful to compute statistics grouped
    by ``'traceback'`` or to compute cumulative statistics: see the
    :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods.
@@ -659,13 +662,25 @@ Traceback
 
    When a snapshot is taken, tracebacks of traces are limited to
    :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function.
+   The original number of frames of the traceback is stored in the
+   :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback
+   has been truncated by the traceback limit.
 
    The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback`
    instance.
 
    .. versionchanged:: 3.7
       Frames are now sorted from the oldest to the most recent, instead of most recent to oldest.
 
+   .. attribute:: total_nframe
+
+      Total number of frames that composed the traceback before truncation.
+      This attribute can be set to ``None`` if the information is not
+      available.
+
+   .. versionchanged:: 3.9
+      The :attr:`Traceback.total_nframe` attribute was added.
+
    .. method:: format(limit=None, most_recent_first=False)
 
       Format the traceback as a list of lines with newlines. Use the

Lib/test/test_tracemalloc.py

@@ -36,7 +36,7 @@ def allocate_bytes(size):
     bytes_len = (size - EMPTY_STRING_SIZE)
     frames = get_frames(nframe, 1)
     data = b'x' * bytes_len
-    return data, tracemalloc.Traceback(frames)
+    return data, tracemalloc.Traceback(frames, min(len(frames), nframe))
 
 def create_snapshots():
     traceback_limit = 2
@@ -45,27 +45,27 @@ def create_snapshots():
     # traceback_frames) tuples. traceback_frames is a tuple of (filename,
     # line_number) tuples.
     raw_traces = [
-        (0, 10, (('a.py', 2), ('b.py', 4))),
-        (0, 10, (('a.py', 2), ('b.py', 4))),
-        (0, 10, (('a.py', 2), ('b.py', 4))),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
 
-        (1, 2, (('a.py', 5), ('b.py', 4))),
+        (1, 2, (('a.py', 5), ('b.py', 4)), 3),
 
-        (2, 66, (('b.py', 1),)),
+        (2, 66, (('b.py', 1),), 1),
 
-        (3, 7, (('<unknown>', 0),)),
+        (3, 7, (('<unknown>', 0),), 1),
     ]
     snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit)
 
     raw_traces2 = [
-        (0, 10, (('a.py', 2), ('b.py', 4))),
-        (0, 10, (('a.py', 2), ('b.py', 4))),
-        (0, 10, (('a.py', 2), ('b.py', 4))),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+        (0, 10, (('a.py', 2), ('b.py', 4)), 3),
 
-        (2, 2, (('a.py', 5), ('b.py', 4))),
-        (2, 5000, (('a.py', 5), ('b.py', 4))),
+        (2, 2, (('a.py', 5), ('b.py', 4)), 3),
+        (2, 5000, (('a.py', 5), ('b.py', 4)), 3),
 
-        (4, 400, (('c.py', 578),)),
+        (4, 400, (('c.py', 578),), 1),
     ]
     snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit)
 
@@ -125,7 +125,7 @@ def test_new_reference(self):
 
         nframe = tracemalloc.get_traceback_limit()
         frames = get_frames(nframe, -3)
-        obj_traceback = tracemalloc.Traceback(frames)
+        obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe))
 
         traceback = tracemalloc.get_object_traceback(obj)
         self.assertIsNotNone(traceback)
@@ -167,7 +167,7 @@ def test_get_traces(self):
         trace = self.find_trace(traces, obj_traceback)
 
         self.assertIsInstance(trace, tuple)
-        domain, size, traceback = trace
+        domain, size, traceback, length = trace
         self.assertEqual(size, obj_size)
         self.assertEqual(traceback, obj_traceback._frames)
 
@@ -197,8 +197,8 @@ def allocate_bytes4(size):
 
         trace1 = self.find_trace(traces, obj1_traceback)
         trace2 = self.find_trace(traces, obj2_traceback)
-        domain1, size1, traceback1 = trace1
-        domain2, size2, traceback2 = trace2
+        domain1, size1, traceback1, length1 = trace1
+        domain2, size2, traceback2, length2 = trace2
         self.assertIs(traceback2, traceback1)
 
     def test_get_traced_memory(self):
@@ -259,6 +259,9 @@ def test_snapshot(self):
         # take a snapshot
         snapshot = tracemalloc.take_snapshot()
 
+        # This can vary
+        self.assertGreater(snapshot.traces[1].traceback.total_nframe, 10)
+
         # write on disk
         snapshot.dump(support.TESTFN)
         self.addCleanup(support.unlink, support.TESTFN)
@@ -321,7 +324,7 @@ class TestSnapshot(unittest.TestCase):
     maxDiff = 4000
 
     def test_create_snapshot(self):
-        raw_traces = [(0, 5, (('a.py', 2),))]
+        raw_traces = [(0, 5, (('a.py', 2),), 10)]
 
         with contextlib.ExitStack() as stack:
             stack.enter_context(patch.object(tracemalloc, 'is_tracing',
@@ -336,6 +339,7 @@ def test_create_snapshot(self):
             self.assertEqual(len(snapshot.traces), 1)
             trace = snapshot.traces[0]
             self.assertEqual(trace.size, 5)
+            self.assertEqual(trace.traceback.total_nframe, 10)
             self.assertEqual(len(trace.traceback), 1)
             self.assertEqual(trace.traceback[0].filename, 'a.py')
             self.assertEqual(trace.traceback[0].lineno, 2)
@@ -351,11 +355,11 @@ def test_filter_traces(self):
         # exclude b.py
         snapshot3 = snapshot.filter_traces((filter1,))
         self.assertEqual(snapshot3.traces._traces, [
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (1, 2, (('a.py', 5), ('b.py', 4))),
-            (3, 7, (('<unknown>', 0),)),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (1, 2, (('a.py', 5), ('b.py', 4)), 3),
+            (3, 7, (('<unknown>', 0),), 1),
         ])
 
         # filter_traces() must not touch the original snapshot
@@ -364,10 +368,10 @@ def test_filter_traces(self):
         # only include two lines of a.py
         snapshot4 = snapshot3.filter_traces((filter2, filter3))
         self.assertEqual(snapshot4.traces._traces, [
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (1, 2, (('a.py', 5), ('b.py', 4))),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (1, 2, (('a.py', 5), ('b.py', 4)), 3),
         ])
 
         # No filter: just duplicate the snapshot
@@ -388,21 +392,21 @@ def test_filter_traces_domain(self):
         # exclude a.py of domain 1
         snapshot3 = snapshot.filter_traces((filter1,))
         self.assertEqual(snapshot3.traces._traces, [
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (2, 66, (('b.py', 1),)),
-            (3, 7, (('<unknown>', 0),)),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (2, 66, (('b.py', 1),), 1),
+            (3, 7, (('<unknown>', 0),), 1),
         ])
 
         # include domain 1
         snapshot3 = snapshot.filter_traces((filter1,))
         self.assertEqual(snapshot3.traces._traces, [
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (2, 66, (('b.py', 1),)),
-            (3, 7, (('<unknown>', 0),)),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (2, 66, (('b.py', 1),), 1),
+            (3, 7, (('<unknown>', 0),), 1),
         ])
 
     def test_filter_traces_domain_filter(self):
@@ -413,17 +417,17 @@ def test_filter_traces_domain_filter(self):
         # exclude domain 2
         snapshot3 = snapshot.filter_traces((filter1,))
         self.assertEqual(snapshot3.traces._traces, [
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (0, 10, (('a.py', 2), ('b.py', 4))),
-            (1, 2, (('a.py', 5), ('b.py', 4))),
-            (2, 66, (('b.py', 1),)),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (0, 10, (('a.py', 2), ('b.py', 4)), 3),
+            (1, 2, (('a.py', 5), ('b.py', 4)), 3),
+            (2, 66, (('b.py', 1),), 1),
         ])
 
         # include domain 2
         snapshot3 = snapshot.filter_traces((filter2,))
         self.assertEqual(snapshot3.traces._traces, [
-            (3, 7, (('<unknown>', 0),)),
+            (3, 7, (('<unknown>', 0),), 1),
         ])
 
     def test_snapshot_group_by_line(self):

Lib/tracemalloc.py

@@ -182,15 +182,20 @@ class Traceback(Sequence):
     Sequence of Frame instances sorted from the oldest frame
     to the most recent frame.
     """
-    __slots__ = ("_frames",)
+    __slots__ = ("_frames", '_total_nframe')
 
-    def __init__(self, frames):
+    def __init__(self, frames, total_nframe=None):
         Sequence.__init__(self)
         # frames is a tuple of frame tuples: see Frame constructor for the
         # format of a frame tuple; it is reversed, because _tracemalloc
         # returns frames sorted from most recent to oldest, but the
         # Python API expects oldest to most recent
         self._frames = tuple(reversed(frames))
+        self._total_nframe = total_nframe
+
+    @property
+    def total_nframe(self):
+        return self._total_nframe
 
     def __len__(self):
         return len(self._frames)
@@ -221,7 +226,12 @@ def __str__(self):
         return str(self[0])
 
     def __repr__(self):
-        return "<Traceback %r>" % (tuple(self),)
+        s = "<Traceback %r" % tuple(self)
+        if self._total_nframe is None:
+            s += ">"
+        else:
+            s += f" total_nframe={self.total_nframe}>"
+        return s
 
     def format(self, limit=None, most_recent_first=False):
         lines = []
@@ -280,7 +290,7 @@ def size(self):
 
     @property
     def traceback(self):
-        return Traceback(self._trace[2])
+        return Traceback(*self._trace[2:])
 
     def __eq__(self, other):
         if not isinstance(other, Trace):
@@ -378,7 +388,7 @@ def _match_traceback(self, traceback):
             return self._match_frame(filename, lineno)
 
     def _match(self, trace):
-        domain, size, traceback = trace
+        domain, size, traceback, total_nframe = trace
         res = self._match_traceback(traceback)
         if self.domain is not None:
             if self.inclusive:
@@ -398,7 +408,7 @@ def domain(self):
         return self._domain
 
     def _match(self, trace):
-        domain, size, traceback = trace
+        domain, size, traceback, total_nframe = trace
         return (domain == self.domain) ^ (not self.inclusive)
 
 
@@ -475,7 +485,7 @@ def _group_by(self, key_type, cumulative):
         tracebacks = {}
         if not cumulative:
             for trace in self.traces._traces:
-                domain, size, trace_traceback = trace
+                domain, size, trace_traceback, total_nframe = trace
                 try:
                     traceback = tracebacks[trace_traceback]
                 except KeyError:
@@ -496,7 +506,7 @@ def _group_by(self, key_type, cumulative):
         else:
             # cumulative statistics
             for trace in self.traces._traces:
-                domain, size, trace_traceback = trace
+                domain, size, trace_traceback, total_nframe = trace
                 for frame in trace_traceback:
                     try:
                         traceback = tracebacks[frame]

Misc/NEWS.d/next/Library/2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst

@@ -0,0 +1,2 @@
+Add a ``total_nframe`` field to the traces collected by the tracemalloc module.
+This field indicates the original number of frames before it was truncated.