Deserialize by using __new__ (rebased 73)

.coveragerc

@@ -13,4 +13,6 @@ patch = subprocess
 [report]
 show_missing = true
 precision = 2
-omit = *migrations*
+omit =
+    *migrations*
+    tests/bad*.py

.github/workflows/github-actions.yml

@@ -251,6 +251,12 @@ jobs:
     - name: test
       run: >
         tox -e ${{ matrix.tox_env }} -v
+    - uses: coverallsapp/github-action@v2
+      if: matrix.cover
+      continue-on-error: true
+      with:
+        flag-name: ${{ matrix.name }}
+        parallel: true
     - uses: codecov/codecov-action@v5
       if: matrix.cover
       with:
@@ -262,3 +268,6 @@ jobs:
     if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:
+    - uses: coverallsapp/github-action@v2
+      with:
+        parallel-finished: true

MANIFEST.in

@@ -20,5 +20,6 @@ include CHANGELOG.rst
 include CONTRIBUTING.rst
 include LICENSE
 include README.rst
+include SECURITY.md
 
 global-exclude *.py[cod] __pycache__/* *.so *.dylib

README.rst

@@ -144,7 +144,7 @@ Raising
 
 ::
 
-    >>> from six import reraise
+    >>> from tblib.decorators import reraise
     >>> reraise(*pickle.loads(s1))
     Traceback (most recent call last):
       ...
@@ -431,22 +431,26 @@ json.JSONDecoder::
     {'tb_frame': {'f_code': {'co_filename': '<doctest README.rst[...]>',
                              'co_name': '<module>'},
                   'f_globals': {'__name__': '__main__'},
-                  'f_lineno': 5},
+                  'f_lineno': 5,
+                  'f_locals': {}},
      'tb_lineno': 2,
      'tb_next': {'tb_frame': {'f_code': {'co_filename': ...,
                                          'co_name': 'inner_2'},
                               'f_globals': {'__name__': '__main__'},
-                              'f_lineno': 2},
+                              'f_lineno': 2,
+                              'f_locals': {}},
                  'tb_lineno': 2,
                  'tb_next': {'tb_frame': {'f_code': {'co_filename': ...,
                                                      'co_name': 'inner_1'},
                                           'f_globals': {'__name__': '__main__'},
-                                          'f_lineno': 2},
+                                          'f_lineno': 2,
+                                          'f_locals': {}},
                              'tb_lineno': 2,
                              'tb_next': {'tb_frame': {'f_code': {'co_filename': ...,
                                                                  'co_name': 'inner_0'},
                                                       'f_globals': {'__name__': '__main__'},
-                                                      'f_lineno': 2},
+                                                      'f_lineno': 2,
+                                                      'f_locals': {}},
                                          'tb_lineno': 2,
                                          'tb_next': None}}}}
 
@@ -501,7 +505,7 @@ tblib.Traceback.from_string
       File "...examples.py", line 10, in func_c
         func_d()
       File "...examples.py", line 14, in func_d
-        raise Exception("Guessing time !")
+        raise Exception('Guessing time !')
     Exception: fail
 
 
@@ -532,7 +536,7 @@ If you use the ``strict=False`` option then parsing is a bit more lax::
       File "...examples.py", line 10, in func_c
         func_d()
       File "...examples.py", line 14, in func_d
-        raise Exception("Guessing time !")
+        raise Exception('Guessing time !')
     Exception: fail
 
 tblib.decorators.return_error
@@ -591,7 +595,7 @@ Not very useful is it? Let's sort this out::
     >>> from tblib.decorators import apply_with_return_error, Error
     >>> from itertools import repeat
     >>> pool = Pool()
-    >>> try:
+    >>> try: # doctest: +SKIP
     ...     for i in pool.map(apply_with_return_error, zip(repeat(func_a), range(5))):
     ...         if isinstance(i, Error):
     ...             i.reraise()
@@ -605,6 +609,8 @@ Not very useful is it? Let's sort this out::
         i.reraise()
       File "...tblib...decorators.py", line ..., in reraise
         reraise(self.exc_type, self.exc_value, self.traceback)
+      File "...tblib...decorators.py", line ..., in reraise
+        raise value.with_traceback(tb)
       File "...tblib...decorators.py", line ..., in return_exceptions_wrapper
         return func(*args, **kwargs)
       File "...tblib...decorators.py", line ..., in apply_with_return_error
@@ -616,7 +622,7 @@ Not very useful is it? Let's sort this out::
       File "...examples.py", line 10, in func_c
         func_d()
       File "...examples.py", line 14, in func_d
-        raise Exception("Guessing time !")
+        raise Exception('Guessing time !')
     Exception: Guessing time !
     <BLANKLINE>
     >>> pool.terminate()
@@ -645,10 +651,11 @@ What if we have a local call stack ?
     >>> def local_2():
     ...     local_1()
     ...
-    >>> try:
+    >>> try: # doctest: +SKIP
     ...     local_2()
     ... except:
     ...     print(traceback.format_exc())
+    ...
     Traceback (most recent call last):
       File "<doctest README.rst[...]>", line 2, in <module>
         local_2()
@@ -658,11 +665,13 @@ What if we have a local call stack ?
         local_0()
       File "<doctest README.rst[...]>", line 6, in local_0
         i.reraise()
-      File "...tblib...decorators.py", line 20, in reraise
+      File "...tblib...decorators.py", line ..., in reraise
         reraise(self.exc_type, self.exc_value, self.traceback)
-      File "...tblib...decorators.py", line 27, in return_exceptions_wrapper
+      File "...tblib...decorators.py", line ..., in reraise
+        raise value.with_traceback(tb)
+      File "...tblib...decorators.py", line ..., in return_exceptions_wrapper
         return func(*args, **kwargs)
-      File "...tblib...decorators.py", line 47, in apply_with_return_error
+      File "...tblib...decorators.py", line ..., in apply_with_return_error
         return args[0](*args[1:])
       File "...tests...examples.py", line 2, in func_a
         func_b()
@@ -671,7 +680,7 @@ What if we have a local call stack ?
       File "...tests...examples.py", line 10, in func_c
         func_d()
       File "...tests...examples.py", line 14, in func_d
-        raise Exception("Guessing time !")
+        raise Exception('Guessing time !')
     Exception: Guessing time !
     <BLANKLINE>
 

ci/templates/.github/workflows/github-actions.yml

@@ -59,6 +59,12 @@ jobs:
     - name: test
       run: >
         tox -e {{ '${{ matrix.tox_env }}' }} -v
+    - uses: coverallsapp/github-action@v2
+      if: matrix.cover
+      continue-on-error: true
+      with:
+        flag-name: {{ '${{ matrix.name }}' }}
+        parallel: true
     - uses: codecov/codecov-action@v5
       if: matrix.cover
       with:
@@ -70,4 +76,7 @@ jobs:
     if: {{ '${{ always() }}' }}
     runs-on: ubuntu-latest
     steps:
+    - uses: coverallsapp/github-action@v2
+      with:
+        parallel-finished: true
 {{ '' }}

pyproject.toml

@@ -55,6 +55,8 @@ dependencies = [
 [dependency-groups]
 test = [
     "pytest",
+    "twisted",
+    "pytest-benchmark",
 ]
 
 [project.urls]
@@ -80,6 +82,7 @@ ignore = [
     "S603",   # flake8-bandit subprocess-without-shell-equals-true
     "S607",   # flake8-bandit start-process-with-partial-path
     "E501",   # pycodestyle line-too-long
+    "S301",
 ]
 select = [
     "B",   # flake8-bugbear

src/tblib/pickling_support.py

@@ -22,6 +22,19 @@ def pickle_traceback(tb, *, get_locals=None):
     )
 
 
+def unpickle_exception_with_attrs(func, attrs, cause, tb, context, suppress_context, notes):
+    inst = func.__new__(func)
+    for key, value in attrs.items():
+        setattr(inst, key, value)
+    inst.__cause__ = cause
+    inst.__traceback__ = tb
+    inst.__context__ = context
+    inst.__suppress_context__ = suppress_context
+    if notes is not None:
+        inst.__notes__ = notes
+    return inst
+
+
 # Note: Older versions of tblib will generate pickle archives that call unpickle_exception() with
 # fewer arguments. We assign default values to some of the arguments to support this.
 def unpickle_exception(func, args, cause, tb, context=None, suppress_context=False, notes=None):
@@ -35,33 +48,62 @@ def unpickle_exception(func, args, cause, tb, context=None, suppress_context=Fal
     return inst
 
 
-def pickle_exception(obj):
-    # All exceptions, unlike generic Python objects, define __reduce_ex__
-    # __reduce_ex__(4) should be no different from __reduce_ex__(3).
-    # __reduce_ex__(5) could bring benefits in the unlikely case the exception
-    # directly contains buffers, but PickleBuffer objects will cause a crash when
-    # running on protocol=4, and there's no clean way to figure out the current
-    # protocol from here. Note that any object returned by __reduce_ex__(3) will
-    # still be pickled with protocol 5 if pickle.dump() is running with it.
-    rv = obj.__reduce_ex__(3)
-    if isinstance(rv, str):
-        raise TypeError('str __reduce__ output is not supported')
-    assert isinstance(rv, tuple)
-    assert len(rv) >= 2
-
-    return (
-        unpickle_exception,
-        (
-            *rv[:2],
-            obj.__cause__,
-            obj.__traceback__,
-            obj.__context__,
-            obj.__suppress_context__,
-            # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent
-            getattr(obj, '__notes__', None),
-        ),
-        *rv[2:],
-    )
+def pickle_exception(
+    obj, builtin_reducers=(OSError.__reduce__, BaseException.__reduce__), builtin_inits=(OSError.__init__, BaseException.__init__)
+):
+    reduced_value = obj.__reduce__()
+    if isinstance(reduced_value, str):
+        raise TypeError('Did not expect {repr(obj)}.__reduce__() to return a string!')
+
+    func = type(obj)
+    # Detect busted objects: they have a custom __init__ but no __reduce__.
+    # This also means the resulting exceptions may be a bit "dulled" down - the args from __reduce__ are discarded.
+    if func.__reduce__ in builtin_reducers and func.__init__ not in builtin_inits:
+        _, args, *optionals = reduced_value
+        attrs = {
+            '__dict__': obj.__dict__,
+            'args': obj.args,
+        }
+        if isinstance(obj, OSError):
+            attrs.update(
+                errno=obj.errno,
+                strerror=obj.strerror,
+                winerror=getattr(obj, 'winerror', None),
+                filename=obj.filename,
+                filename2=obj.filename2,
+            )
+
+        return (
+            unpickle_exception_with_attrs,
+            (
+                func,
+                attrs,
+                obj.__cause__,
+                obj.__traceback__,
+                obj.__context__,
+                obj.__suppress_context__,
+                # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent
+                getattr(obj, '__notes__', None),
+            ),
+            *optionals,
+        )
+    else:
+        func, args, *optionals = reduced_value
+
+        return (
+            unpickle_exception,
+            (
+                func,
+                args,
+                obj.__cause__,
+                obj.__traceback__,
+                obj.__context__,
+                obj.__suppress_context__,
+                # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent
+                getattr(obj, '__notes__', None),
+            ),
+            *optionals,
+        )
 
 
 def _get_subclasses(cls):

tests/test_issue30.py

@@ -20,7 +20,7 @@ def test_30():
 
     f = None
     try:
-        _etype, evalue, etb = pickle.loads(s)  # noqa: S301
+        _etype, evalue, etb = pickle.loads(s)
         raise evalue.with_traceback(etb)
     except ValueError:
         f = Failure()

tests/test_issue65.py

@@ -0,0 +1,27 @@
+import pickle
+
+from tblib import pickling_support
+
+
+class HTTPrettyError(Exception):
+    pass
+
+
+class UnmockedError(HTTPrettyError):
+    def __init__(self):
+        super().__init__('No mocking was registered, and real connections are not allowed (httpretty.allow_net_connect = False).')
+
+
+def test_65():
+    pickling_support.install()
+
+    try:
+        raise UnmockedError
+    except Exception as e:
+        exc = e
+
+    exc = pickle.loads(pickle.dumps(exc))
+
+    assert isinstance(exc, UnmockedError)
+    assert exc.args == ('No mocking was registered, and real connections are not allowed (httpretty.allow_net_connect = False).',)
+    assert exc.__traceback__ is not None