diff --git a/.coveragerc b/.coveragerc index 1fc951c..ca3d005 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,4 +13,6 @@ patch = subprocess [report] show_missing = true precision = 2 -omit = *migrations* +omit = + *migrations* + tests/bad*.py diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 8f18b6c..a76bf38 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -251,6 +251,12 @@ jobs: - name: test run: > tox -e ${{ matrix.tox_env }} -v + - uses: coverallsapp/github-action@v2 + if: matrix.cover + continue-on-error: true + with: + flag-name: ${{ matrix.name }} + parallel: true - uses: codecov/codecov-action@v5 if: matrix.cover with: @@ -262,3 +268,6 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest steps: + - uses: coverallsapp/github-action@v2 + with: + parallel-finished: true diff --git a/MANIFEST.in b/MANIFEST.in index 4bac1f0..4ad6232 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,5 +20,6 @@ include CHANGELOG.rst include CONTRIBUTING.rst include LICENSE include README.rst +include SECURITY.md global-exclude *.py[cod] __pycache__/* *.so *.dylib diff --git a/README.rst b/README.rst index 1ed2db9..bed352a 100644 --- a/README.rst +++ b/README.rst @@ -144,7 +144,7 @@ Raising :: - >>> from six import reraise + >>> from tblib.decorators import reraise >>> reraise(*pickle.loads(s1)) Traceback (most recent call last): ... @@ -431,22 +431,26 @@ json.JSONDecoder:: {'tb_frame': {'f_code': {'co_filename': '', 'co_name': ''}, 'f_globals': {'__name__': '__main__'}, - 'f_lineno': 5}, + 'f_lineno': 5, + 'f_locals': {}}, 'tb_lineno': 2, 'tb_next': {'tb_frame': {'f_code': {'co_filename': ..., 'co_name': 'inner_2'}, 'f_globals': {'__name__': '__main__'}, - 'f_lineno': 2}, + 'f_lineno': 2, + 'f_locals': {}}, 'tb_lineno': 2, 'tb_next': {'tb_frame': {'f_code': {'co_filename': ..., 'co_name': 'inner_1'}, 'f_globals': {'__name__': '__main__'}, - 'f_lineno': 2}, + 'f_lineno': 2, + 'f_locals': {}}, 'tb_lineno': 2, 'tb_next': {'tb_frame': {'f_code': {'co_filename': ..., 'co_name': 'inner_0'}, 'f_globals': {'__name__': '__main__'}, - 'f_lineno': 2}, + 'f_lineno': 2, + 'f_locals': {}}, 'tb_lineno': 2, 'tb_next': None}}}} @@ -501,7 +505,7 @@ tblib.Traceback.from_string File "...examples.py", line 10, in func_c func_d() File "...examples.py", line 14, in func_d - raise Exception("Guessing time !") + raise Exception('Guessing time !') Exception: fail @@ -532,7 +536,7 @@ If you use the ``strict=False`` option then parsing is a bit more lax:: File "...examples.py", line 10, in func_c func_d() File "...examples.py", line 14, in func_d - raise Exception("Guessing time !") + raise Exception('Guessing time !') Exception: fail tblib.decorators.return_error @@ -591,7 +595,7 @@ Not very useful is it? Let's sort this out:: >>> from tblib.decorators import apply_with_return_error, Error >>> from itertools import repeat >>> pool = Pool() - >>> try: + >>> try: # doctest: +SKIP ... for i in pool.map(apply_with_return_error, zip(repeat(func_a), range(5))): ... if isinstance(i, Error): ... i.reraise() @@ -605,6 +609,8 @@ Not very useful is it? Let's sort this out:: i.reraise() File "...tblib...decorators.py", line ..., in reraise reraise(self.exc_type, self.exc_value, self.traceback) + File "...tblib...decorators.py", line ..., in reraise + raise value.with_traceback(tb) File "...tblib...decorators.py", line ..., in return_exceptions_wrapper return func(*args, **kwargs) File "...tblib...decorators.py", line ..., in apply_with_return_error @@ -616,7 +622,7 @@ Not very useful is it? Let's sort this out:: File "...examples.py", line 10, in func_c func_d() File "...examples.py", line 14, in func_d - raise Exception("Guessing time !") + raise Exception('Guessing time !') Exception: Guessing time ! >>> pool.terminate() @@ -645,10 +651,11 @@ What if we have a local call stack ? >>> def local_2(): ... local_1() ... - >>> try: + >>> try: # doctest: +SKIP ... local_2() ... except: ... print(traceback.format_exc()) + ... Traceback (most recent call last): File "", line 2, in local_2() @@ -658,11 +665,13 @@ What if we have a local call stack ? local_0() File "", line 6, in local_0 i.reraise() - File "...tblib...decorators.py", line 20, in reraise + File "...tblib...decorators.py", line ..., in reraise reraise(self.exc_type, self.exc_value, self.traceback) - File "...tblib...decorators.py", line 27, in return_exceptions_wrapper + File "...tblib...decorators.py", line ..., in reraise + raise value.with_traceback(tb) + File "...tblib...decorators.py", line ..., in return_exceptions_wrapper return func(*args, **kwargs) - File "...tblib...decorators.py", line 47, in apply_with_return_error + File "...tblib...decorators.py", line ..., in apply_with_return_error return args[0](*args[1:]) File "...tests...examples.py", line 2, in func_a func_b() @@ -671,7 +680,7 @@ What if we have a local call stack ? File "...tests...examples.py", line 10, in func_c func_d() File "...tests...examples.py", line 14, in func_d - raise Exception("Guessing time !") + raise Exception('Guessing time !') Exception: Guessing time ! diff --git a/ci/templates/.github/workflows/github-actions.yml b/ci/templates/.github/workflows/github-actions.yml index ac17262..ff7b53f 100644 --- a/ci/templates/.github/workflows/github-actions.yml +++ b/ci/templates/.github/workflows/github-actions.yml @@ -59,6 +59,12 @@ jobs: - name: test run: > tox -e {{ '${{ matrix.tox_env }}' }} -v + - uses: coverallsapp/github-action@v2 + if: matrix.cover + continue-on-error: true + with: + flag-name: {{ '${{ matrix.name }}' }} + parallel: true - uses: codecov/codecov-action@v5 if: matrix.cover with: @@ -70,4 +76,7 @@ jobs: if: {{ '${{ always() }}' }} runs-on: ubuntu-latest steps: + - uses: coverallsapp/github-action@v2 + with: + parallel-finished: true {{ '' }} diff --git a/pyproject.toml b/pyproject.toml index 1e99e81..54f2108 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,8 @@ dependencies = [ [dependency-groups] test = [ "pytest", + "twisted", + "pytest-benchmark", ] [project.urls] @@ -80,6 +82,7 @@ ignore = [ "S603", # flake8-bandit subprocess-without-shell-equals-true "S607", # flake8-bandit start-process-with-partial-path "E501", # pycodestyle line-too-long + "S301", ] select = [ "B", # flake8-bugbear diff --git a/src/tblib/pickling_support.py b/src/tblib/pickling_support.py index 48670ba..a561aa4 100644 --- a/src/tblib/pickling_support.py +++ b/src/tblib/pickling_support.py @@ -22,6 +22,19 @@ def pickle_traceback(tb, *, get_locals=None): ) +def unpickle_exception_with_attrs(func, attrs, cause, tb, context, suppress_context, notes): + inst = func.__new__(func) + for key, value in attrs.items(): + setattr(inst, key, value) + inst.__cause__ = cause + inst.__traceback__ = tb + inst.__context__ = context + inst.__suppress_context__ = suppress_context + if notes is not None: + inst.__notes__ = notes + return inst + + # Note: Older versions of tblib will generate pickle archives that call unpickle_exception() with # fewer arguments. We assign default values to some of the arguments to support this. def unpickle_exception(func, args, cause, tb, context=None, suppress_context=False, notes=None): @@ -35,33 +48,62 @@ def unpickle_exception(func, args, cause, tb, context=None, suppress_context=Fal return inst -def pickle_exception(obj): - # All exceptions, unlike generic Python objects, define __reduce_ex__ - # __reduce_ex__(4) should be no different from __reduce_ex__(3). - # __reduce_ex__(5) could bring benefits in the unlikely case the exception - # directly contains buffers, but PickleBuffer objects will cause a crash when - # running on protocol=4, and there's no clean way to figure out the current - # protocol from here. Note that any object returned by __reduce_ex__(3) will - # still be pickled with protocol 5 if pickle.dump() is running with it. - rv = obj.__reduce_ex__(3) - if isinstance(rv, str): - raise TypeError('str __reduce__ output is not supported') - assert isinstance(rv, tuple) - assert len(rv) >= 2 - - return ( - unpickle_exception, - ( - *rv[:2], - obj.__cause__, - obj.__traceback__, - obj.__context__, - obj.__suppress_context__, - # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent - getattr(obj, '__notes__', None), - ), - *rv[2:], - ) +def pickle_exception( + obj, builtin_reducers=(OSError.__reduce__, BaseException.__reduce__), builtin_inits=(OSError.__init__, BaseException.__init__) +): + reduced_value = obj.__reduce__() + if isinstance(reduced_value, str): + raise TypeError('Did not expect {repr(obj)}.__reduce__() to return a string!') + + func = type(obj) + # Detect busted objects: they have a custom __init__ but no __reduce__. + # This also means the resulting exceptions may be a bit "dulled" down - the args from __reduce__ are discarded. + if func.__reduce__ in builtin_reducers and func.__init__ not in builtin_inits: + _, args, *optionals = reduced_value + attrs = { + '__dict__': obj.__dict__, + 'args': obj.args, + } + if isinstance(obj, OSError): + attrs.update( + errno=obj.errno, + strerror=obj.strerror, + winerror=getattr(obj, 'winerror', None), + filename=obj.filename, + filename2=obj.filename2, + ) + + return ( + unpickle_exception_with_attrs, + ( + func, + attrs, + obj.__cause__, + obj.__traceback__, + obj.__context__, + obj.__suppress_context__, + # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent + getattr(obj, '__notes__', None), + ), + *optionals, + ) + else: + func, args, *optionals = reduced_value + + return ( + unpickle_exception, + ( + func, + args, + obj.__cause__, + obj.__traceback__, + obj.__context__, + obj.__suppress_context__, + # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent + getattr(obj, '__notes__', None), + ), + *optionals, + ) def _get_subclasses(cls): diff --git a/tests/test_issue30.py b/tests/test_issue30.py index 09d3069..a446914 100644 --- a/tests/test_issue30.py +++ b/tests/test_issue30.py @@ -20,7 +20,7 @@ def test_30(): f = None try: - _etype, evalue, etb = pickle.loads(s) # noqa: S301 + _etype, evalue, etb = pickle.loads(s) raise evalue.with_traceback(etb) except ValueError: f = Failure() diff --git a/tests/test_issue65.py b/tests/test_issue65.py new file mode 100644 index 0000000..9da4a9f --- /dev/null +++ b/tests/test_issue65.py @@ -0,0 +1,27 @@ +import pickle + +from tblib import pickling_support + + +class HTTPrettyError(Exception): + pass + + +class UnmockedError(HTTPrettyError): + def __init__(self): + super().__init__('No mocking was registered, and real connections are not allowed (httpretty.allow_net_connect = False).') + + +def test_65(): + pickling_support.install() + + try: + raise UnmockedError + except Exception as e: + exc = e + + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, UnmockedError) + assert exc.args == ('No mocking was registered, and real connections are not allowed (httpretty.allow_net_connect = False).',) + assert exc.__traceback__ is not None diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py index a16c3a1..476a47d 100644 --- a/tests/test_pickle_exception.py +++ b/tests/test_pickle_exception.py @@ -72,7 +72,7 @@ def test_install(clear_dispatch_table, how, protocol): if how == 'instance': tblib.pickling_support.install(exc) if protocol: - exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) # noqa: S301 + exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) assert isinstance(exc, CustomError) assert exc.args == ('foo',) @@ -105,7 +105,7 @@ def test_install_decorator(): raise RegisteredError('foo') exc = ewrap.value exc.x = 1 - exc = pickle.loads(pickle.dumps(exc)) # noqa: S301 + exc = pickle.loads(pickle.dumps(exc)) assert isinstance(exc, RegisteredError) assert exc.args == ('foo',) @@ -162,5 +162,157 @@ def func(my_arg='2'): if how == 'instance': tblib.pickling_support.install(exc, get_locals=get_locals) - exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) # noqa: S301 + exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) assert exc.__traceback__.tb_next.tb_frame.f_locals == {'my_variable': 1} + + +class CustomWithAttributesException(Exception): + def __init__(self, message, arg1, arg2, arg3): + super().__init__(message) + self.values12 = (arg1, arg2) + self.value3 = arg3 + + +def test_custom_with_attributes(): + try: + raise CustomWithAttributesException('bar', 1, 2, 3) + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, CustomWithAttributesException) + assert exc.args == ('bar',) + assert exc.values12 == (1, 2) + assert exc.value3 == 3 + assert exc.__traceback__ is not None + + +class CustomOSError(OSError): + def __init__(self, message, errno, strerror: str, filename, none: None, filename2): + super().__init__(errno, strerror, filename, none, filename2) + self.message = message + + +def test_custom_oserror(): + try: + raise CustomOSError('bar', 2, 'err', 3, None, 5) + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, CustomOSError) + assert exc.message == 'bar' + assert exc.errno == 2 + assert exc.strerror == 'err' + assert exc.filename == 3 + assert exc.filename2 == 5 + assert exc.__traceback__ is not None + + +def test_oserror(): + try: + raise OSError(2, 'err', 3, None, 5) + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, OSError) + assert exc.errno == 2 + assert exc.strerror == 'err' + assert exc.filename == 3 + assert exc.filename2 == 5 + assert exc.__traceback__ is not None + + +class BadError(Exception): + def __init__(self): + super().__init__('Bad Bad Bad!') + + +def test_baderror(): + try: + raise BadError + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, BadError) + assert exc.args == ('Bad Bad Bad!',) + assert exc.__traceback__ is not None + + +class CustomReduceException(Exception): + def __init__(self, message, arg1, arg2, arg3): + super().__init__(message) + self.values12 = (arg1, arg2) + self.value3 = arg3 + + def __reduce__(self): + return self.__class__, self.args + self.values12 + (self.value3,) + + +def test_custom_reduce(): + try: + raise CustomReduceException('foo', 1, 2, 3) + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, CustomReduceException) + assert exc.args == ('foo',) + assert exc.values12 == (1, 2) + assert exc.value3 == 3 + assert exc.__traceback__ is not None + + +class CustomReduceExException(Exception): + def __init__(self, message, arg1, arg2, protocol): + super().__init__(message) + self.values12 = (arg1, arg2) + self.value3 = protocol + + def __reduce_ex__(self, protocol): + return self.__class__, self.args + self.values12 + (self.value3,) + + +@pytest.mark.parametrize('protocol', [None, *list(range(1, pickle.HIGHEST_PROTOCOL + 1))]) +def test_custom_reduce_ex(protocol): + try: + raise CustomReduceExException('foo', 1, 2, 3) + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) + + assert isinstance(exc, CustomReduceExException) + assert exc.args == ('foo',) + assert exc.values12 == (1, 2) + assert exc.value3 == 3 + assert exc.__traceback__ is not None + + +def test_oserror_simple(): + try: + raise OSError(13, 'Permission denied') + except Exception as e: + exc = e + + tblib.pickling_support.install(exc) + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, OSError) + assert exc.args == (13, 'Permission denied') + assert exc.errno == 13 + assert exc.strerror == 'Permission denied' + assert exc.__traceback__ is not None diff --git a/tests/test_tblib.py b/tests/test_tblib.py index bac06e1..6d01ebe 100644 --- a/tests/test_tblib.py +++ b/tests/test_tblib.py @@ -107,7 +107,7 @@ def test_parse_traceback(): }, } tb3 = Traceback.from_dict(expected_dict) - tb4 = pickle.loads(pickle.dumps(tb3)) # noqa: S301 + tb4 = pickle.loads(pickle.dumps(tb3)) assert tb4.as_dict() == tb3.as_dict() == tb2.as_dict() == tb1.as_dict() == expected_dict diff --git a/tox.ini b/tox.ini index ed14c7c..542cbf8 100644 --- a/tox.ini +++ b/tox.ini @@ -31,10 +31,9 @@ dependency_groups = test deps = pytest-cov - pytest-benchmark setuptools>=80 commands = - {posargs:pytest --cov --cov-report=term-missing --cov-report=xml -vv tests} + {posargs:pytest --cov --cov-report=term-missing --cov-report=xml -vv tests README.rst} [testenv:check] deps =