From a617210c7ddb84e38e76eab2e503747f86296f2b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 19 Jun 2023 13:31:42 -0300 Subject: [PATCH 1/7] Add support for py3.11. Remove py27, py35, py36, py37 --- .github/workflows/main.yml | 30 ++++++------------------------ setup.py | 7 ++----- tox.ini | 2 +- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2f096ac..6091126d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,30 +20,6 @@ jobs: env: TOXENV: "json" - - python-version: "2.7" - env: - TOXENV: "msgpack" - - python-version: "2.7" - env: - TOXENV: "json" - - python-version: "3.5" - env: - TOXENV: "msgpack" - - python-version: "3.5" - env: - TOXENV: "json" - - python-version: "3.6" - env: - TOXENV: "msgpack" - - python-version: "3.6" - env: - TOXENV: "json" - - python-version: "3.7" - env: - TOXENV: "msgpack" - - python-version: "3.7" - env: - TOXENV: "json" - python-version: "3.8" env: TOXENV: "msgpack" @@ -62,6 +38,12 @@ jobs: - python-version: "3.10" env: TOXENV: "json" + - python-version: "3.11" + env: + TOXENV: "msgpack" + - python-version: "3.11" + env: + TOXENV: "json" steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 35a80c4e..219ffe86 100644 --- a/setup.py +++ b/setup.py @@ -28,19 +28,16 @@ package_data={'scrapinghub': ['VERSION']}, install_requires=['requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'], extras_require={'msgpack': mpack_required}, - python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', + python_requires='>=3.8', classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index f6d21d45..9098f7ff 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py{27,35,36,py3,37,38,39}-{json,msgpack} +envlist = py{py3,38,39,310,311}-{json,msgpack} [testenv] deps = From 7343eb755493c8467a065b715ce2609f569a502e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 13:20:15 +0100 Subject: [PATCH 2/7] Upgrade PyPy in the CI --- .github/workflows/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6091126d..ada735f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,13 +13,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "pypy3" - env: - TOXENV: "msgpack" - - python-version: "pypy3" - env: - TOXENV: "json" - - python-version: "3.8" env: TOXENV: "msgpack" @@ -45,10 +38,17 @@ jobs: env: TOXENV: "json" + - python-version: "pypy3.10-v7.3.13" + env: + TOXENV: "msgpack" + - python-version: "pypy3.10-v7.3.13" + env: + TOXENV: "json" + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 46419cf70e605d0dcfe28d396a8efa11947bd93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 13:20:44 +0100 Subject: [PATCH 3/7] Fix mock issues --- tests/client/test_proxy.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/client/test_proxy.py b/tests/client/test_proxy.py index d9894a0f..dfde07f3 100644 --- a/tests/client/test_proxy.py +++ b/tests/client/test_proxy.py @@ -41,14 +41,22 @@ def test_format_iter_filters(): def test_item_resource_iter_no_params(): - items_proxy = _ItemsResourceProxy(mock.Mock, mock.Mock(), 'mocked_key') + class MockClient: + def __init__(self): + self._hsclient = object() + + items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') items_proxy._origin = mock.Mock() items_proxy.iter(count=123) assert items_proxy._origin.list.call_args == mock.call(None, count=123) def test_item_resource_iter_with_params(): - items_proxy = _ItemsResourceProxy(mock.Mock, mock.Mock(), 'mocked_key') + class MockClient: + def __init__(self): + self._hsclient = object() + + items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') items_proxy._origin = mock.Mock() items_proxy.iter(count=123, startts=12345) assert (items_proxy._origin.list.call_args == From 2fab84c12fc525bc184ae111cdb4b49847f3a260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 14:32:11 +0100 Subject: [PATCH 4/7] Upgrade cassettes when using Python 3.10+ --- tests/conftest.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9eb79d83..0eb41aff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- import base64 import os import pickle import pytest import re +import sys import zlib from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE @@ -31,6 +31,17 @@ TEST_DASH_ENDPOINT = os.getenv('DASH_ENDPOINT', DEFAULT_DASH_ENDPOINT) +# https://github.com/kevin1024/vcrpy/issues/719#issuecomment-1811544263 +def upgrade_cassette(cassette): + for interaction in cassette['interactions']: + response = interaction.get('response', {}) + headers = response.get('headers', {}) + contentType = headers.get('content-encoding') or headers.get('Content-Encoding') + compressed_string = response['body']['string'] + if contentType and contentType[0] == 'gzip': + response['body']['string'] = zlib.decompress(compressed_string, zlib.MAX_WBITS | 16).decode('utf-8') + + class VCRGzipSerializer(object): """Custom ZIP serializer for VCR.py.""" @@ -45,7 +56,10 @@ def serialize(self, cassette_dict): def deserialize(self, cassette_string): # receives a string, must return a dict decoded = base64.b64decode(cassette_string.encode('utf8')) - return pickle.loads(zlib.decompress(decoded)) + cassette = pickle.loads(zlib.decompress(decoded)) + if sys.version_info >= (3, 10): + upgrade_cassette(cassette) + return cassette def normalize_endpoint(uri, endpoint, default_endpoint): From d7b3601390c15e55a29e67773d5f38a374648af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 14:41:14 +0100 Subject: [PATCH 5/7] Do not UTF-8-decode potentially-binary responses --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0eb41aff..59ad94f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,8 @@ def upgrade_cassette(cassette): contentType = headers.get('content-encoding') or headers.get('Content-Encoding') compressed_string = response['body']['string'] if contentType and contentType[0] == 'gzip': - response['body']['string'] = zlib.decompress(compressed_string, zlib.MAX_WBITS | 16).decode('utf-8') + response['body']['string'] = zlib.decompress(compressed_string, zlib.MAX_WBITS | 16) + class VCRGzipSerializer(object): From 88e57703f454222f7b32ad0894c71e52cf8ba64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 14:43:19 +0100 Subject: [PATCH 6/7] Support Python 3.12 officially --- .github/workflows/main.yml | 6 ++++++ setup.py | 1 + 2 files changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ada735f8..956ed484 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,6 +37,12 @@ jobs: - python-version: "3.11" env: TOXENV: "json" + - python-version: "3.12" + env: + TOXENV: "msgpack" + - python-version: "3.12" + env: + TOXENV: "json" - python-version: "pypy3.10-v7.3.13" env: diff --git a/setup.py b/setup.py index 219ffe86..ae5ce0e8 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 7c1bbe0d6a6a733f67434c14aa4e8851ff84dfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 Dec 2023 14:58:23 +0100 Subject: [PATCH 7/7] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9098f7ff..f8a512ab 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py{py3,38,39,310,311}-{json,msgpack} +envlist = py{py3,38,39,310,311,312}-{json,msgpack} [testenv] deps =