diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml index 252c6ddb..bdef0e34 100644 --- a/.github/workflows/update-plugin-list.yml +++ b/.github/workflows/update-plugin-list.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install packaging requests tabulate[widechars] tqdm + pip install packaging httpx tabulate[widechars] tqdm - name: Update Plugin List run: python scripts/update_plugin_list.py diff --git a/docs/source/changes.md b/docs/source/changes.md index 5416f1f0..1f4c3470 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -40,6 +40,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and the value whose state is `None`. - {pull}`594` publishes `NodeLoadError`. - {pull}`596` add project management with rye. +- {pull}`598` replaces requests with httpx. ## 0.4.7 - 2024-03-19 diff --git a/docs/source/developers_guide.md b/docs/source/developers_guide.md index 1a987b0c..fbd06b77 100644 --- a/docs/source/developers_guide.md +++ b/docs/source/developers_guide.md @@ -9,7 +9,7 @@ number of open file descriptors on Unix systems, which causes some tests and the the test suite to fail. If that happens, increase the limit with the following command. ```console -$ ulimit -n 4096 +ulimit -n 4096 ``` ## How to release diff --git a/docs/source/how_to_guides/remote_files.md b/docs/source/how_to_guides/remote_files.md index 6125af9c..f43bd84c 100644 --- a/docs/source/how_to_guides/remote_files.md +++ b/docs/source/how_to_guides/remote_files.md @@ -73,9 +73,9 @@ file as the ETag. If the file changes, so does the ETag, and pytask will detect Many files on the web do not provide an ETag like this version of the iris dataset. ```pycon ->>> import requests +>>> import httpx >>> url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" ->>> r = requests.head(url) +>>> r = httpx.head(url) >>> r.headers {'Server': 'nginx/1.25.3', 'Date': 'Sun, 10 Dec 2023 23:59:21 GMT', 'Connection': 'keep-alive'} ``` diff --git a/pyproject.toml b/pyproject.toml index 5fcd58d4..7af629bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,9 +68,7 @@ test = [ "pytest-cov", "pytest-xdist", "syrupy", - # For HTTPPath tests. "aiohttp", - "requests", ] [project.urls] diff --git a/requirements-dev.lock b/requirements-dev.lock index 6b0a2800..df7f9104 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -8,10 +8,6 @@ # with-sources: false -e file:. -aiohttp==3.9.4 - # via pytask -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.16 # via sphinx apeye==1.4.1 @@ -21,7 +17,6 @@ apeye-core==1.1.5 asttokens==2.4.1 # via stack-data attrs==23.2.0 - # via aiohttp # via jsonschema # via pytask # via referencing @@ -91,9 +86,6 @@ filelock==3.13.4 # via sphinx-toolbox fonttools==4.51.0 # via matplotlib -frozenlist==1.4.1 - # via aiohttp - # via aiosignal fsspec==2024.3.1 # via universal-pathlib furo==2024.1.29 @@ -105,7 +97,6 @@ html5lib==1.1 idna==3.7 # via apeye-core # via requests - # via yarl imagesize==1.4.1 # via sphinx iniconfig==2.0.0 @@ -160,9 +151,6 @@ mistune==3.0.2 # via nbconvert msgpack==1.0.8 # via cachecontrol -multidict==6.0.5 - # via aiohttp - # via yarl myst-parser==2.0.0 # via pytask natsort==8.4.0 @@ -258,7 +246,6 @@ referencing==0.34.0 requests==2.31.0 # via apeye # via cachecontrol - # via pytask # via sphinx rich==13.7.1 # via pytask @@ -364,5 +351,3 @@ webencodings==0.5.1 # via bleach # via html5lib # via tinycss2 -yarl==1.9.4 - # via aiohttp diff --git a/requirements.lock b/requirements.lock index 6b0a2800..df7f9104 100644 --- a/requirements.lock +++ b/requirements.lock @@ -8,10 +8,6 @@ # with-sources: false -e file:. -aiohttp==3.9.4 - # via pytask -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.16 # via sphinx apeye==1.4.1 @@ -21,7 +17,6 @@ apeye-core==1.1.5 asttokens==2.4.1 # via stack-data attrs==23.2.0 - # via aiohttp # via jsonschema # via pytask # via referencing @@ -91,9 +86,6 @@ filelock==3.13.4 # via sphinx-toolbox fonttools==4.51.0 # via matplotlib -frozenlist==1.4.1 - # via aiohttp - # via aiosignal fsspec==2024.3.1 # via universal-pathlib furo==2024.1.29 @@ -105,7 +97,6 @@ html5lib==1.1 idna==3.7 # via apeye-core # via requests - # via yarl imagesize==1.4.1 # via sphinx iniconfig==2.0.0 @@ -160,9 +151,6 @@ mistune==3.0.2 # via nbconvert msgpack==1.0.8 # via cachecontrol -multidict==6.0.5 - # via aiohttp - # via yarl myst-parser==2.0.0 # via pytask natsort==8.4.0 @@ -258,7 +246,6 @@ referencing==0.34.0 requests==2.31.0 # via apeye # via cachecontrol - # via pytask # via sphinx rich==13.7.1 # via pytask @@ -364,5 +351,3 @@ webencodings==0.5.1 # via bleach # via html5lib # via tinycss2 -yarl==1.9.4 - # via aiohttp diff --git a/scripts/update_plugin_list.py b/scripts/update_plugin_list.py index cab04e18..56050e25 100644 --- a/scripts/update_plugin_list.py +++ b/scripts/update_plugin_list.py @@ -37,8 +37,8 @@ from textwrap import indent from typing import Generator +import httpx import packaging.version -import requests import tabulate import wcwidth from tqdm import tqdm @@ -87,7 +87,7 @@ def _escape_rst(text: str) -> str: def _iter_plugins() -> Generator[dict[str, str], None, None]: # noqa: C901 """Iterate over all plugins and format entries.""" regex = r">([\d\w-]*)" - response = requests.get("https://pypi.org/simple", timeout=20) + response = httpx.get("https://pypi.org/simple", timeout=20) matches = [ match @@ -98,7 +98,7 @@ def _iter_plugins() -> Generator[dict[str, str], None, None]: # noqa: C901 for match in tqdm(matches, smoothing=0): name = match.groups()[0] - response = requests.get(f"https://pypi.org/pypi/{name}/json", timeout=20) + response = httpx.get(f"https://pypi.org/pypi/{name}/json", timeout=20) if response.status_code == 404: # noqa: PLR2004 # Some packages might return a 404. continue diff --git a/tests/test_execute.py b/tests/test_execute.py index e74b5dc2..d92ce186 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1141,3 +1141,24 @@ def create_file() -> Annotated[Path, path]: assert "3 Collected task" in result.output assert "1 Succeeded" in result.output assert "2 Skipped because unchanged" in result.output + + +@pytest.mark.end_to_end() +def test_download_file(runner, tmp_path): + source = """ + from pathlib import Path + from typing_extensions import Annotated + from upath import UPath + + url = UPath( + "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" + ) + + def task_download_file(path: UPath = url) -> Annotated[str, Path("data.csv")]: + return path.read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "1 Succeeded" in result.output