From 77c5f3caa5fda92de0850e229863623fa113dd3e Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Thu, 9 Jan 2025 01:37:09 +0000 Subject: [PATCH] fix: filter download_kwargs in BlobReader --- google/cloud/storage/fileio.py | 12 +++++++++--- tests/system/test_fileio.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index f1b5e2bdd..7ffd24d45 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -89,6 +89,7 @@ class BlobReader(io.BufferedIOBase): configuration changes for Retry objects such as delays and deadlines are respected. + :type download_kwargs: dict :param download_kwargs: Keyword arguments to pass to the underlying API calls. The following arguments are supported: @@ -98,9 +99,10 @@ class BlobReader(io.BufferedIOBase): - ``if_metageneration_match`` - ``if_metageneration_not_match`` - ``timeout`` + - ``raw_download`` - Note that download_kwargs are also applied to blob.reload(), if a reload - is needed during seek(). + Note that download_kwargs (excluding ``raw_download``) are also applied to blob.reload(), + if a reload is needed during seek(). """ def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs): @@ -175,7 +177,10 @@ def seek(self, pos, whence=0): self._checkClosed() # Raises ValueError if closed. if self._blob.size is None: - self._blob.reload(**self._download_kwargs) + reload_kwargs = { + k: v for k, v in self._download_kwargs.items() if k != "raw_download" + } + self._blob.reload(**reload_kwargs) initial_offset = self._pos + self._buffer.tell() @@ -272,6 +277,7 @@ class BlobWriter(io.BufferedIOBase): configuration changes for Retry objects such as delays and deadlines are respected. + :type upload_kwargs: dict :param upload_kwargs: Keyword arguments to pass to the underlying API calls. The following arguments are supported: diff --git a/tests/system/test_fileio.py b/tests/system/test_fileio.py index 58c0cd5b5..ba12d3bc2 100644 --- a/tests/system/test_fileio.py +++ b/tests/system/test_fileio.py @@ -116,3 +116,26 @@ def test_blobwriter_exit( blobs_to_delete.append(blob) # blob should have been uploaded assert blob.exists() + + +def test_blobreader_w_raw_download( + shared_bucket, + blobs_to_delete, + file_data, +): + blob = shared_bucket.blob("LargeFile") + info = file_data["big"] + with open(info["path"], "rb") as file_obj: + with blob.open("wb", chunk_size=256 * 1024, if_generation_match=0) as writer: + writer.write(file_obj.read()) + blobs_to_delete.append(blob) + + # Test BlobReader read and seek handles raw downloads. + with open(info["path"], "rb") as file_obj: + with blob.open("rb", chunk_size=256 * 1024, raw_download=True) as reader: + reader.seek(0) + file_obj.seek(0) + assert file_obj.read() == reader.read() + # End of file reached; further reads should be blank but not + # raise an error. + assert reader.read() == b""