From b0a3349ddae921a046ef6763b86f4e72063913b3 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 3 Nov 2025 06:37:57 +0000 Subject: [PATCH 1/6] test --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index a7db13a25..761c5f260 100644 --- a/README.rst +++ b/README.rst @@ -276,3 +276,4 @@ Next Steps .. _Google Cloud Storage Product documentation: https://cloud.google.com/storage .. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst +# test From 778e68ee4e565ec270f1972bda37700de228ccbf Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 3 Nov 2025 06:44:00 +0000 Subject: [PATCH 2/6] Revert "test" This reverts commit b0a3349ddae921a046ef6763b86f4e72063913b3. --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 761c5f260..a7db13a25 100644 --- a/README.rst +++ b/README.rst @@ -276,4 +276,3 @@ Next Steps .. _Google Cloud Storage Product documentation: https://cloud.google.com/storage .. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst -# test From eb1bc74a4435e1824d84e7ba3ff510a19f798cb6 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 8 Dec 2025 18:31:28 +0000 Subject: [PATCH 3/6] feat: send entire object checksum in the final api call of resumable upload --- google/cloud/storage/_media/_upload.py | 7 +++++ tests/unit/test_blob.py | 37 +++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/_media/_upload.py b/google/cloud/storage/_media/_upload.py index 765716882..4a919d18a 100644 --- a/google/cloud/storage/_media/_upload.py +++ b/google/cloud/storage/_media/_upload.py @@ -688,6 +688,13 @@ def _prepare_request(self): _CONTENT_TYPE_HEADER: self._content_type, _helpers.CONTENT_RANGE_HEADER: content_range, } + if (start_byte + len(payload) == self._total_bytes) and ( + self._checksum_object is not None + ): + local_checksum = _helpers.prepare_checksum_digest( + self._checksum_object.digest() + ) + headers["x-goog-hash"] = f"{self._checksum_type}={local_checksum}" return _PUT, self.resumable_url, payload, headers def _update_checksum(self, start_byte, payload): diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index f3b6da5d1..cdf096fb4 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -3049,7 +3049,14 @@ def test__initiate_resumable_upload_with_client_custom_headers(self): self._initiate_resumable_helper(client=client) def _make_resumable_transport( - self, headers1, headers2, headers3, total_bytes, data_corruption=False + self, + headers1, + headers2, + headers3, + total_bytes, + data_corruption=False, + md5_checksum_value=None, + crc32c_checksum_value=None, ): fake_transport = mock.Mock(spec=["request"]) @@ -3057,7 +3064,7 @@ def _make_resumable_transport( fake_response2 = self._mock_requests_response( http.client.PERMANENT_REDIRECT, headers2 ) - json_body = f'{{"size": "{total_bytes:d}"}}' + json_body = f'{{"size": "{total_bytes:d}", "md5Hash": "{md5_checksum_value}", "crc32c": "{crc32c_checksum_value}"}}' if data_corruption: fake_response3 = DataCorruption(None) else: @@ -3151,6 +3158,9 @@ def _do_resumable_upload_call2( if_metageneration_match=None, if_metageneration_not_match=None, timeout=None, + checksum=None, + crc32c_checksum_value=None, + md5_checksum_value=None, ): # Third mock transport.request() does sends last chunk. content_range = f"bytes {blob.chunk_size:d}-{total_bytes - 1:d}/{total_bytes:d}" @@ -3161,6 +3171,11 @@ def _do_resumable_upload_call2( "content-type": content_type, "content-range": content_range, } + if checksum == "crc32c": + expected_headers["x-goog-hash"] = f"crc32c={crc32c_checksum_value}" + elif checksum == "md5": + expected_headers["x-goog-hash"] = f"md5={md5_checksum_value}" + payload = data[blob.chunk_size :] return mock.call( "PUT", @@ -3181,12 +3196,17 @@ def _do_resumable_helper( timeout=None, data_corruption=False, retry=None, + checksum=None, # None is also a valid value, when user decides to disable checksum validation. ): CHUNK_SIZE = 256 * 1024 USER_AGENT = "testing 1.2.3" content_type = "text/html" # Data to be uploaded. data = b"" + (b"A" * CHUNK_SIZE) + b"" + + # Data calcuated offline and entered here. (Unit test best practice). + crc32c_checksum_value = "mQ30hg==" + md5_checksum_value = "wajHeg1f2Q2u9afI6fjPOw==" total_bytes = len(data) if use_size: size = total_bytes @@ -3213,6 +3233,8 @@ def _do_resumable_helper( headers3, total_bytes, data_corruption=data_corruption, + md5_checksum_value=md5_checksum_value, + crc32c_checksum_value=crc32c_checksum_value, ) # Create some mock arguments and call the method under test. @@ -3247,7 +3269,7 @@ def _do_resumable_helper( if_generation_not_match, if_metageneration_match, if_metageneration_not_match, - checksum=None, + checksum=checksum, retry=retry, **timeout_kwarg, ) @@ -3296,6 +3318,9 @@ def _do_resumable_helper( if_metageneration_match=if_metageneration_match, if_metageneration_not_match=if_metageneration_not_match, timeout=expected_timeout, + checksum=checksum, + crc32c_checksum_value=crc32c_checksum_value, + md5_checksum_value=md5_checksum_value, ) self.assertEqual(transport.request.mock_calls, [call0, call1, call2]) @@ -3308,6 +3333,12 @@ def test__do_resumable_upload_no_size(self): def test__do_resumable_upload_with_size(self): self._do_resumable_helper(use_size=True) + def test__do_resumable_upload_with_size_with_crc32c_checksum(self): + self._do_resumable_helper(use_size=True, checksum="crc32c") + + def test__do_resumable_upload_with_size_with_md5_checksum(self): + self._do_resumable_helper(use_size=True, checksum="md5") + def test__do_resumable_upload_with_retry(self): self._do_resumable_helper(retry=DEFAULT_RETRY) From 48c7b6dcace2e19d2538812e09758f1beae2b7e9 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 8 Dec 2025 19:00:18 +0000 Subject: [PATCH 4/6] remove test which sends checksum at begining --- .../system/requests/test_upload.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tests/resumable_media/system/requests/test_upload.py b/tests/resumable_media/system/requests/test_upload.py index dd90aa53b..38a703d12 100644 --- a/tests/resumable_media/system/requests/test_upload.py +++ b/tests/resumable_media/system/requests/test_upload.py @@ -372,29 +372,6 @@ def test_resumable_upload_with_headers( _resumable_upload_helper(authorized_transport, img_stream, cleanup, headers=headers) -@pytest.mark.parametrize("checksum", ["md5", "crc32c"]) -def test_resumable_upload_with_bad_checksum( - authorized_transport, img_stream, bucket, cleanup, checksum -): - fake_checksum_object = _helpers._get_checksum_object(checksum) - fake_checksum_object.update(b"bad data") - fake_prepared_checksum_digest = _helpers.prepare_checksum_digest( - fake_checksum_object.digest() - ) - with mock.patch.object( - _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest - ): - with pytest.raises(DataCorruption) as exc_info: - _resumable_upload_helper( - authorized_transport, img_stream, cleanup, checksum=checksum - ) - expected_checksums = {"md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA=="} - expected_message = _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( - checksum.upper(), fake_prepared_checksum_digest, expected_checksums[checksum] - ) - assert exc_info.value.args[0] == expected_message - - def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream): blob_name = os.path.basename(img_stream.name) # Create the actual upload object. From 37a62a0850ee87c2494a45a5a19e3632ec5fef2e Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 8 Dec 2025 19:21:27 +0000 Subject: [PATCH 5/6] remove unused imports --- tests/resumable_media/system/requests/test_upload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/resumable_media/system/requests/test_upload.py b/tests/resumable_media/system/requests/test_upload.py index 38a703d12..47f4f6003 100644 --- a/tests/resumable_media/system/requests/test_upload.py +++ b/tests/resumable_media/system/requests/test_upload.py @@ -27,7 +27,6 @@ import google.cloud.storage._media.requests as resumable_requests from google.cloud.storage._media import _helpers from .. import utils -from google.cloud.storage._media import _upload from google.cloud.storage.exceptions import InvalidResponse from google.cloud.storage.exceptions import DataCorruption From 7f220404ef71a61b7dd4784258e181c849522620 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Tue, 9 Dec 2025 01:14:59 +0530 Subject: [PATCH 6/6] Update tests/unit/test_blob.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- tests/unit/test_blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index cdf096fb4..cbf53b398 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -3064,7 +3064,7 @@ def _make_resumable_transport( fake_response2 = self._mock_requests_response( http.client.PERMANENT_REDIRECT, headers2 ) - json_body = f'{{"size": "{total_bytes:d}", "md5Hash": "{md5_checksum_value}", "crc32c": "{crc32c_checksum_value}"}}' + json_body = json.dumps({"size": str(total_bytes), "md5Hash": md5_checksum_value, "crc32c": crc32c_checksum_value}) if data_corruption: fake_response3 = DataCorruption(None) else: