Skip to content

[GCP-5538] Storage: Missing support for HTTP Headers and Query String Parameters #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
HemangChothani wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

[GCP-5538] Storage: Missing support for HTTP Headers and Query String Parameters #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
192b273
add support for http headers and query parameters
HemangChothani Jul 31, 2019
01b13a6
add query parameters in client file
HemangChothani Jul 31, 2019
633f68e
cosmetic changes
HemangChothani Aug 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 135 additions & 17 deletions storage/google/cloud/storage/blob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,13 +517,17 @@ def _get_transport(self, client):
client = self._require_client(client)
return client._http

def _get_download_url(self):
def _get_download_url(self, query_params=None):
"""Get the download URL for the current blob.

If the ``media_link`` has been loaded, it will be used, otherwise
the URL will be constructed from the current blob's path (and possibly
generation) to avoid a round trip.

:type query_params: dict or ``NoneType``
:param query_params: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:rtype: str
:returns: The download URL for the current blob.
"""
Expand All @@ -538,6 +542,10 @@ def _get_download_url(self):
if self.user_project is not None:
name_value_pairs.append(("userProject", self.user_project))

if query_params is not None:
query_params_list = list(query_params.items())
name_value_pairs.extend(query_params_list)

return _add_query_parameters(base_url, name_value_pairs)

def _do_download(
Expand Down Expand Up @@ -586,7 +594,9 @@ def _do_download(
while not download.finished:
download.consume_next_chunk(transport)

def download_to_file(self, file_obj, client=None, start=None, end=None):
def download_to_file(
self, file_obj, client=None, start=None, end=None, query_params=None
):
"""Download the contents of this blob into a file-like object.

.. note::
Expand Down Expand Up @@ -626,9 +636,13 @@ def download_to_file(self, file_obj, client=None, start=None, end=None):
:type end: int
:param end: Optional, The last byte in a range to be downloaded.

:type query_params: dict or ``NoneType``
:param query_params: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:raises: :class:`google.cloud.exceptions.NotFound`
"""
download_url = self._get_download_url()
download_url = self._get_download_url(query_params)
headers = _get_encryption_headers(self._encryption_key)
headers["accept-encoding"] = "gzip"

Expand All @@ -638,7 +652,9 @@ def download_to_file(self, file_obj, client=None, start=None, end=None):
except resumable_media.InvalidResponse as exc:
_raise_from_invalid_response(exc)

def download_to_filename(self, filename, client=None, start=None, end=None):
def download_to_filename(
self, filename, client=None, start=None, end=None, query_params=None
):
"""Download the contents of this blob into a named file.

If :attr:`user_project` is set on the bucket, bills the API request
Expand All @@ -658,11 +674,21 @@ def download_to_filename(self, filename, client=None, start=None, end=None):
:type end: int
:param end: Optional, The last byte in a range to be downloaded.

:type query_params: dict or ``NoneType``
:param query_params: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:raises: :class:`google.cloud.exceptions.NotFound`
"""
try:
with open(filename, "wb") as file_obj:
self.download_to_file(file_obj, client=client, start=start, end=end)
self.download_to_file(
file_obj,
client=client,
start=start,
end=end,
query_params=query_params,
Copy link

@sumit-ql sumit-ql Aug 4, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove the extra "comma" after query params.

Copy link
Collaborator Author

@HemangChothani HemangChothani Aug 5, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if i remove "comma" than blacken test failed, you can see also in client.py that added "comma" after client parameter

)
except resumable_media.DataCorruption:
# Delete the corrupt downloaded file.
os.remove(filename)
Expand All @@ -673,7 +699,7 @@ def download_to_filename(self, filename, client=None, start=None, end=None):
mtime = time.mktime(updated.timetuple())
os.utime(file_obj.name, (mtime, mtime))

def download_as_string(self, client=None, start=None, end=None):
def download_as_string(self, client=None, start=None, end=None, query_params=None):
"""Download the contents of this blob as a string.

If :attr:`user_project` is set on the bucket, bills the API request
Expand All @@ -690,12 +716,22 @@ def download_as_string(self, client=None, start=None, end=None):
:type end: int
:param end: Optional, The last byte in a range to be downloaded.

:type query_params: dict or ``NoneType``
:param query_params: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:rtype: bytes
:returns: The data stored in this blob.
:raises: :class:`google.cloud.exceptions.NotFound`
"""
string_buffer = BytesIO()
self.download_to_file(string_buffer, client=client, start=start, end=end)
self.download_to_file(
string_buffer,
client=client,
start=start,
end=end,
query_params=query_params,
Copy link

@sumit-ql sumit-ql Aug 4, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove extra comma.

Copy link
Collaborator Author

@HemangChothani HemangChothani Aug 5, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above

)
return string_buffer.getvalue()

def _get_content_type(self, content_type, filename=None):
Expand Down Expand Up @@ -784,7 +820,14 @@ def _get_upload_arguments(self, content_type):
return headers, object_metadata, content_type

def _do_multipart_upload(
self, client, stream, content_type, size, num_retries, predefined_acl
self,
client,
stream,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
Copy link

@sumit-ql sumit-ql Aug 4, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove "comma" after extra_headers.

Copy link
Collaborator Author

@HemangChothani HemangChothani Aug 5, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above

):
"""Perform a multipart upload.

Expand Down Expand Up @@ -817,6 +860,10 @@ def _do_multipart_upload(
:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:rtype: :class:`~requests.Response`
:returns: The "200 OK" response object returned after the multipart
upload request.
Expand All @@ -834,6 +881,8 @@ def _do_multipart_upload(
transport = self._get_transport(client)
info = self._get_upload_arguments(content_type)
headers, object_metadata, content_type = info
if extra_headers is not None:
headers.update(extra_headers)

base_url = _MULTIPART_URL_TEMPLATE.format(bucket_path=self.bucket.path)
name_value_pairs = []
Expand Down Expand Up @@ -963,7 +1012,14 @@ def _initiate_resumable_upload(
return upload, transport

def _do_resumable_upload(
self, client, stream, content_type, size, num_retries, predefined_acl
self,
client,
stream,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
):
"""Perform a resumable upload.

Expand Down Expand Up @@ -998,6 +1054,10 @@ def _do_resumable_upload(
:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:rtype: :class:`~requests.Response`
:returns: The "200 OK" response object returned after the final chunk
is uploaded.
Expand All @@ -1009,6 +1069,7 @@ def _do_resumable_upload(
size,
num_retries,
predefined_acl=predefined_acl,
extra_headers=extra_headers,
)

while not upload.finished:
Expand All @@ -1017,7 +1078,14 @@ def _do_resumable_upload(
return response

def _do_upload(
self, client, stream, content_type, size, num_retries, predefined_acl
self,
client,
stream,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
):
"""Determine an upload strategy and then perform the upload.

Expand Down Expand Up @@ -1054,18 +1122,34 @@ def _do_upload(
:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:rtype: dict
:returns: The parsed JSON from the "200 OK" response. This will be the
**only** response in the multipart case and it will be the
**final** response in the resumable case.
"""
if size is not None and size <= _MAX_MULTIPART_SIZE:
response = self._do_multipart_upload(
client, stream, content_type, size, num_retries, predefined_acl
client,
stream,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
)
else:
response = self._do_resumable_upload(
client, stream, content_type, size, num_retries, predefined_acl
client,
stream,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
)

return response.json()
Expand All @@ -1079,6 +1163,7 @@ def upload_from_file(
num_retries=None,
client=None,
predefined_acl=None,
extra_headers=None,
):
"""Upload the contents of this blob from a file-like object.

Expand Down Expand Up @@ -1140,6 +1225,10 @@ def upload_from_file(
:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:raises: :class:`~google.cloud.exceptions.GoogleCloudError`
if the upload response returns an error status.

Expand All @@ -1155,14 +1244,25 @@ def upload_from_file(

try:
created_json = self._do_upload(
client, file_obj, content_type, size, num_retries, predefined_acl
client,
file_obj,
content_type,
size,
num_retries,
predefined_acl,
extra_headers,
)
self._set_properties(created_json)
except resumable_media.InvalidResponse as exc:
_raise_from_invalid_response(exc)

def upload_from_filename(
self, filename, content_type=None, client=None, predefined_acl=None
self,
filename,
content_type=None,
client=None,
predefined_acl=None,
extra_headers=None,
):
"""Upload this blob's contents from the content of a named file.

Expand Down Expand Up @@ -1200,6 +1300,10 @@ def upload_from_filename(

:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters
"""
content_type = self._get_content_type(content_type, filename=filename)

Expand All @@ -1211,10 +1315,16 @@ def upload_from_filename(
client=client,
size=total_bytes,
predefined_acl=predefined_acl,
extra_headers=extra_headers,
)

def upload_from_string(
self, data, content_type="text/plain", client=None, predefined_acl=None
self,
data,
content_type="text/plain",
client=None,
predefined_acl=None,
extra_headers=None,
):
"""Upload contents of this blob from the provided string.

Expand Down Expand Up @@ -1247,6 +1357,10 @@ def upload_from_string(

:type predefined_acl: str
:param predefined_acl: (Optional) predefined access control list

:type extra_headers: dict or ``NoneType``
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters
"""
data = _to_bytes(data, encoding="utf-8")
string_buffer = BytesIO(data)
Expand All @@ -1256,10 +1370,11 @@ def upload_from_string(
content_type=content_type,
client=client,
predefined_acl=predefined_acl,
extra_headers=extra_headers,
)

def create_resumable_upload_session(
self, content_type=None, size=None, origin=None, client=None
self, content_type=None, size=None, origin=None, client=None, extra_headers={}
):
"""Create a resumable upload session.

Expand Down Expand Up @@ -1322,10 +1437,13 @@ def create_resumable_upload_session(
completed by making an HTTP PUT request with the
file's contents.

:type extra_headers: dict
:param extra_headers: (Optional) Extension (custom) query parameters.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters

:raises: :class:`google.cloud.exceptions.GoogleCloudError`
if the session creation response returns an error status.
"""
extra_headers = {}
if origin is not None:
# This header is specifically for client-side uploads, it
# determines the origins allowed for CORS.
Expand Down
Loading