feat(storage): Add cname support for V4 signature (#72)

* feat(storage): add cname support for v4 signature

* docs(storage): comment changes

* feat(storage): address comment

* feat(storage): doc fix

* feat(storage): nit addressed

* feat(storage): add conformance tests

* feat(storage): nit

Co-authored-by: Frank Natividad <[email protected]>

Author: HemangChothani

google/cloud/storage/_signing.py

@@ -553,7 +553,7 @@ def generate_signed_url_v4(
 
     header_names = [key.lower() for key in headers]
     if "host" not in header_names:
-        headers["Host"] = "storage.googleapis.com"
+        headers["Host"] = six.moves.urllib.parse.urlparse(api_access_endpoint).netloc
 
     if method.upper() == "RESUMABLE":
         method = "POST"

google/cloud/storage/blob.py

@@ -362,6 +362,8 @@ def generate_signed_url(
         service_account_email=None,
         access_token=None,
         virtual_hosted_style=False,
+        bucket_bound_hostname=None,
+        scheme="http",
     ):
         """Generates a signed URL for this blob.
 
@@ -380,6 +382,21 @@ def generate_signed_url(
         amount of time, you can use this method to generate a URL that
         is only valid within a certain time period.
 
+        If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,
+        ``https`` works only if using a ``CDN``.
+
+        Example:
+            Generates a signed URL for this blob using bucket_bound_hostname and scheme.
+
+            >>> from google.cloud import storage
+            >>> client = storage.Client()
+            >>> bucket = client.get_bucket('my-bucket-name')
+            >>> blob = client.get_blob('my-blob-name')
+            >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
+            >>>                                  version='v4')
+            >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
+            >>>                                  version='v4',scheme='https')  # If using ``CDN``
+
         This is particularly useful if you don't want publicly
         accessible blobs, but don't want to require users to explicitly
         log in.
@@ -460,6 +477,18 @@ def generate_signed_url(
             (Optional) If true, then construct the URL relative the bucket's
             virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
 
+        :type bucket_bound_hostname: str
+        :param bucket_bound_hostname:
+            (Optional) If pass, then construct the URL relative to the bucket-bound hostname.
+            Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
+            See: https://cloud.google.com/storage/docs/request-endpoints#cname
+
+        :type scheme: str
+        :param scheme:
+            (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
+            this value as the scheme.  ``https`` will work only when using a CDN.
+            Defaults to ``"http"``.
+
         :raises: :exc:`ValueError` when version is invalid.
         :raises: :exc:`TypeError` when expiration is not a valid type.
         :raises: :exc:`AttributeError` if credentials is not an instance
@@ -480,12 +509,21 @@ def generate_signed_url(
             api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format(
                 bucket_name=self.bucket.name
             )
-            resource = "/{quoted_name}".format(quoted_name=quoted_name)
+        elif bucket_bound_hostname:
+            if ":" in bucket_bound_hostname:
+                api_access_endpoint = bucket_bound_hostname
+            else:
+                api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+                    scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
+                )
         else:
             resource = "/{bucket_name}/{quoted_name}".format(
                 bucket_name=self.bucket.name, quoted_name=quoted_name
             )
 
+        if virtual_hosted_style or bucket_bound_hostname:
+            resource = "/{quoted_name}".format(quoted_name=quoted_name)
+
         if credentials is None:
             client = self._require_client(client)
             credentials = client._credentials

google/cloud/storage/bucket.py

@@ -2395,6 +2395,8 @@ def generate_signed_url(
         credentials=None,
         version=None,
         virtual_hosted_style=False,
+        bucket_bound_hostname=None,
+        scheme="http",
     ):
         """Generates a signed URL for this bucket.
 
@@ -2413,6 +2415,20 @@ def generate_signed_url(
         amount of time, you can use this method to generate a URL that
         is only valid within a certain time period.
 
+        If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,
+        ``https`` works only if using a ``CDN``.
+
+        Example:
+            Generates a signed URL for this bucket using bucket_bound_hostname and scheme.
+
+            >>> from google.cloud import storage
+            >>> client = storage.Client()
+            >>> bucket = client.get_bucket('my-bucket-name')
+            >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
+            >>>                                  version='v4')
+            >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
+            >>>                                  version='v4',scheme='https')  # If using ``CDN``
+
         This is particularly useful if you don't want publicly
         accessible buckets, but don't want to require users to explicitly
         log in.
@@ -2462,6 +2478,18 @@ def generate_signed_url(
             (Optional) If true, then construct the URL relative the bucket's
             virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
 
+        :type bucket_bound_hostname: str
+        :param bucket_bound_hostname:
+            (Optional) If pass, then construct the URL relative to the bucket-bound hostname.
+            Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
+            See: https://cloud.google.com/storage/docs/request-endpoints#cname
+
+        :type scheme: str
+        :param scheme:
+            (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
+            this value as the scheme.  ``https`` will work only when using a CDN.
+            Defaults to ``"http"``.
+
         :raises: :exc:`ValueError` when version is invalid.
         :raises: :exc:`TypeError` when expiration is not a valid type.
         :raises: :exc:`AttributeError` if credentials is not an instance
@@ -2480,10 +2508,19 @@ def generate_signed_url(
             api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format(
                 bucket_name=self.name
             )
-            resource = "/"
+        elif bucket_bound_hostname:
+            if ":" in bucket_bound_hostname:
+                api_access_endpoint = bucket_bound_hostname
+            else:
+                api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+                    scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
+                )
         else:
             resource = "/{bucket_name}".format(bucket_name=self.name)
 
+        if virtual_hosted_style or bucket_bound_hostname:
+            resource = "/"
+
         if credentials is None:
             client = self._require_client(client)
             credentials = client._credentials

tests/unit/test__signing.py

@@ -778,13 +778,15 @@ def dummy_service_account():
     return _DUMMY_SERVICE_ACCOUNT
 
 
-def _run_conformance_test(resource, test_data):
+def _run_conformance_test(
+    resource, test_data, api_access_endpoint="https://storage.googleapis.com"
+):
     credentials = dummy_service_account()
-
     url = Test_generate_signed_url_v4._call_fut(
         credentials,
         resource,
         expiration=test_data["expiration"],
+        api_access_endpoint=api_access_endpoint,
         method=test_data["method"],
         _request_timestamp=test_data["timestamp"],
         headers=test_data.get("headers"),
@@ -802,14 +804,31 @@ def test_conformance_client(test_data):
 
 @pytest.mark.parametrize("test_data", _BUCKET_TESTS)
 def test_conformance_bucket(test_data):
-    resource = "/{}".format(test_data["bucket"])
-    _run_conformance_test(resource, test_data)
+    if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME":
+        api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+            scheme=test_data["scheme"],
+            bucket_bound_hostname=test_data["bucketBoundHostname"],
+        )
+        resource = "/"
+        _run_conformance_test(resource, test_data, api_access_endpoint)
+    else:
+        resource = "/{}".format(test_data["bucket"])
+        _run_conformance_test(resource, test_data)
 
 
 @pytest.mark.parametrize("test_data", _BLOB_TESTS)
 def test_conformance_blob(test_data):
-    resource = "/{}/{}".format(test_data["bucket"], test_data["object"])
-    _run_conformance_test(resource, test_data)
+    if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME":
+        api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+            scheme=test_data["scheme"],
+            bucket_bound_hostname=test_data["bucketBoundHostname"],
+        )
+        resource = "/{}".format(test_data["object"])
+        _run_conformance_test(resource, test_data, api_access_endpoint)
+    else:
+
+        resource = "/{}/{}".format(test_data["bucket"], test_data["object"])
+        _run_conformance_test(resource, test_data)
 
 
 def _make_credentials(signer_email=None):

tests/unit/test_blob.py

@@ -400,6 +400,8 @@ def _generate_signed_url_helper(
         access_token=None,
         service_account_email=None,
         virtual_hosted_style=False,
+        bucket_bound_hostname=None,
+        scheme="http",
     ):
         from six.moves.urllib import parse
         from google.cloud._helpers import UTC
@@ -444,6 +446,7 @@ def _generate_signed_url_helper(
                 access_token=access_token,
                 service_account_email=service_account_email,
                 virtual_hosted_style=virtual_hosted_style,
+                bucket_bound_hostname=bucket_bound_hostname,
             )
 
         self.assertEqual(signed_uri, signer.return_value)
@@ -460,11 +463,20 @@ def _generate_signed_url_helper(
             expected_api_access_endpoint = "https://{}.storage.googleapis.com".format(
                 bucket.name
             )
-            expected_resource = "/{}".format(quoted_name)
+        elif bucket_bound_hostname:
+            if ":" in bucket_bound_hostname:
+                expected_api_access_endpoint = bucket_bound_hostname
+            else:
+                expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+                    scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
+                )
         else:
             expected_api_access_endpoint = api_access_endpoint
             expected_resource = "/{}/{}".format(bucket.name, quoted_name)
 
+        if virtual_hosted_style or bucket_bound_hostname:
+            expected_resource = "/{}".format(quoted_name)
+
         if encryption_key is not None:
             expected_headers = headers or {}
             if effective_version == "v2":
@@ -619,6 +631,14 @@ def test_generate_signed_url_v4_w_csek_and_headers(self):
     def test_generate_signed_url_v4_w_virtual_hostname(self):
         self._generate_signed_url_v4_helper(virtual_hosted_style=True)
 
+    def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self):
+        self._generate_signed_url_v4_helper(
+            bucket_bound_hostname="http://cdn.example.com"
+        )
+
+    def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self):
+        self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com")
+
     def test_generate_signed_url_v4_w_credentials(self):
         credentials = object()
         self._generate_signed_url_v4_helper(credentials=credentials)

tests/unit/test_bucket.py

@@ -2779,6 +2779,8 @@ def _generate_signed_url_helper(
         credentials=None,
         expiration=None,
         virtual_hosted_style=False,
+        bucket_bound_hostname=None,
+        scheme="http",
     ):
         from six.moves.urllib import parse
         from google.cloud._helpers import UTC
@@ -2814,6 +2816,7 @@ def _generate_signed_url_helper(
                 query_parameters=query_parameters,
                 version=version,
                 virtual_hosted_style=virtual_hosted_style,
+                bucket_bound_hostname=bucket_bound_hostname,
             )
 
         self.assertEqual(signed_uri, signer.return_value)
@@ -2827,11 +2830,20 @@ def _generate_signed_url_helper(
             expected_api_access_endpoint = "https://{}.storage.googleapis.com".format(
                 bucket_name
             )
-            expected_resource = "/"
+        elif bucket_bound_hostname:
+            if ":" in bucket_bound_hostname:
+                expected_api_access_endpoint = bucket_bound_hostname
+            else:
+                expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
+                    scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
+                )
         else:
             expected_api_access_endpoint = api_access_endpoint
             expected_resource = "/{}".format(parse.quote(bucket_name))
 
+        if virtual_hosted_style or bucket_bound_hostname:
+            expected_resource = "/"
+
         expected_kwargs = {
             "resource": expected_resource,
             "expiration": expiration,
@@ -2967,6 +2979,14 @@ def test_generate_signed_url_v4_w_credentials(self):
     def test_generate_signed_url_v4_w_virtual_hostname(self):
         self._generate_signed_url_v4_helper(virtual_hosted_style=True)
 
+    def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self):
+        self._generate_signed_url_v4_helper(
+            bucket_bound_hostname="http://cdn.example.com"
+        )
+
+    def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self):
+        self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com")
+
 
 class _Connection(object):
     _delete_bucket = False

tests/unit/url_signer_v4_test_data.json

@@ -118,5 +118,55 @@
         "expiration": 10,
         "timestamp": "20190201T090000Z",
         "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b"
+    },
+
+    {
+      "description": "HTTP Bucket Bound Hostname Support",
+      "bucket": "test-bucket",
+      "object": "test-object",
+      "method": "GET",
+      "expiration": 10,
+      "timestamp": "20190201T090000Z",
+      "expectedUrl": "http://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56",
+      "scheme": "http",
+      "urlStyle": "BUCKET_BOUND_HOSTNAME",
+      "bucketBoundHostname": "mydomain.tld"
+    },
+
+    {
+        "description": "HTTPS Bucket Bound Hostname Support",
+        "bucket": "test-bucket",
+        "object": "test-object",
+        "method": "GET",
+        "expiration": 10,
+        "timestamp": "20190201T090000Z",
+        "expectedUrl": "https://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56",
+        "scheme": "https",
+        "urlStyle": "BUCKET_BOUND_HOSTNAME",
+        "bucketBoundHostname": "mydomain.tld"
+    },
+
+    {
+        "description": "HTTP Bucket Bound Hostname Support",
+        "bucket": "test-bucket",
+        "method": "GET",
+        "expiration": 10,
+        "timestamp": "20190201T090000Z",
+        "expectedUrl": "http://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf",
+        "scheme": "http",
+        "urlStyle": "BUCKET_BOUND_HOSTNAME",
+        "bucketBoundHostname": "mydomain.tld"
+    },
+
+    {
+        "description": "HTTPS Bucket Bound Hostname Support",
+        "bucket": "test-bucket",
+        "method": "GET",
+        "expiration": 10,
+        "timestamp": "20190201T090000Z",
+        "expectedUrl": "https://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf",
+        "scheme": "https",
+        "urlStyle": "BUCKET_BOUND_HOSTNAME",
+        "bucketBoundHostname": "mydomain.tld"
     }
 ]