diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 8877c679aa90..e740cd4febc2 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -175,10 +175,14 @@ def exists(self, client=None): :returns: True if the bucket exists in Cloud Storage. """ client = self._require_client(client) + # We only need the status code (200 or not) so we seek to + # minimize the returned payload. + query_params = {'fields': 'name'} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + try: - # We only need the status code (200 or not) so we seek to - # minimize the returned payload. - query_params = {'fields': 'name'} # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. client._connection.api_request( @@ -204,6 +208,9 @@ def create(self, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. """ + if self.user_project is not None: + raise ValueError("Cannot create bucket with 'user_project' set.") + client = self._require_client(client) query_params = {'project': client.project} properties = {key: self._properties[key] for key in self._changes} @@ -264,10 +271,18 @@ def get_blob(self, blob_name, client=None): :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + blob = Blob(bucket=self, name=blob_name) try: response = client._connection.api_request( - method='GET', path=blob.path, _target_object=blob) + method='GET', + path=blob.path, + query_params=query_params, + _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -321,7 +336,7 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. """ - extra_params = {} + extra_params = {'projection': projection} if prefix is not None: extra_params['prefix'] = prefix @@ -332,11 +347,12 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, if versions is not None: extra_params['versions'] = versions - extra_params['projection'] = projection - if fields is not None: extra_params['fields'] = fields + if self.user_project is not None: + extra_params['userProject'] = self.user_project + client = self._require_client(client) path = self.path + '/o' iterator = HTTPIterator( @@ -376,6 +392,11 @@ def delete(self, force=False, client=None): contains more than 256 objects / blobs. """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + if force: blobs = list(self.list_blobs( max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, @@ -397,7 +418,10 @@ def delete(self, force=False, client=None): # request has no response value (whether in a standard request or # in a batch request). client._connection.api_request( - method='DELETE', path=self.path, _target_object=None) + method='DELETE', + path=self.path, + query_params=query_params, + _target_object=None) def delete_blob(self, blob_name, client=None): """Deletes a blob from the current bucket. @@ -429,12 +453,20 @@ def delete_blob(self, blob_name, client=None): """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + blob_path = Blob.path_helper(self.path, blob_name) # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). client._connection.api_request( - method='DELETE', path=blob_path, _target_object=None) + method='DELETE', + path=blob_path, + query_params=query_params, + _target_object=None) def delete_blobs(self, blobs, on_error=None, client=None): """Deletes a list of blobs from the current bucket. @@ -497,14 +529,26 @@ def copy_blob(self, blob, destination_bucket, new_name=None, :returns: The new Blob. """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + if new_name is None: new_name = blob.name + new_blob = Blob(bucket=destination_bucket, name=new_name) api_path = blob.path + '/copyTo' + new_blob.path copy_result = client._connection.api_request( - method='POST', path=api_path, _target_object=new_blob) + method='POST', + path=api_path, + query_params=query_params, + _target_object=new_blob, + ) + if not preserve_acl: new_blob.acl.save(acl={}, client=client) + new_blob._set_properties(copy_result) return new_blob @@ -912,9 +956,15 @@ def get_iam_policy(self, client=None): the ``getIamPolicy`` API request. """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + info = client._connection.api_request( method='GET', path='%s/iam' % (self.path,), + query_params=query_params, _target_object=None) return Policy.from_api_repr(info) @@ -937,11 +987,17 @@ def set_iam_policy(self, policy, client=None): the ``setIamPolicy`` API request. """ client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + resource = policy.to_api_repr() resource['resourceId'] = self.path info = client._connection.api_request( method='PUT', path='%s/iam' % (self.path,), + query_params=query_params, data=resource, _target_object=None) return Policy.from_api_repr(info) @@ -965,12 +1021,16 @@ def test_iam_permissions(self, permissions, client=None): request. """ client = self._require_client(client) - query = {'permissions': permissions} + query_params = {'permissions': permissions} + + if self.user_project is not None: + query_params['userProject'] = self.user_project + path = '%s/iam/testPermissions' % (self.path,) resp = client._connection.api_request( method='GET', path=path, - query_params=query) + query_params=query_params) return resp.get('permissions', []) def make_public(self, recursive=False, future=False, client=None): diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index d68cd4ca980a..b6231fa2192a 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -38,11 +38,16 @@ def _get_target_class(): from google.cloud.storage.bucket import Bucket return Bucket - def _make_one(self, client=None, name=None, properties=None): + def _make_one( + self, client=None, name=None, properties=None, user_project=None): if client is None: connection = _Connection() client = _Client(connection) - bucket = self._get_target_class()(client, name=name) + if user_project is None: + bucket = self._get_target_class()(client, name=name) + else: + bucket = self._get_target_class()( + client, name=name, user_project=user_project) bucket._properties = properties or {} return bucket @@ -63,8 +68,7 @@ def test_ctor_w_user_project(self): USER_PROJECT = 'user-project-123' connection = _Connection() client = _Client(connection) - klass = self._get_target_class() - bucket = klass(client, name=NAME, user_project=USER_PROJECT) + bucket = self._make_one(client, name=NAME, user_project=USER_PROJECT) self.assertEqual(bucket.name, NAME) self.assertEqual(bucket._properties, {}) self.assertEqual(bucket.user_project, USER_PROJECT) @@ -137,7 +141,9 @@ def api_request(cls, *args, **kwargs): expected_cw = [((), expected_called_kwargs)] self.assertEqual(_FakeConnection._called_with, expected_cw) - def test_exists_hit(self): + def test_exists_hit_w_user_project(self): + USER_PROJECT = 'user-project-123' + class _FakeConnection(object): _called_with = [] @@ -149,7 +155,7 @@ def api_request(cls, *args, **kwargs): return object() BUCKET_NAME = 'bucket-name' - bucket = self._make_one(name=BUCKET_NAME) + bucket = self._make_one(name=BUCKET_NAME, user_project=USER_PROJECT) client = _Client(_FakeConnection) self.assertTrue(bucket.exists(client=client)) expected_called_kwargs = { @@ -157,17 +163,29 @@ def api_request(cls, *args, **kwargs): 'path': bucket.path, 'query_params': { 'fields': 'name', + 'userProject': USER_PROJECT, }, '_target_object': None, } expected_cw = [((), expected_called_kwargs)] self.assertEqual(_FakeConnection._called_with, expected_cw) + def test_create_w_user_project(self): + PROJECT = 'PROJECT' + BUCKET_NAME = 'bucket-name' + USER_PROJECT = 'user-project-123' + connection = _Connection() + client = _Client(connection, project=PROJECT) + bucket = self._make_one(client, BUCKET_NAME, user_project=USER_PROJECT) + + with self.assertRaises(ValueError): + bucket.create() + def test_create_hit(self): + PROJECT = 'PROJECT' BUCKET_NAME = 'bucket-name' DATA = {'name': BUCKET_NAME} connection = _Connection(DATA) - PROJECT = 'PROJECT' client = _Client(connection, project=PROJECT) bucket = self._make_one(client=client, name=BUCKET_NAME) bucket.create() @@ -259,18 +277,20 @@ def test_get_blob_miss(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, NONESUCH)) - def test_get_blob_hit(self): + def test_get_blob_hit_w_user_project(self): NAME = 'name' BLOB_NAME = 'blob-name' + USER_PROJECT = 'user-project-123' connection = _Connection({'name': BLOB_NAME}) client = _Client(connection) - bucket = self._make_one(name=NAME) + bucket = self._make_one(name=NAME, user_project=USER_PROJECT) blob = bucket.get_blob(BLOB_NAME, client=client) self.assertIs(blob.bucket, bucket) self.assertEqual(blob.name, BLOB_NAME) kw, = connection._requested self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + self.assertEqual(kw['query_params'], {'userProject': USER_PROJECT}) def test_list_blobs_defaults(self): NAME = 'name' @@ -285,8 +305,9 @@ def test_list_blobs_defaults(self): self.assertEqual(kw['path'], '/b/%s/o' % NAME) self.assertEqual(kw['query_params'], {'projection': 'noAcl'}) - def test_list_blobs_w_all_arguments(self): + def test_list_blobs_w_all_arguments_and_user_project(self): NAME = 'name' + USER_PROJECT = 'user-project-123' MAX_RESULTS = 10 PAGE_TOKEN = 'ABCD' PREFIX = 'subfolder' @@ -302,10 +323,11 @@ def test_list_blobs_w_all_arguments(self): 'versions': VERSIONS, 'projection': PROJECTION, 'fields': FIELDS, + 'userProject': USER_PROJECT, } connection = _Connection({'items': []}) client = _Client(connection) - bucket = self._make_one(name=NAME) + bucket = self._make_one(name=NAME, user_project=USER_PROJECT) iterator = bucket.list_blobs( max_results=MAX_RESULTS, page_token=PAGE_TOKEN, @@ -347,23 +369,27 @@ def test_delete_miss(self): expected_cw = [{ 'method': 'DELETE', 'path': bucket.path, + 'query_params': {}, '_target_object': None, }] self.assertEqual(connection._deleted_buckets, expected_cw) - def test_delete_hit(self): + def test_delete_hit_with_user_project(self): NAME = 'name' + USER_PROJECT = 'user-project-123' GET_BLOBS_RESP = {'items': []} connection = _Connection(GET_BLOBS_RESP) connection._delete_bucket = True client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) result = bucket.delete(force=True) self.assertIsNone(result) expected_cw = [{ 'method': 'DELETE', 'path': bucket.path, '_target_object': None, + 'query_params': {'userProject': USER_PROJECT}, }] self.assertEqual(connection._deleted_buckets, expected_cw) @@ -388,6 +414,7 @@ def test_delete_force_delete_blobs(self): expected_cw = [{ 'method': 'DELETE', 'path': bucket.path, + 'query_params': {}, '_target_object': None, }] self.assertEqual(connection._deleted_buckets, expected_cw) @@ -406,6 +433,7 @@ def test_delete_force_miss_blobs(self): expected_cw = [{ 'method': 'DELETE', 'path': bucket.path, + 'query_params': {}, '_target_object': None, }] self.assertEqual(connection._deleted_buckets, expected_cw) @@ -442,18 +470,22 @@ def test_delete_blob_miss(self): kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, NONESUCH)) + self.assertEqual(kw['query_params'], {}) - def test_delete_blob_hit(self): + def test_delete_blob_hit_with_user_project(self): NAME = 'name' BLOB_NAME = 'blob-name' + USER_PROJECT = 'user-project-123' connection = _Connection({}) client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) result = bucket.delete_blob(BLOB_NAME) self.assertIsNone(result) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + self.assertEqual(kw['query_params'], {'userProject': USER_PROJECT}) def test_delete_blobs_empty(self): NAME = 'name' @@ -463,17 +495,20 @@ def test_delete_blobs_empty(self): bucket.delete_blobs([]) self.assertEqual(connection._requested, []) - def test_delete_blobs_hit(self): + def test_delete_blobs_hit_w_user_project(self): NAME = 'name' BLOB_NAME = 'blob-name' + USER_PROJECT = 'user-project-123' connection = _Connection({}) client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) bucket.delete_blobs([BLOB_NAME]) kw = connection._requested self.assertEqual(len(kw), 1) self.assertEqual(kw[0]['method'], 'DELETE') self.assertEqual(kw[0]['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + self.assertEqual(kw[0]['query_params'], {'userProject': USER_PROJECT}) def test_delete_blobs_miss_no_on_error(self): from google.cloud.exceptions import NotFound @@ -531,6 +566,7 @@ class _Blob(object): DEST, BLOB_NAME) self.assertEqual(kw['method'], 'POST') self.assertEqual(kw['path'], COPY_PATH) + self.assertEqual(kw['query_params'], {}) def test_copy_blobs_preserve_acl(self): from google.cloud.storage.acl import ObjectACL @@ -562,14 +598,17 @@ class _Blob(object): self.assertEqual(len(kw), 2) self.assertEqual(kw[0]['method'], 'POST') self.assertEqual(kw[0]['path'], COPY_PATH) + self.assertEqual(kw[0]['query_params'], {}) self.assertEqual(kw[1]['method'], 'PATCH') self.assertEqual(kw[1]['path'], NEW_BLOB_PATH) + self.assertEqual(kw[1]['query_params'], {'projection': 'full'}) - def test_copy_blobs_w_name(self): + def test_copy_blobs_w_name_and_user_project(self): SOURCE = 'source' DEST = 'dest' BLOB_NAME = 'blob-name' NEW_NAME = 'new_name' + USER_PROJECT = 'user-project-123' class _Blob(object): name = BLOB_NAME @@ -577,7 +616,8 @@ class _Blob(object): connection = _Connection({}) client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) + source = self._make_one( + client=client, name=SOURCE, user_project=USER_PROJECT) dest = self._make_one(client=client, name=DEST) blob = _Blob() new_blob = source.copy_blob(blob, dest, NEW_NAME) @@ -588,6 +628,7 @@ class _Blob(object): DEST, NEW_NAME) self.assertEqual(kw['method'], 'POST') self.assertEqual(kw['path'], COPY_PATH) + self.assertEqual(kw['query_params'], {'userProject': USER_PROJECT}) def test_rename_blob(self): BUCKET_NAME = 'BUCKET_NAME' @@ -979,6 +1020,40 @@ def test_get_iam_policy(self): self.assertEqual(len(kw), 1) self.assertEqual(kw[0]['method'], 'GET') self.assertEqual(kw[0]['path'], '%s/iam' % (PATH,)) + self.assertEqual(kw[0]['query_params'], {}) + + def test_get_iam_policy_w_user_project(self): + from google.cloud.iam import Policy + + NAME = 'name' + USER_PROJECT = 'user-project-123' + PATH = '/b/%s' % (NAME,) + ETAG = 'DEADBEEF' + VERSION = 17 + RETURNED = { + 'resourceId': PATH, + 'etag': ETAG, + 'version': VERSION, + 'bindings': [], + } + EXPECTED = {} + connection = _Connection(RETURNED) + client = _Client(connection, None) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) + + policy = bucket.get_iam_policy() + + self.assertIsInstance(policy, Policy) + self.assertEqual(policy.etag, RETURNED['etag']) + self.assertEqual(policy.version, RETURNED['version']) + self.assertEqual(dict(policy), EXPECTED) + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]['method'], 'GET') + self.assertEqual(kw[0]['path'], '%s/iam' % (PATH,)) + self.assertEqual(kw[0]['query_params'], {'userProject': USER_PROJECT}) def test_set_iam_policy(self): import operator @@ -1025,6 +1100,66 @@ def test_set_iam_policy(self): self.assertEqual(len(kw), 1) self.assertEqual(kw[0]['method'], 'PUT') self.assertEqual(kw[0]['path'], '%s/iam' % (PATH,)) + self.assertEqual(kw[0]['query_params'], {}) + sent = kw[0]['data'] + self.assertEqual(sent['resourceId'], PATH) + self.assertEqual(len(sent['bindings']), len(BINDINGS)) + key = operator.itemgetter('role') + for found, expected in zip( + sorted(sent['bindings'], key=key), + sorted(BINDINGS, key=key)): + self.assertEqual(found['role'], expected['role']) + self.assertEqual( + sorted(found['members']), sorted(expected['members'])) + + def test_set_iam_policy_w_user_project(self): + import operator + from google.cloud.storage.iam import STORAGE_OWNER_ROLE + from google.cloud.storage.iam import STORAGE_EDITOR_ROLE + from google.cloud.storage.iam import STORAGE_VIEWER_ROLE + from google.cloud.iam import Policy + + NAME = 'name' + USER_PROJECT = 'user-project-123' + PATH = '/b/%s' % (NAME,) + ETAG = 'DEADBEEF' + VERSION = 17 + OWNER1 = 'user:phred@example.com' + OWNER2 = 'group:cloud-logs@google.com' + EDITOR1 = 'domain:google.com' + EDITOR2 = 'user:phred@example.com' + VIEWER1 = 'serviceAccount:1234-abcdef@service.example.com' + VIEWER2 = 'user:phred@example.com' + BINDINGS = [ + {'role': STORAGE_OWNER_ROLE, 'members': [OWNER1, OWNER2]}, + {'role': STORAGE_EDITOR_ROLE, 'members': [EDITOR1, EDITOR2]}, + {'role': STORAGE_VIEWER_ROLE, 'members': [VIEWER1, VIEWER2]}, + ] + RETURNED = { + 'etag': ETAG, + 'version': VERSION, + 'bindings': BINDINGS, + } + policy = Policy() + for binding in BINDINGS: + policy[binding['role']] = binding['members'] + + connection = _Connection(RETURNED) + client = _Client(connection, None) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) + + returned = bucket.set_iam_policy(policy) + + self.assertEqual(returned.etag, ETAG) + self.assertEqual(returned.version, VERSION) + self.assertEqual(dict(returned), dict(policy)) + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]['method'], 'PUT') + self.assertEqual(kw[0]['path'], '%s/iam' % (PATH,)) + self.assertEqual(kw[0]['query_params'], {'userProject': USER_PROJECT}) sent = kw[0]['data'] self.assertEqual(sent['resourceId'], PATH) self.assertEqual(len(sent['bindings']), len(BINDINGS)) @@ -1064,6 +1199,38 @@ def test_test_iam_permissions(self): self.assertEqual(kw[0]['path'], '%s/iam/testPermissions' % (PATH,)) self.assertEqual(kw[0]['query_params'], {'permissions': PERMISSIONS}) + def test_test_iam_permissions_w_user_project(self): + from google.cloud.storage.iam import STORAGE_OBJECTS_LIST + from google.cloud.storage.iam import STORAGE_BUCKETS_GET + from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE + + NAME = 'name' + USER_PROJECT = 'user-project-123' + PATH = '/b/%s' % (NAME,) + PERMISSIONS = [ + STORAGE_OBJECTS_LIST, + STORAGE_BUCKETS_GET, + STORAGE_BUCKETS_UPDATE, + ] + ALLOWED = PERMISSIONS[1:] + RETURNED = {'permissions': ALLOWED} + connection = _Connection(RETURNED) + client = _Client(connection, None) + bucket = self._make_one( + client=client, name=NAME, user_project=USER_PROJECT) + + allowed = bucket.test_iam_permissions(PERMISSIONS) + + self.assertEqual(allowed, ALLOWED) + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]['method'], 'GET') + self.assertEqual(kw[0]['path'], '%s/iam/testPermissions' % (PATH,)) + self.assertEqual( + kw[0]['query_params'], + {'permissions': PERMISSIONS, 'userProject': USER_PROJECT}) + def test_make_public_defaults(self): from google.cloud.storage.acl import _ACLEntity