diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 68dcad9032d7..c373c64726dc 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -52,7 +52,8 @@ class _BlobIterator(Iterator): def __init__(self, bucket, extra_params=None, connection=None): connection = _require_connection(connection) self.bucket = bucket - self.prefixes = () + self.prefixes = set() + self._current_prefixes = None super(_BlobIterator, self).__init__( connection=connection, path=bucket.path + '/o', extra_params=extra_params) @@ -63,7 +64,8 @@ def get_items_from_response(self, response): :type response: dict :param response: The JSON API response for a page of blobs. """ - self.prefixes = tuple(response.get('prefixes', ())) + self._current_prefixes = tuple(response.get('prefixes', ())) + self.prefixes.update(self._current_prefixes) for item in response.get('items', []): name = item.get('name') blob = Blob(name, bucket=self.bucket) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 49b306717706..5bfd7e57d7bb 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -37,7 +37,7 @@ def test_ctor_w_implicit_connection(self): self.assertEqual(iterator.path, '%s/o' % bucket.path) self.assertEqual(iterator.page_number, 0) self.assertEqual(iterator.next_page_token, None) - self.assertEqual(iterator.prefixes, ()) + self.assertEqual(iterator.prefixes, set()) def test_ctor_w_explicit_connection(self): connection = _Connection() @@ -48,33 +48,52 @@ def test_ctor_w_explicit_connection(self): self.assertEqual(iterator.path, '%s/o' % bucket.path) self.assertEqual(iterator.page_number, 0) self.assertEqual(iterator.next_page_token, None) - self.assertEqual(iterator.prefixes, ()) + self.assertEqual(iterator.prefixes, set()) def test_get_items_from_response_empty(self): - from gcloud.storage._testing import _monkey_defaults connection = _Connection() bucket = _Bucket() - with _monkey_defaults(connection=connection): - iterator = self._makeOne(bucket) - blobs = list(iterator.get_items_from_response({})) + iterator = self._makeOne(bucket, connection=connection) + blobs = list(iterator.get_items_from_response({})) self.assertEqual(blobs, []) - self.assertEqual(iterator.prefixes, ()) + self.assertEqual(iterator.prefixes, set()) def test_get_items_from_response_non_empty(self): from gcloud.storage.blob import Blob - from gcloud.storage._testing import _monkey_defaults BLOB_NAME = 'blob-name' response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']} connection = _Connection() bucket = _Bucket() - with _monkey_defaults(connection=connection): - iterator = self._makeOne(bucket) - blobs = list(iterator.get_items_from_response(response)) + iterator = self._makeOne(bucket, connection=connection) + blobs = list(iterator.get_items_from_response(response)) + self.assertEqual(len(blobs), 1) + blob = blobs[0] + self.assertTrue(isinstance(blob, Blob)) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(iterator.prefixes, set(['foo'])) + + def test_get_items_from_response_cumulative_prefixes(self): + from gcloud.storage.blob import Blob + BLOB_NAME = 'blob-name1' + response1 = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']} + response2 = { + 'items': [], + 'prefixes': ['foo', 'bar'], + } + connection = _Connection() + bucket = _Bucket() + iterator = self._makeOne(bucket, connection=connection) + # Parse first response. + blobs = list(iterator.get_items_from_response(response1)) self.assertEqual(len(blobs), 1) blob = blobs[0] self.assertTrue(isinstance(blob, Blob)) self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(iterator.prefixes, ('foo',)) + self.assertEqual(iterator.prefixes, set(['foo'])) + # Parse second response. + blobs = list(iterator.get_items_from_response(response2)) + self.assertEqual(len(blobs), 0) + self.assertEqual(iterator.prefixes, set(['foo', 'bar'])) class Test_Bucket(unittest2.TestCase): @@ -243,13 +262,11 @@ def test_get_blob_hit(self): self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) def test_list_blobs_defaults(self): - from gcloud.storage._testing import _monkey_defaults NAME = 'name' connection = _Connection({'items': []}) bucket = self._makeOne(NAME) - with _monkey_defaults(connection=connection): - iterator = bucket.list_blobs() - blobs = list(iterator) + iterator = bucket.list_blobs(connection=connection) + blobs = list(iterator) self.assertEqual(blobs, []) kw, = connection._requested self.assertEqual(kw['method'], 'GET') @@ -257,7 +274,6 @@ def test_list_blobs_defaults(self): self.assertEqual(kw['query_params'], {'projection': 'noAcl'}) def test_list_blobs_explicit(self): - from gcloud.storage._testing import _monkey_defaults NAME = 'name' MAX_RESULTS = 10 PAGE_TOKEN = 'ABCD' @@ -277,17 +293,17 @@ def test_list_blobs_explicit(self): } connection = _Connection({'items': []}) bucket = self._makeOne(NAME) - with _monkey_defaults(connection=connection): - iterator = bucket.list_blobs( - max_results=MAX_RESULTS, - page_token=PAGE_TOKEN, - prefix=PREFIX, - delimiter=DELIMITER, - versions=VERSIONS, - projection=PROJECTION, - fields=FIELDS, - ) - blobs = list(iterator) + iterator = bucket.list_blobs( + max_results=MAX_RESULTS, + page_token=PAGE_TOKEN, + prefix=PREFIX, + delimiter=DELIMITER, + versions=VERSIONS, + projection=PROJECTION, + fields=FIELDS, + connection=connection, + ) + blobs = list(iterator) self.assertEqual(blobs, []) kw, = connection._requested self.assertEqual(kw['method'], 'GET') diff --git a/regression/storage.py b/regression/storage.py index 15212f3e1d75..17d578a639ec 100644 --- a/regression/storage.py +++ b/regression/storage.py @@ -259,7 +259,7 @@ def test_root_level_w_delimiter(self): self.assertEqual([blob.name for blob in blobs], ['file01.txt']) self.assertEqual(iterator.page_number, 1) self.assertTrue(iterator.next_page_token is None) - self.assertEqual(iterator.prefixes, ('parent/',)) + self.assertEqual(iterator.prefixes, set(['parent/'])) def test_first_level(self): iterator = self.bucket.list_blobs(delimiter='/', prefix='parent/') @@ -268,7 +268,7 @@ def test_first_level(self): self.assertEqual([blob.name for blob in blobs], ['parent/file11.txt']) self.assertEqual(iterator.page_number, 1) self.assertTrue(iterator.next_page_token is None) - self.assertEqual(iterator.prefixes, ('parent/child/',)) + self.assertEqual(iterator.prefixes, set(['parent/child/'])) def test_second_level(self): iterator = self.bucket.list_blobs(delimiter='/', @@ -281,7 +281,7 @@ def test_second_level(self): self.assertEqual(iterator.page_number, 1) self.assertTrue(iterator.next_page_token is None) self.assertEqual(iterator.prefixes, - ('parent/child/grand/', 'parent/child/other/')) + set(['parent/child/grand/', 'parent/child/other/'])) def test_third_level(self): # Pseudo-hierarchy can be arbitrarily deep, subject to the limit @@ -296,7 +296,7 @@ def test_third_level(self): ['parent/child/grand/file31.txt']) self.assertEqual(iterator.page_number, 1) self.assertTrue(iterator.next_page_token is None) - self.assertEqual(iterator.prefixes, ()) + self.assertEqual(iterator.prefixes, set()) class TestStorageSignURLs(TestStorageFiles):