diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 41a2727dfa41..c7ff6f8e2240 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -283,32 +283,6 @@ should either be: ``EXTRA_TOX_ENVS``. This value is unencrypted in ``gcloud-python-wheels`` to make ongoing maintenance easier. -Shared Code with External Projects ----------------------------------- - -In order to enable high-quality HTTP transfer of large data (for Cloud -Storage), we have temporarily included some code from the -`apitools `__ library. - -We have chosen to partially include it, rather than include it as -a dependency because - -- The library is not yet included on PyPI. -- The library's ``protorpc`` dependency is not Python 3 friendly, so - would block us from Python 3 support if fully included. - -The included code in lives in the -`_gcloud_vendor `__ -directory. It is a snapshot of the ``e5a5c36e24926310712d20b93b4cdd02424a81f5`` -commit from the main project imported in -``4c27079cf6d7f9814b36cfd16f3402455f768094``. In addition to the raw import, -we have customized (e.g. rewriting imports) for our library: - -- ``334961054d875641d150eec4d6938f6f824ea655`` -- ``565750ee7d19742b520dd62e2a4ff38325987284`` -- ``67b06019549a4db8168ff4c5171c9d701ac94a15`` -- ``f4a53ee64fad5f3d7f29a0341e6a72a060edfcc2`` - Supported Python Versions ------------------------- diff --git a/_gcloud_vendor/__init__.py b/_gcloud_vendor/__init__.py deleted file mode 100644 index 9ee34b0c867b..000000000000 --- a/_gcloud_vendor/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Dependencies "vendored in", due to dependencies, Python versions, etc. - -Current set ------------ - -``apitools`` (pending release to PyPI, plus acceptable Python version - support for its dependencies). Review before M2. -""" diff --git a/_gcloud_vendor/apitools/__init__.py b/_gcloud_vendor/apitools/__init__.py deleted file mode 100644 index 9870b5e53b94..000000000000 --- a/_gcloud_vendor/apitools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/__init__.py b/_gcloud_vendor/apitools/base/__init__.py deleted file mode 100644 index 9870b5e53b94..000000000000 --- a/_gcloud_vendor/apitools/base/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/py/__init__.py b/_gcloud_vendor/apitools/base/py/__init__.py deleted file mode 100644 index 9870b5e53b94..000000000000 --- a/_gcloud_vendor/apitools/base/py/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/py/exceptions.py b/_gcloud_vendor/apitools/base/py/exceptions.py deleted file mode 100644 index 55faa4970ebb..000000000000 --- a/_gcloud_vendor/apitools/base/py/exceptions.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -"""Exceptions for generated client libraries.""" - - -class Error(Exception): - """Base class for all exceptions.""" - - -class TypecheckError(Error, TypeError): - """An object of an incorrect type is provided.""" - - -class NotFoundError(Error): - """A specified resource could not be found.""" - - -class UserError(Error): - """Base class for errors related to user input.""" - - -class InvalidDataError(Error): - """Base class for any invalid data error.""" - - -class CommunicationError(Error): - """Any communication error talking to an API server.""" - - -class HttpError(CommunicationError): - """Error making a request. Soon to be HttpError.""" - - def __init__(self, response, content, url): - super(HttpError, self).__init__() - self.response = response - self.content = content - self.url = url - - def __str__(self): - content = self.content.decode('ascii', 'replace') - return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( - self.url, self.response, content) - - @property - def status_code(self): - # TODO(craigcitro): Turn this into something better than a - # KeyError if there is no status. - return int(self.response['status']) - - @classmethod - def FromResponse(cls, http_response): - return cls(http_response.info, http_response.content, - http_response.request_url) - - -class InvalidUserInputError(InvalidDataError): - """User-provided input is invalid.""" - - -class InvalidDataFromServerError(InvalidDataError, CommunicationError): - """Data received from the server is malformed.""" - - -class BatchError(Error): - """Error generated while constructing a batch request.""" - - -class ConfigurationError(Error): - """Base class for configuration errors.""" - - -class GeneratedClientError(Error): - """The generated client configuration is invalid.""" - - -class ConfigurationValueError(UserError): - """Some part of the user-specified client configuration is invalid.""" - - -class ResourceUnavailableError(Error): - """User requested an unavailable resource.""" - - -class CredentialsError(Error): - """Errors related to invalid credentials.""" - - -class TransferError(CommunicationError): - """Errors related to transfers.""" - - -class TransferInvalidError(TransferError): - """The given transfer is invalid.""" - - -class NotYetImplementedError(GeneratedClientError): - """This functionality is not yet implemented.""" - - -class StreamExhausted(Error): - """Attempted to read more bytes from a stream than were available.""" diff --git a/_gcloud_vendor/apitools/base/py/http_wrapper.py b/_gcloud_vendor/apitools/base/py/http_wrapper.py deleted file mode 100644 index 8b8b6cfc08aa..000000000000 --- a/_gcloud_vendor/apitools/base/py/http_wrapper.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python -"""HTTP wrapper for apitools. - -This library wraps the underlying http library we use, which is -currently httplib2. -""" - -import collections -import logging -import socket -import time - -import httplib2 -from six.moves import http_client -from six.moves import range -from six.moves.urllib.parse import urlsplit - -from _gcloud_vendor.apitools.base.py import exceptions -from _gcloud_vendor.apitools.base.py import util - -__all__ = [ - 'GetHttp', - 'MakeRequest', - 'Request', -] - - -# 308 and 429 don't have names in httplib. -RESUME_INCOMPLETE = 308 -TOO_MANY_REQUESTS = 429 -_REDIRECT_STATUS_CODES = ( - http_client.MOVED_PERMANENTLY, - http_client.FOUND, - http_client.SEE_OTHER, - http_client.TEMPORARY_REDIRECT, - RESUME_INCOMPLETE, -) - - -class Request(object): - """Class encapsulating the data for an HTTP request.""" - - def __init__(self, url='', http_method='GET', headers=None, body=''): - self.url = url - self.http_method = http_method - self.headers = headers or {} - self.__body = None - self.body = body - - @property - def body(self): - return self.__body - - @body.setter - def body(self, value): - self.__body = value - if value is not None: - self.headers['content-length'] = str(len(self.__body)) - else: - self.headers.pop('content-length', None) - - -# Note: currently the order of fields here is important, since we want -# to be able to pass in the result from httplib2.request. -class Response(collections.namedtuple( - 'HttpResponse', ['info', 'content', 'request_url'])): - """Class encapsulating data for an HTTP response.""" - __slots__ = () - - def __len__(self): - def ProcessContentRange(content_range): - _, _, range_spec = content_range.partition(' ') - byte_range, _, _ = range_spec.partition('/') - start, _, end = byte_range.partition('-') - return int(end) - int(start) + 1 - - if '-content-encoding' in self.info and 'content-range' in self.info: - # httplib2 rewrites content-length in the case of a compressed - # transfer; we can't trust the content-length header in that - # case, but we *can* trust content-range, if it's present. - return ProcessContentRange(self.info['content-range']) - elif 'content-length' in self.info: - return int(self.info.get('content-length')) - elif 'content-range' in self.info: - return ProcessContentRange(self.info['content-range']) - return len(self.content) - - @property - def status_code(self): - return int(self.info['status']) - - @property - def retry_after(self): - if 'retry-after' in self.info: - return int(self.info['retry-after']) - - @property - def is_redirect(self): - return (self.status_code in _REDIRECT_STATUS_CODES and - 'location' in self.info) - - -def MakeRequest(http, http_request, retries=5, redirections=5): - """Send http_request via the given http. - - This wrapper exists to handle translation between the plain httplib2 - request/response types and the Request and Response types above. - This will also be the hook for error/retry handling. - - Args: - http: An httplib2.Http instance, or a http multiplexer that delegates to - an underlying http, for example, HTTPMultiplexer. - http_request: A Request to send. - retries: (int, default 5) Number of retries to attempt on 5XX replies. - redirections: (int, default 5) Number of redirects to follow. - - Returns: - A Response object. - - Raises: - InvalidDataFromServerError: if there is no response after retries. - """ - response = None - exc = None - connection_type = None - # Handle overrides for connection types. This is used if the caller - # wants control over the underlying connection for managing callbacks - # or hash digestion. - if getattr(http, 'connections', None): - url_scheme = urlsplit(http_request.url).scheme - if url_scheme and url_scheme in http.connections: - connection_type = http.connections[url_scheme] - for retry in range(retries + 1): - # Note that the str() calls here are important for working around - # some funny business with message construction and unicode in - # httplib itself. See, eg, - # http://bugs.python.org/issue11898 - info = None - try: - info, content = http.request( - str(http_request.url), method=str(http_request.http_method), - body=http_request.body, headers=http_request.headers, - redirections=redirections, connection_type=connection_type) - except http_client.BadStatusLine as e: - logging.error('Caught BadStatusLine from httplib, retrying: %s', e) - exc = e - except socket.error as e: - if http_request.http_method != 'GET': - raise - logging.error('Caught socket error, retrying: %s', e) - exc = e - except http_client.IncompleteRead as e: - if http_request.http_method != 'GET': - raise - logging.error('Caught IncompleteRead error, retrying: %s', e) - exc = e - if info is not None: - response = Response(info, content, http_request.url) - if (response.status_code < 500 and - response.status_code != TOO_MANY_REQUESTS and - not response.retry_after): - break - logging.info('Retrying request to url <%s> after status code %s.', - response.request_url, response.status_code) - elif isinstance(exc, http_client.IncompleteRead): - logging.info('Retrying request to url <%s> after incomplete read.', - str(http_request.url)) - else: - logging.info('Retrying request to url <%s> after connection break.', - str(http_request.url)) - # TODO(craigcitro): Make this timeout configurable. - if response: - time.sleep(response.retry_after or util.CalculateWaitForRetry(retry)) - else: - time.sleep(util.CalculateWaitForRetry(retry)) - if response is None: - raise exceptions.InvalidDataFromServerError( - 'HTTP error on final retry: %s' % exc) - return response - - -def GetHttp(): - return httplib2.Http() diff --git a/_gcloud_vendor/apitools/base/py/transfer.py b/_gcloud_vendor/apitools/base/py/transfer.py deleted file mode 100644 index c98d5798b5eb..000000000000 --- a/_gcloud_vendor/apitools/base/py/transfer.py +++ /dev/null @@ -1,717 +0,0 @@ -#!/usr/bin/env python -"""Upload and download support for apitools.""" -from __future__ import print_function - -import email.generator as email_generator -import email.mime.multipart as mime_multipart -import email.mime.nonmultipart as mime_nonmultipart -import io -import json -import mimetypes -import os -import threading - -from six.moves import http_client - -from _gcloud_vendor.apitools.base.py import exceptions -from _gcloud_vendor.apitools.base.py import http_wrapper -from _gcloud_vendor.apitools.base.py import util - -__all__ = [ - 'Download', - 'Upload', -] - -_RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 -_SIMPLE_UPLOAD = 'simple' -_RESUMABLE_UPLOAD = 'resumable' - - -class _Transfer(object): - """Generic bits common to Uploads and Downloads.""" - - def __init__(self, stream, close_stream=False, chunksize=None, - auto_transfer=True, http=None): - self.__bytes_http = None - self.__close_stream = close_stream - self.__http = http - self.__stream = stream - self.__url = None - - self.auto_transfer = auto_transfer - self.chunksize = chunksize or 1048576 - - def __repr__(self): - return str(self) - - @property - def close_stream(self): - return self.__close_stream - - @property - def http(self): - return self.__http - - @property - def bytes_http(self): - return self.__bytes_http or self.http - - @bytes_http.setter - def bytes_http(self, value): - self.__bytes_http = value - - @property - def stream(self): - return self.__stream - - @property - def url(self): - return self.__url - - def _Initialize(self, http, url): - """Initialize this download by setting self.http and self.url. - - We want the user to be able to override self.http by having set - the value in the constructor; in that case, we ignore the provided - http. - - Args: - http: An httplib2.Http instance or None. - url: The url for this transfer. - - Returns: - None. Initializes self. - """ - self.EnsureUninitialized() - if self.http is None: - self.__http = http or http_wrapper.GetHttp() - self.__url = url - - @property - def initialized(self): - return self.url is not None and self.http is not None - - @property - def _type_name(self): - return type(self).__name__ - - def EnsureInitialized(self): - if not self.initialized: - raise exceptions.TransferInvalidError( - 'Cannot use uninitialized %s', self._type_name) - - def EnsureUninitialized(self): - if self.initialized: - raise exceptions.TransferInvalidError( - 'Cannot re-initialize %s', self._type_name) - - def __del__(self): - if self.__close_stream: - self.__stream.close() - - def _ExecuteCallback(self, callback, response): - # TODO(craigcitro): Push these into a queue. - if callback is not None: - threading.Thread(target=callback, args=(response, self)).start() - - -class Download(_Transfer): - """Data for a single download. - - Public attributes: - chunksize: default chunksize to use for transfers. - """ - _ACCEPTABLE_STATUSES = set(( - http_client.OK, - http_client.NO_CONTENT, - http_client.PARTIAL_CONTENT, - http_client.REQUESTED_RANGE_NOT_SATISFIABLE, - )) - _REQUIRED_SERIALIZATION_KEYS = set(( - 'auto_transfer', 'progress', 'total_size', 'url')) - - def __init__(self, *args, **kwds): - super(Download, self).__init__(*args, **kwds) - self.__initial_response = None - self.__progress = 0 - self.__total_size = None - - @property - def progress(self): - return self.__progress - - @classmethod - def FromFile(cls, filename, overwrite=False, auto_transfer=True): - """Create a new download object from a filename.""" - path = os.path.expanduser(filename) - if os.path.exists(path) and not overwrite: - raise exceptions.InvalidUserInputError( - 'File %s exists and overwrite not specified' % path) - return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer) - - @classmethod - def FromStream(cls, stream, auto_transfer=True): - """Create a new Download object from a stream.""" - return cls(stream, auto_transfer=auto_transfer) - - @classmethod - def FromData(cls, stream, json_data, http=None, auto_transfer=None): - """Create a new Download object from a stream and serialized data.""" - info = json.loads(json_data) - missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) - if missing_keys: - raise exceptions.InvalidDataError( - 'Invalid serialization data, missing keys: %s' % ( - ', '.join(missing_keys))) - download = cls.FromStream(stream) - if auto_transfer is not None: - download.auto_transfer = auto_transfer - else: - download.auto_transfer = info['auto_transfer'] - setattr(download, '_Download__progress', info['progress']) - setattr(download, '_Download__total_size', info['total_size']) - download._Initialize(http, info['url']) # pylint: disable=protected-access - return download - - @property - def serialization_data(self): - self.EnsureInitialized() - return { - 'auto_transfer': self.auto_transfer, - 'progress': self.progress, - 'total_size': self.total_size, - 'url': self.url, - } - - @property - def total_size(self): - return self.__total_size - - def __str__(self): - if not self.initialized: - return 'Download (uninitialized)' - else: - return 'Download with %d/%s bytes transferred from url %s' % ( - self.progress, self.total_size, self.url) - - def ConfigureRequest(self, http_request, url_builder): - url_builder.query_params['alt'] = 'media' - http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) - - def __SetTotal(self, info): - if 'content-range' in info: - _, _, total = info['content-range'].rpartition('/') - if total != '*': - self.__total_size = int(total) - # Note "total_size is None" means we don't know it; if no size - # info was returned on our initial range request, that means we - # have a 0-byte file. (That last statement has been verified - # empirically, but is not clearly documented anywhere.) - if self.total_size is None: - self.__total_size = 0 - - def InitializeDownload(self, http_request, http=None, client=None): - """Initialize this download by making a request. - - Args: - http_request: The HttpRequest to use to initialize this download. - http: The httplib2.Http instance for this request. - client: If provided, let this client process the final URL before - sending any additional requests. If client is provided and - http is not, client.http will be used instead. - """ - self.EnsureUninitialized() - if http is None and client is None: - raise exceptions.UserError('Must provide client or http.') - http = http or client.http - if client is not None: - http_request.url = client.FinalizeTransferUrl(http_request.url) - response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) - if response.status_code not in self._ACCEPTABLE_STATUSES: - raise exceptions.HttpError.FromResponse(response) - self.__initial_response = response - self.__SetTotal(response.info) - url = response.info.get('content-location', response.request_url) - if client is not None: - url = client.FinalizeTransferUrl(url) - self._Initialize(http, url) - # Unless the user has requested otherwise, we want to just - # go ahead and pump the bytes now. - if self.auto_transfer: - self.StreamInChunks() - - @staticmethod - def _ArgPrinter(response, unused_download): - if 'content-range' in response.info: - print('Received %s' % response.info['content-range']) - else: - print('Received %d bytes' % len(response)) - - @staticmethod - def _CompletePrinter(*unused_args): - print('Download complete') - - def __NormalizeStartEnd(self, start, end=None): - if end is not None: - if start < 0: - raise exceptions.TransferInvalidError( - 'Cannot have end index with negative start index') - elif start >= self.total_size: - raise exceptions.TransferInvalidError( - 'Cannot have start index greater than total size') - end = min(end, self.total_size - 1) - if end < start: - raise exceptions.TransferInvalidError( - 'Range requested with end[%s] < start[%s]' % (end, start)) - return start, end - else: - if start < 0: - start = max(0, start + self.total_size) - return start, self.total_size - - def __SetRangeHeader(self, request, start, end=None): - if start < 0: - request.headers['range'] = 'bytes=%d' % start - elif end is None: - request.headers['range'] = 'bytes=%d-' % start - else: - request.headers['range'] = 'bytes=%d-%d' % (start, end) - - def __GetChunk(self, start, end=None, additional_headers=None): - """Retrieve a chunk, and return the full response.""" - self.EnsureInitialized() - end_byte = min(end or start + self.chunksize, self.total_size) - request = http_wrapper.Request(url=self.url) - self.__SetRangeHeader(request, start, end=end_byte) - if additional_headers is not None: - request.headers.update(additional_headers) - return http_wrapper.MakeRequest(self.bytes_http, request) - - def __ProcessResponse(self, response): - """Process this response (by updating self and writing to self.stream).""" - if response.status_code not in self._ACCEPTABLE_STATUSES: - raise exceptions.TransferInvalidError(response.content) - if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): - self.stream.write(response.content) - self.__progress += len(response) - elif response.status_code == http_client.NO_CONTENT: - # It's important to write something to the stream for the case - # of a 0-byte download to a file, as otherwise python won't - # create the file. - self.stream.write('') - return response - - def GetRange(self, start, end=None, additional_headers=None): - """Retrieve a given byte range from this download, inclusive. - - Range must be of one of these three forms: - * 0 <= start, end = None: Fetch from start to the end of the file. - * 0 <= start <= end: Fetch the bytes from start to end. - * start < 0, end = None: Fetch the last -start bytes of the file. - - (These variations correspond to those described in the HTTP 1.1 - protocol for range headers in RFC 2616, sec. 14.35.1.) - - Args: - start: (int) Where to start fetching bytes. (See above.) - end: (int, optional) Where to stop fetching bytes. (See above.) - additional_headers: (bool, optional) Any additional headers to - pass with the request. - - Returns: - None. Streams bytes into self.stream. - """ - self.EnsureInitialized() - progress, end = self.__NormalizeStartEnd(start, end) - while progress < end: - chunk_end = min(progress + self.chunksize, end) - response = self.__GetChunk(progress, end=chunk_end, - additional_headers=additional_headers) - response = self.__ProcessResponse(response) - progress += len(response) - if not response: - raise exceptions.TransferInvalidError( - 'Zero bytes unexpectedly returned in download response') - - def StreamInChunks(self, callback=None, finish_callback=None, - additional_headers=None): - """Stream the entire download.""" - callback = callback or self._ArgPrinter - finish_callback = finish_callback or self._CompletePrinter - - self.EnsureInitialized() - while True: - if self.__initial_response is not None: - response = self.__initial_response - self.__initial_response = None - else: - response = self.__GetChunk(self.progress, - additional_headers=additional_headers) - response = self.__ProcessResponse(response) - self._ExecuteCallback(callback, response) - if (response.status_code == http_client.OK or - self.progress >= self.total_size): - break - self._ExecuteCallback(finish_callback, response) - - -class Upload(_Transfer): - """Data for a single Upload. - - Fields: - stream: The stream to upload. - mime_type: MIME type of the upload. - total_size: (optional) Total upload size for the stream. - close_stream: (default: False) Whether or not we should close the - stream when finished with the upload. - auto_transfer: (default: True) If True, stream all bytes as soon as - the upload is created. - """ - _REQUIRED_SERIALIZATION_KEYS = set(( - 'auto_transfer', 'mime_type', 'total_size', 'url')) - - def __init__(self, stream, mime_type, total_size=None, http=None, - close_stream=False, chunksize=None, auto_transfer=True): - super(Upload, self).__init__( - stream, close_stream=close_stream, chunksize=chunksize, - auto_transfer=auto_transfer, http=http) - self.__complete = False - self.__mime_type = mime_type - self.__progress = 0 - self.__server_chunk_granularity = None - self.__strategy = None - - self.total_size = total_size - - @property - def progress(self): - return self.__progress - - @classmethod - def FromFile(cls, filename, mime_type=None, auto_transfer=True): - """Create a new Upload object from a filename.""" - path = os.path.expanduser(filename) - if not os.path.exists(path): - raise exceptions.NotFoundError('Could not find file %s' % path) - if not mime_type: - mime_type, _ = mimetypes.guess_type(path) - if mime_type is None: - raise exceptions.InvalidUserInputError( - 'Could not guess mime type for %s' % path) - size = os.stat(path).st_size - return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, - auto_transfer=auto_transfer) - - @classmethod - def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True): - """Create a new Upload object from a stream.""" - if mime_type is None: - raise exceptions.InvalidUserInputError( - 'No mime_type specified for stream') - return cls(stream, mime_type, total_size=total_size, close_stream=False, - auto_transfer=auto_transfer) - - @classmethod - def FromData(cls, stream, json_data, http, auto_transfer=None): - """Create a new Upload of stream from serialized json_data using http.""" - info = json.loads(json_data) - missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) - if missing_keys: - raise exceptions.InvalidDataError( - 'Invalid serialization data, missing keys: %s' % ( - ', '.join(missing_keys))) - upload = cls.FromStream(stream, info['mime_type'], - total_size=info.get('total_size')) - if isinstance(stream, io.IOBase) and not stream.seekable(): - raise exceptions.InvalidUserInputError( - 'Cannot restart resumable upload on non-seekable stream') - if auto_transfer is not None: - upload.auto_transfer = auto_transfer - else: - upload.auto_transfer = info['auto_transfer'] - upload.strategy = _RESUMABLE_UPLOAD - upload._Initialize(http, info['url']) # pylint: disable=protected-access - upload._RefreshResumableUploadState() # pylint: disable=protected-access - upload.EnsureInitialized() - if upload.auto_transfer: - upload.StreamInChunks() - return upload - - @property - def serialization_data(self): - self.EnsureInitialized() - if self.strategy != _RESUMABLE_UPLOAD: - raise exceptions.InvalidDataError( - 'Serialization only supported for resumable uploads') - return { - 'auto_transfer': self.auto_transfer, - 'mime_type': self.mime_type, - 'total_size': self.total_size, - 'url': self.url, - } - - @property - def complete(self): - return self.__complete - - @property - def mime_type(self): - return self.__mime_type - - def __str__(self): - if not self.initialized: - return 'Upload (uninitialized)' - else: - return 'Upload with %d/%s bytes transferred for url %s' % ( - self.progress, self.total_size or '???', self.url) - - @property - def strategy(self): - return self.__strategy - - @strategy.setter - def strategy(self, value): - if value not in (_SIMPLE_UPLOAD, _RESUMABLE_UPLOAD): - raise exceptions.UserError(( - 'Invalid value "%s" for upload strategy, must be one of ' - '"simple" or "resumable".') % value) - self.__strategy = value - - @property - def total_size(self): - return self.__total_size - - @total_size.setter - def total_size(self, value): - self.EnsureUninitialized() - self.__total_size = value - - def __SetDefaultUploadStrategy(self, upload_config, http_request): - """Determine and set the default upload strategy for this upload. - - We generally prefer simple or multipart, unless we're forced to - use resumable. This happens when any of (1) the upload is too - large, (2) the simple endpoint doesn't support multipart requests - and we have metadata, or (3) there is no simple upload endpoint. - - Args: - upload_config: Configuration for the upload endpoint. - http_request: The associated http request. - - Returns: - None. - """ - if self.strategy is not None: - return - strategy = _SIMPLE_UPLOAD - if (self.total_size is not None and - self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): - strategy = _RESUMABLE_UPLOAD - if http_request.body and not upload_config.simple_multipart: - strategy = _RESUMABLE_UPLOAD - if not upload_config.simple_path: - strategy = _RESUMABLE_UPLOAD - self.strategy = strategy - - def ConfigureRequest(self, upload_config, http_request, url_builder): - """Configure the request and url for this upload.""" - # Validate total_size vs. max_size - if (self.total_size and upload_config.max_size and - self.total_size > upload_config.max_size): - raise exceptions.InvalidUserInputError( - 'Upload too big: %s larger than max size %s' % ( - self.total_size, upload_config.max_size)) - # Validate mime type - if not util.AcceptableMimeType(upload_config.accept, self.mime_type): - raise exceptions.InvalidUserInputError( - 'MIME type %s does not match any accepted MIME ranges %s' % ( - self.mime_type, upload_config.accept)) - - self.__SetDefaultUploadStrategy(upload_config, http_request) - if self.strategy == _SIMPLE_UPLOAD: - url_builder.relative_path = upload_config.simple_path - if http_request.body: - url_builder.query_params['uploadType'] = 'multipart' - self.__ConfigureMultipartRequest(http_request) - else: - url_builder.query_params['uploadType'] = 'media' - self.__ConfigureMediaRequest(http_request) - else: - url_builder.relative_path = upload_config.resumable_path - url_builder.query_params['uploadType'] = 'resumable' - self.__ConfigureResumableRequest(http_request) - - def __ConfigureMediaRequest(self, http_request): - """Configure http_request as a simple request for this upload.""" - http_request.headers['content-type'] = self.mime_type - http_request.body = self.stream.read() - - def __ConfigureMultipartRequest(self, http_request): - """Configure http_request as a multipart request for this upload.""" - # This is a multipart/related upload. - msg_root = mime_multipart.MIMEMultipart('related') - # msg_root should not write out its own headers - setattr(msg_root, '_write_headers', lambda self: None) - - # attach the body as one part - msg = mime_nonmultipart.MIMENonMultipart( - *http_request.headers['content-type'].split('/')) - msg.set_payload(http_request.body) - msg_root.attach(msg) - - # attach the media as the second part - msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) - msg['Content-Transfer-Encoding'] = 'binary' - msg.set_payload(self.stream.read()) - msg_root.attach(msg) - - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = io.StringIO() - g = email_generator.Generator(fp, mangle_from_=False) - g.flatten(msg_root, unixfrom=False) - http_request.body = fp.getvalue() - - multipart_boundary = msg_root.get_boundary() - http_request.headers['content-type'] = ( - 'multipart/related; boundary=%r' % multipart_boundary) - - def __ConfigureResumableRequest(self, http_request): - http_request.headers['X-Upload-Content-Type'] = self.mime_type - if self.total_size is not None: - http_request.headers['X-Upload-Content-Length'] = str(self.total_size) - - def _RefreshResumableUploadState(self): - """Talk to the server and refresh the state of this resumable upload.""" - if self.strategy != _RESUMABLE_UPLOAD: - return - self.EnsureInitialized() - refresh_request = http_wrapper.Request( - url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) - refresh_response = http_wrapper.MakeRequest( - self.http, refresh_request, redirections=0) - range_header = refresh_response.info.get( - 'Range', refresh_response.info.get('range')) - if refresh_response.status_code in (http_client.OK, http_client.CREATED): - self.__complete = True - elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: - if range_header is None: - self.__progress = 0 - else: - self.__progress = self.__GetLastByte(range_header) + 1 - self.stream.seek(self.progress) - else: - raise exceptions.HttpError.FromResponse(refresh_response) - - def InitializeUpload(self, http_request, http=None, client=None): - """Initialize this upload from the given http_request.""" - if self.strategy is None: - raise exceptions.UserError( - 'No upload strategy set; did you call ConfigureRequest?') - if http is None and client is None: - raise exceptions.UserError('Must provide client or http.') - if self.strategy != _RESUMABLE_UPLOAD: - return - if self.total_size is None: - raise exceptions.InvalidUserInputError( - 'Cannot stream upload without total size') - http = http or client.http - if client is not None: - http_request.url = client.FinalizeTransferUrl(http_request.url) - self.EnsureUninitialized() - http_response = http_wrapper.MakeRequest(http, http_request) - if http_response.status_code != http_client.OK: - raise exceptions.HttpError.FromResponse(http_response) - - self.__server_chunk_granularity = http_response.info.get( - 'X-Goog-Upload-Chunk-Granularity') - self.__ValidateChunksize() - url = http_response.info['location'] - if client is not None: - url = client.FinalizeTransferUrl(url) - self._Initialize(http, url) - - # Unless the user has requested otherwise, we want to just - # go ahead and pump the bytes now. - if self.auto_transfer: - return self.StreamInChunks() - - def __GetLastByte(self, range_header): - _, _, end = range_header.partition('-') - # TODO(craigcitro): Validate start == 0? - return int(end) - - def __ValidateChunksize(self, chunksize=None): - if self.__server_chunk_granularity is None: - return - chunksize = chunksize or self.chunksize - if chunksize % self.__server_chunk_granularity: - raise exceptions.ConfigurationValueError( - 'Server requires chunksize to be a multiple of %d', - self.__server_chunk_granularity) - - @staticmethod - def _ArgPrinter(response, unused_upload): - print('Sent %s' % response.info['range']) - - @staticmethod - def _CompletePrinter(*unused_args): - print('Upload complete') - - def StreamInChunks(self, callback=None, finish_callback=None, - additional_headers=None): - """Send this (resumable) upload in chunks.""" - if self.strategy != _RESUMABLE_UPLOAD: - raise exceptions.InvalidUserInputError( - 'Cannot stream non-resumable upload') - if self.total_size is None: - raise exceptions.InvalidUserInputError( - 'Cannot stream upload without total size') - callback = callback or self._ArgPrinter - finish_callback = finish_callback or self._CompletePrinter - response = None - self.__ValidateChunksize(self.chunksize) - self.EnsureInitialized() - while not self.complete: - response = self.__SendChunk(self.stream.tell(), - additional_headers=additional_headers) - if response.status_code in (http_client.OK, http_client.CREATED): - self.__complete = True - break - self.__progress = self.__GetLastByte(response.info['range']) - if self.progress + 1 != self.stream.tell(): - # TODO(craigcitro): Add a better way to recover here. - raise exceptions.CommunicationError( - 'Failed to transfer all bytes in chunk, upload paused at byte ' - '%d' % self.progress) - self._ExecuteCallback(callback, response) - self._ExecuteCallback(finish_callback, response) - return response - - def __SendChunk(self, start, additional_headers=None, data=None): - """Send the specified chunk.""" - self.EnsureInitialized() - if data is None: - data = self.stream.read(self.chunksize) - end = start + len(data) - - request = http_wrapper.Request(url=self.url, http_method='PUT', body=data) - request.headers['Content-Type'] = self.mime_type - if data: - request.headers['Content-Range'] = 'bytes %s-%s/%s' % ( - start, end - 1, self.total_size) - if additional_headers: - request.headers.update(additional_headers) - - response = http_wrapper.MakeRequest(self.bytes_http, request) - if response.status_code not in (http_client.OK, http_client.CREATED, - http_wrapper.RESUME_INCOMPLETE): - raise exceptions.HttpError.FromResponse(response) - if response.status_code in (http_client.OK, http_client.CREATED): - return response - # TODO(craigcitro): Add retries on no progress? - last_byte = self.__GetLastByte(response.info['range']) - if last_byte + 1 != end: - new_start = last_byte + 1 - start - response = self.__SendChunk(last_byte + 1, data=data[new_start:]) - return response diff --git a/_gcloud_vendor/apitools/base/py/util.py b/_gcloud_vendor/apitools/base/py/util.py deleted file mode 100644 index 3c3fff53768b..000000000000 --- a/_gcloud_vendor/apitools/base/py/util.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -"""Assorted utilities shared between parts of apitools.""" - -import collections -import os -import random - -import six -from six.moves import http_client -from six.moves.urllib.error import URLError -from six.moves.urllib.parse import quote -from six.moves.urllib.request import urlopen - -from _gcloud_vendor.apitools.base.py import exceptions - -__all__ = [ - 'DetectGae', - 'DetectGce', -] - -_RESERVED_URI_CHARS = r":/?#[]@!$&'()*+,;=" - - -def DetectGae(): - """Determine whether or not we're running on GAE. - - This is based on: - https://developers.google.com/appengine/docs/python/#The_Environment - - Returns: - True iff we're running on GAE. - """ - server_software = os.environ.get('SERVER_SOFTWARE', '') - return (server_software.startswith('Development/') or - server_software.startswith('Google App Engine/')) - - -def DetectGce(): - """Determine whether or not we're running on GCE. - - This is based on: - https://cloud.google.com/compute/docs/metadata#runninggce - - Returns: - True iff we're running on a GCE instance. - """ - try: - o = urlopen('http://metadata.google.internal') - except URLError: - return False - return (o.getcode() == http_client.OK and - o.headers.get('metadata-flavor') == 'Google') - - -def NormalizeScopes(scope_spec): - """Normalize scope_spec to a set of strings.""" - if isinstance(scope_spec, six.string_types): - return set(scope_spec.split(' ')) - elif isinstance(scope_spec, collections.Iterable): - return set(scope_spec) - raise exceptions.TypecheckError( - 'NormalizeScopes expected string or iterable, found %s' % ( - type(scope_spec),)) - - -def Typecheck(arg, arg_type, msg=None): - if not isinstance(arg, arg_type): - if msg is None: - if isinstance(arg_type, tuple): - msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) - else: - msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) - raise exceptions.TypecheckError(msg) - return arg - - -def ExpandRelativePath(method_config, params, relative_path=None): - """Determine the relative path for request.""" - path = relative_path or method_config.relative_path or '' - - for param in method_config.path_params: - param_template = '{%s}' % param - # For more details about "reserved word expansion", see: - # http://tools.ietf.org/html/rfc6570#section-3.2.2 - reserved_chars = '' - reserved_template = '{+%s}' % param - if reserved_template in path: - reserved_chars = _RESERVED_URI_CHARS - path = path.replace(reserved_template, param_template) - if param_template not in path: - raise exceptions.InvalidUserInputError( - 'Missing path parameter %s' % param) - try: - # TODO(craigcitro): Do we want to support some sophisticated - # mapping here? - value = params[param] - except KeyError: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - if value is None: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - try: - if not isinstance(value, six.string_types): - value = str(value) - path = path.replace(param_template, - quote(value.encode('utf_8'), reserved_chars)) - except TypeError as e: - raise exceptions.InvalidUserInputError( - 'Error setting required parameter %s to value %s: %s' % ( - param, value, e)) - return path - - -def CalculateWaitForRetry(retry_attempt, max_wait=60): - """Calculates amount of time to wait before a retry attempt. - - Wait time grows exponentially with the number of attempts. - A random amount of jitter is added to spread out retry attempts from different - clients. - - Args: - retry_attempt: Retry attempt counter. - max_wait: Upper bound for wait time. - - Returns: - Amount of time to wait before retrying request. - """ - - wait_time = 2 ** retry_attempt - # randrange requires a nonzero interval, so we want to drop it if - # the range is too small for jitter. - if retry_attempt: - max_jitter = (2 ** retry_attempt) / 2 - wait_time += random.randrange(-max_jitter, max_jitter) - return min(wait_time, max_wait) - - -def AcceptableMimeType(accept_patterns, mime_type): - """Return True iff mime_type is acceptable for one of accept_patterns. - - Note that this function assumes that all patterns in accept_patterns - will be simple types of the form "type/subtype", where one or both - of these can be "*". We do not support parameters (i.e. "; q=") in - patterns. - - Args: - accept_patterns: list of acceptable MIME types. - mime_type: the mime type we would like to match. - - Returns: - Whether or not mime_type matches (at least) one of these patterns. - """ - unsupported_patterns = [p for p in accept_patterns if ';' in p] - if unsupported_patterns: - raise exceptions.GeneratedClientError( - 'MIME patterns with parameter unsupported: "%s"' % ', '.join( - unsupported_patterns)) - def MimeTypeMatches(pattern, mime_type): - """Return True iff mime_type is acceptable for pattern.""" - # Some systems use a single '*' instead of '*/*'. - if pattern == '*': - pattern = '*/*' - return all(accept in ('*', provided) for accept, provided - in zip(pattern.split('/'), mime_type.split('/'))) - - return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index ff5aef4f9f4b..90348bf38ef5 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -25,8 +25,8 @@ import six from six.moves.urllib.parse import quote # pylint: disable=F0401 -from _gcloud_vendor.apitools.base.py import http_wrapper -from _gcloud_vendor.apitools.base.py import transfer +from apitools.base.py import http_wrapper +from apitools.base.py import transfer from gcloud.credentials import generate_signed_url from gcloud.exceptions import NotFound @@ -347,7 +347,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None, # Should we be passing callbacks through from caller? We can't # pass them as None, because apitools wants to print to the console # by default. - if upload.strategy == transfer._RESUMABLE_UPLOAD: + if upload.strategy == transfer.RESUMABLE_UPLOAD: http_response = upload.StreamInChunks( callback=lambda *args: None, finish_callback=lambda *args: None) diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 242477630780..42c6f8fdf972 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -386,8 +386,8 @@ def test_upload_from_file_resumable(self): from six.moves.urllib.parse import urlsplit from tempfile import NamedTemporaryFile from gcloud._testing import _Monkey - from _gcloud_vendor.apitools.base.py import http_wrapper - from _gcloud_vendor.apitools.base.py import transfer + from apitools.base.py import http_wrapper + from apitools.base.py import transfer BLOB_NAME = 'blob-name' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = b'ABCDEF' @@ -445,7 +445,7 @@ def test_upload_from_file_w_slash_in_name(self): from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit from tempfile import NamedTemporaryFile - from _gcloud_vendor.apitools.base.py import http_wrapper + from apitools.base.py import http_wrapper BLOB_NAME = 'parent/child' UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild' DATA = b'ABCDEF' @@ -465,8 +465,12 @@ def test_upload_from_file_w_slash_in_name(self): fh.write(DATA) fh.flush() blob.upload_from_file(fh, rewind=True) + self.assertEqual(fh.tell(), len(DATA)) rq = connection.http._requested self.assertEqual(len(rq), 1) + self.assertEqual(rq[0]['redirections'], 5) + self.assertEqual(rq[0]['body'], DATA) + self.assertEqual(rq[0]['connection_type'], None) self.assertEqual(rq[0]['method'], 'POST') uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) @@ -487,7 +491,7 @@ def _upload_from_filename_test_helper(self, properties=None, from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit from tempfile import NamedTemporaryFile - from _gcloud_vendor.apitools.base.py import http_wrapper + from apitools.base.py import http_wrapper BLOB_NAME = 'blob-name' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = b'ABCDEF' @@ -551,7 +555,7 @@ def test_upload_from_string_w_bytes(self): from six.moves.http_client import OK from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit - from _gcloud_vendor.apitools.base.py import http_wrapper + from apitools.base.py import http_wrapper BLOB_NAME = 'blob-name' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = b'ABCDEF' @@ -588,7 +592,7 @@ def test_upload_from_string_w_text(self): from six.moves.http_client import OK from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit - from _gcloud_vendor.apitools.base.py import http_wrapper + from apitools.base.py import http_wrapper BLOB_NAME = 'blob-name' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = u'ABCDEF\u1234' @@ -1052,7 +1056,11 @@ def build_api_url(self, path, query_params=None, class _HTTP(_Responder): + connections = {} # For google-apitools debugging. + def request(self, uri, method, headers, body, **kw): + if hasattr(body, 'read'): + body = body.read() return self._respond(uri=uri, method=method, headers=headers, body=body, **kw) diff --git a/run_pylint.py b/run_pylint.py index 107b25d649a0..efefcb1bb0fe 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -29,7 +29,6 @@ IGNORED_DIRECTORIES = [ - '_gcloud_vendor/', ] IGNORED_FILES = [ 'gcloud/datastore/_datastore_v1_pb2.py', diff --git a/setup.py b/setup.py index c66fb94a4103..c8a2d7ac0c8d 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ REQUIREMENTS = [ + 'google-apitools', 'httplib2', 'oauth2client >= 1.4.6', 'protobuf >= 2.5.0', diff --git a/tox.ini b/tox.ini index 03e669373a2f..747f692d574e 100644 --- a/tox.ini +++ b/tox.ini @@ -18,9 +18,7 @@ basepython = commands = nosetests --with-xunit --with-xcoverage --cover-package=gcloud --nocapture --cover-erase --cover-tests --cover-branches --cover-min-percentage=100 deps = - nose - unittest2 - protobuf>=3.0.0-alpha-1 + {[testenv]deps} coverage nosexcover @@ -43,7 +41,7 @@ deps = Sphinx [pep8] -exclude = gcloud/datastore/_datastore_v1_pb2.py,docs/conf.py,*.egg/,.*/,_gcloud_vendor/ +exclude = gcloud/datastore/_datastore_v1_pb2.py,docs/conf.py,*.egg/,.*/ verbose = 1 [testenv:lint] @@ -63,9 +61,6 @@ basepython = python2.7 commands = {toxinidir}/scripts/run_regression.sh -deps = - unittest2 - protobuf>=3.0.0-alpha-1 [testenv:regression3] basepython = @@ -73,9 +68,8 @@ basepython = commands = {toxinidir}/scripts/run_regression.sh deps = - unittest2 + {[testenv]deps} # Use a development checkout of httplib2 until a release is made # incorporating https://github.com/jcgregorio/httplib2/pull/291 # and https://github.com/jcgregorio/httplib2/pull/296 -egit+https://github.com/jcgregorio/httplib2.git#egg=httplib2 - protobuf>=3.0.0-alpha-1