diff --git a/docs/gcloud-config.rst b/docs/gcloud-config.rst new file mode 100644 index 000000000000..8d30d7573cdb --- /dev/null +++ b/docs/gcloud-config.rst @@ -0,0 +1,60 @@ +Configuration +************* + +Overview +======== + +Use service client objects to configure your applications. + +For example: + +.. code-block:: python + + >>> from gcloud import bigquery + >>> client = bigquery.Client() + +When creating a client in this way, the project ID will be determined by +searching these locations in the following order. + +* GCLOUD_PROJECT environment variable +* GOOGLE_APPLICATION_CREDENTIALS JSON file +* Default service configuration path from + ``$ gcloud beta auth application-default login``. +* Google App Engine application ID +* Google Compute Engine project ID (from metadata server) + +You can override the detection of your default project by setting the + ``project`` parameter when creating client objects. + +.. code-block:: python + + >>> from gcloud import bigquery + >>> client = bigquery.Client(project='my-project') + +You can see what project ID a client is referencing by accessing the ``project`` +property on the client object. + +.. code-block:: python + + >>> client.project + u'my-project' + +Authentication +============== + +The authentication credentials can be implicitly determined from the +environment or directly. See :doc:`gcloud-auth`. + +Logging in via ``gcloud beta auth application-default login`` will +automatically configure a JSON key file with your default project ID and +credentials. + +Setting the ``GOOGLE_APPLICATION_CREDENTIALS`` and ``GCLOUD_PROJECT`` +environment variables will override the automatically configured credentials. + +You can change your default project ID to ``my-new-default-project`` by +using the ``gcloud`` CLI tool to change the configuration. + +.. code-block:: bash + + $ gcloud config set project my-new-default-project diff --git a/docs/index.rst b/docs/index.rst index c026d3bf8887..df0aa0ea9980 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,6 +4,7 @@ :caption: gcloud gcloud-api + gcloud-config gcloud-auth .. toctree:: diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 5853785adfaf..05d1fd698791 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -18,6 +18,7 @@ import calendar import datetime +import json import os import re import socket @@ -27,8 +28,10 @@ from google.protobuf import timestamp_pb2 import six from six.moves.http_client import HTTPConnection +from six.moves import configparser from gcloud.environment_vars import PROJECT +from gcloud.environment_vars import CREDENTIALS try: from google.appengine.api import app_identity @@ -48,6 +51,7 @@ (?P\d{1,9}) # nanoseconds, maybe truncated Z # Zulu """, re.VERBOSE) +DEFAULT_CONFIGURATION_PATH = '~/.config/gcloud/configurations/config_default' class _LocalStack(Local): @@ -129,13 +133,13 @@ def _ensure_tuple_or_list(arg_name, tuple_or_list): This effectively reduces the iterable types allowed to a very short whitelist: list and tuple. - :type arg_name: string + :type arg_name: str :param arg_name: Name of argument to use in error message. - :type tuple_or_list: sequence of string + :type tuple_or_list: sequence of str :param tuple_or_list: Sequence to be verified. - :rtype: list of string + :rtype: list of str :returns: The ``tuple_or_list`` passed in cast to a ``list``. :raises: class:`TypeError` if the ``tuple_or_list`` is not a tuple or list. @@ -149,7 +153,7 @@ def _ensure_tuple_or_list(arg_name, tuple_or_list): def _app_engine_id(): """Gets the App Engine application ID if it can be inferred. - :rtype: string or ``NoneType`` + :rtype: str or ``NoneType`` :returns: App Engine application ID if running in App Engine, else ``None``. """ @@ -159,6 +163,42 @@ def _app_engine_id(): return app_identity.get_application_id() +def _file_project_id(): + """Gets the project id from the credentials file if one is available. + + :rtype: str or ``NoneType`` + :returns: Project-ID from JSON credentials file if value exists, + else ``None``. + """ + credentials_file_path = os.getenv(CREDENTIALS) + if credentials_file_path: + with open(credentials_file_path, 'rb') as credentials_file: + credentials_json = credentials_file.read() + credentials = json.loads(credentials_json.decode('utf-8')) + return credentials.get('project_id') + + +def _default_service_project_id(): + """Retrieves the project ID from the gcloud command line tool. + + Files that cannot be opened with configparser are silently ignored; this is + designed so that you can specify a list of potential configuration file + locations. + + :rtype: str or ``NoneType`` + :returns: Project-ID from default configuration file else ``None`` + """ + full_config_path = os.path.expanduser(DEFAULT_CONFIGURATION_PATH) + win32_config_path = os.path.join(os.getenv('APPDATA', ''), + 'gcloud', 'configurations', + 'config_default') + config = configparser.RawConfigParser() + config.read([full_config_path, win32_config_path]) + + if config.has_section('core'): + return config.get('core', 'project') + + def _compute_engine_id(): """Gets the Compute Engine project ID if it can be inferred. @@ -172,7 +212,7 @@ def _compute_engine_id(): See https://github.com/google/oauth2client/issues/93 for context about DNS latency. - :rtype: string or ``NoneType`` + :rtype: str or ``NoneType`` :returns: Compute Engine project ID if the metadata service is available, else ``None``. """ @@ -204,18 +244,27 @@ def _determine_default_project(project=None): implicit environments are: * GCLOUD_PROJECT environment variable + * GOOGLE_APPLICATION_CREDENTIALS JSON file + * Get default service project from + ``$ gcloud beta auth application-default login`` * Google App Engine application ID * Google Compute Engine project ID (from metadata server) - :type project: string + :type project: str :param project: Optional. The project name to use as default. - :rtype: string or ``NoneType`` + :rtype: str or ``NoneType`` :returns: Default project if it can be determined. """ if project is None: project = _get_production_project() + if project is None: + project = _file_project_id() + + if project is None: + project = _default_service_project_id() + if project is None: project = _app_engine_id() @@ -231,7 +280,7 @@ def _millis(when): :type when: :class:`datetime.datetime` :param when: the datetime to convert - :rtype: integer + :rtype: int :returns: milliseconds since epoch for ``when`` """ micros = _microseconds_from_datetime(when) @@ -256,7 +305,7 @@ def _microseconds_from_datetime(value): :type value: :class:`datetime.datetime` :param value: The timestamp to convert. - :rtype: integer + :rtype: int :returns: The timestamp, in microseconds. """ if not value.tzinfo: @@ -273,7 +322,7 @@ def _millis_from_datetime(value): :type value: :class:`datetime.datetime`, or None :param value: the timestamp - :rtype: integer, or ``NoneType`` + :rtype: int, or ``NoneType`` :returns: the timestamp, in milliseconds, or None """ if value is not None: @@ -430,20 +479,20 @@ def _datetime_to_pb_timestamp(when): def _name_from_project_path(path, project, template): """Validate a URI path and get the leaf object's name. - :type path: string + :type path: str :param path: URI path containing the name. - :type project: string or NoneType + :type project: str or NoneType :param project: The project associated with the request. It is included for validation purposes. If passed as None, disables validation. - :type template: string + :type template: str :param template: Template regex describing the expected form of the path. The regex must have two named groups, 'project' and 'name'. - :rtype: string + :rtype: str :returns: Name parsed from ``path``. :raises: :class:`ValueError` if the ``path`` is ill-formed or if the project from the ``path`` does not agree with the diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index dffd8f0cc38c..21bb8b002633 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -148,6 +148,85 @@ def test_value_set(self): self.assertEqual(dataset_id, APP_ENGINE_ID) +class Test__get_credentials_file_project_id(unittest2.TestCase): + + def _callFUT(self): + from gcloud._helpers import _file_project_id + return _file_project_id() + + def setUp(self): + import os + self.old_env = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + + def tearDown(self): + import os + if (not self.old_env and + 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ): + del os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + + def test_success(self): + import os + import tempfile + with tempfile.NamedTemporaryFile(mode='w') as credential_file: + credential_file.write('{"project_id": "test-project-id"}') + credential_file.seek(0) + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credential_file.name + + self.assertEqual('test-project-id', self._callFUT()) + + def test_no_environment(self): + self.assertEqual(None, self._callFUT()) + + +class Test__get_default_service_project_id(unittest2.TestCase): + temp_config_path = None + config_path = '.config/gcloud/configurations/' + config_file = 'config_default' + + def setUp(self): + import tempfile + import os + self.temp_config_path = tempfile.gettempdir() + + conf_path = os.path.join(self.temp_config_path, self.config_path) + os.makedirs(conf_path) + full_config_path = os.path.join(conf_path, self.config_file) + + self.temp_config_file = full_config_path + + with open(full_config_path, 'w') as conf_file: + conf_file.write('[core]\nproject = test-project-id') + + def tearDown(self): + import os + import shutil + + if self.temp_config_path: + shutil.rmtree(os.path.join(self.temp_config_path, + '.config')) + + def callFUT(self, project_id=None): + from gcloud._helpers import _default_service_project_id + import os + + def mock_expanduser(path=''): + if project_id and path.startswith('~'): + return self.temp_config_file + return '' + + from gcloud._testing import _Monkey + with _Monkey(os.path, expanduser=mock_expanduser): + return _default_service_project_id() + + def test_read_from_cli_info(self): + project_id = self.callFUT('test-project-id') + self.assertEqual('test-project-id', project_id) + + def test_info_value_not_present(self): + project_id = self.callFUT() + self.assertEqual(None, project_id) + + class Test__compute_engine_id(unittest2.TestCase): def _callFUT(self): @@ -219,7 +298,7 @@ def _callFUT(self, project=None): return _determine_default_project(project=project) def _determine_default_helper(self, prod=None, gae=None, gce=None, - project=None): + file_id=None, srv_id=None, project=None): from gcloud._testing import _Monkey from gcloud import _helpers @@ -229,6 +308,14 @@ def prod_mock(): _callers.append('prod_mock') return prod + def file_id_mock(): + _callers.append('file_id_mock') + return file_id + + def srv_id_mock(): + _callers.append('srv_id_mock') + return srv_id + def gae_mock(): _callers.append('gae_mock') return gae @@ -239,6 +326,8 @@ def gce_mock(): patched_methods = { '_get_production_project': prod_mock, + '_file_project_id': file_id_mock, + '_default_service_project_id': srv_id_mock, '_app_engine_id': gae_mock, '_compute_engine_id': gce_mock, } @@ -251,7 +340,8 @@ def gce_mock(): def test_no_value(self): project, callers = self._determine_default_helper() self.assertEqual(project, None) - self.assertEqual(callers, ['prod_mock', 'gae_mock', 'gce_mock']) + self.assertEqual(callers, ['prod_mock', 'file_id_mock', 'srv_id_mock', + 'gae_mock', 'gce_mock']) def test_explicit(self): PROJECT = object() @@ -269,13 +359,15 @@ def test_gae(self): PROJECT = object() project, callers = self._determine_default_helper(gae=PROJECT) self.assertEqual(project, PROJECT) - self.assertEqual(callers, ['prod_mock', 'gae_mock']) + self.assertEqual(callers, ['prod_mock', 'file_id_mock', + 'srv_id_mock', 'gae_mock']) def test_gce(self): PROJECT = object() project, callers = self._determine_default_helper(gce=PROJECT) self.assertEqual(project, PROJECT) - self.assertEqual(callers, ['prod_mock', 'gae_mock', 'gce_mock']) + self.assertEqual(callers, ['prod_mock', 'file_id_mock', 'srv_id_mock', + 'gae_mock', 'gce_mock']) class Test__millis(unittest2.TestCase):