diff --git a/docs/_static/js/main.js b/docs/_static/js/main.js index 11b8202481a0..8f95b772f38a 100755 --- a/docs/_static/js/main.js +++ b/docs/_static/js/main.js @@ -16,7 +16,7 @@ $('.headerlink').parent().each(function() { $('.side-nav').children('ul:nth-child(2)').children().each(function() { var itemName = $(this).text(); if (itemName !== 'Datastore' && itemName !== 'Storage' && - itemName !== 'Pub/Sub') { + itemName !== 'Pub/Sub' && itemName !== 'Search') { $(this).css('padding-left','2em'); } }); diff --git a/docs/index.rst b/docs/index.rst index 963d2fe1cbce..0af1385f48b3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,14 +9,20 @@ datastore-queries datastore-transactions datastore-batches - storage-api - storage-blobs - storage-buckets - storage-acl + datastore-dataset pubsub-api pubsub-usage pubsub-subscription pubsub-topic + search-api + search-client + search-index + search-document + search-field + storage-api + storage-blobs + storage-buckets + storage-acl Getting started diff --git a/docs/search-api.rst b/docs/search-api.rst new file mode 100644 index 000000000000..4fdab1688204 --- /dev/null +++ b/docs/search-api.rst @@ -0,0 +1,137 @@ +.. toctree:: + :maxdepth: 1 + :hidden: + +Search +------ + +Overview +~~~~~~~~ + +Cloud Search allows you to quickly perform full-text and geospatial searches +against your data without having to spin up your own instances +and without the hassle of managing and maintaining a search service. + +Cloud Search provides a model for indexing your documents +that contain structured data, +with documents and indexes saved to a separate persistent store +optimized for search operations. +You can search an index, and organize and present your search results. +The API supports full text matching on string fields +and allows you to index any number of documents in any number of indexes. + +Indexes +~~~~~~~ + +Here's an example of how you might deal with indexes:: + + >>> from gcloud import search + >>> client = search.Client() + + >>> # List all indexes in your project + >>> for index in client.list_indexes(): + ... print index + + >>> # Create a new index + >>> new_index = client.index('index-id-here') + >>> new_index.name = 'My new index' + >>> new_index.create() + + >>> # Update an existing index + >>> index = client.get_index('existing-index-id') + >>> print index + + >>> index.name = 'Modified name' + >>> index.update() + >>> print index + + + >>> # Delete an index + >>> index = client.get_index('existing-index-id') + >>> index.delete() + +Documents +~~~~~~~~~ + +Documents are the things that you search for. +The typical process is: + +#. Create a document +#. Add fields to the document +#. Add the document to an index to be searched for later + +Here's an example of how you might deal with documents:: + + >>> from gcloud import search + >>> client = search.Client() + + >>> # Create a document + >>> document = search.Document('document-id') + + >>> # Add a field to the document + >>> field = search.Field('fieldname') + >>> field.add_value('string') + >>> document.add_field(field) + + >>> # Add the document to an index + >>> index = client.get_index('existing-index-id') + >>> index.add_document(document) + +Fields +~~~~~~ + +Fields belong to documents and are the data that actually gets searched. +Each field can have multiple values, +and there are three different types of tokenization for string values: + +- **Atom** (``atom``) means "don't tokenize this string", treat it as one thing + to compare against. +- **Text** (``text``) means "treat this string as normal text" and split words + apart to be compared against. +- **HTML** (``html``) means "treat this string as HTML", understanding the + tags, and treating the rest of the content like Text. + +You can set this using the ``tokenization`` paramater when adding a field +value:: + + >>> from gcloud import search + >>> document = search.Document('document-id') + >>> document.add_field(search.Field('field-name', values=[ + ... search.Value('britney spears', tokenization='atom'), + ... search.Value('

Britney Spears

', tokenization='html'), + ... ])) + +Searching +~~~~~~~~~ + +Once you have indexes full of documents, you can search through them by +issuing a search query. + +Here's a simple example of how you might start searching:: + + >>> from gcloud import search + >>> client = search.Client() + + >>> index = client.get_index('existing-index-id') + >>> query = search.Query('britney spears') + >>> matching_documents = index.search(query) + >>> for document in matching_documents: + ... print document + +By default, all queries are sorted by the ``rank`` value you set +when the documented was created. +If you want to sort differently, use the ``order_by`` parameter:: + + >>> from gcloud import search + >>> query = search.Query('britney spears', order_by=['field1', '-field2']) + +Note that the ``-`` character before ``field2`` means that +this query will be sorted ascending by ``field1`` +and then descending by ``field2``. + +If you want only want certain fields to be returned in the match, +you can use the ``fields`` paramater:: + + >>> from gcloud import search + >>> query = search.Query('britney spears', fields=['field1', 'field2']) + diff --git a/docs/search-client.rst b/docs/search-client.rst new file mode 100644 index 000000000000..09a79e5d6f20 --- /dev/null +++ b/docs/search-client.rst @@ -0,0 +1,19 @@ +.. toctree:: + :maxdepth: 0 + :hidden: + +Client +------ + +.. automodule:: gcloud.search.client + :members: + :undoc-members: + :show-inheritance: + +Connection +~~~~~~~~~~ + +.. automodule:: gcloud.search.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/search-document.rst b/docs/search-document.rst new file mode 100644 index 000000000000..ebc6c53ed133 --- /dev/null +++ b/docs/search-document.rst @@ -0,0 +1,11 @@ +.. toctree:: + :maxdepth: 0 + :hidden: + +Document +-------- + +.. automodule:: gcloud.search.document + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/search-field.rst b/docs/search-field.rst new file mode 100644 index 000000000000..9dcc2c9bcb88 --- /dev/null +++ b/docs/search-field.rst @@ -0,0 +1,19 @@ +.. toctree:: + :maxdepth: 0 + :hidden: + +Field +----- + +.. automodule:: gcloud.search.field + :members: + :undoc-members: + :show-inheritance: + +Value +~~~~~ + +.. automodule:: gcloud.search.value + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/search-index.rst b/docs/search-index.rst new file mode 100644 index 000000000000..468d8e0f4e36 --- /dev/null +++ b/docs/search-index.rst @@ -0,0 +1,11 @@ +.. toctree:: + :maxdepth: 0 + :hidden: + +Index +----- + +.. automodule:: gcloud.search.index + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/search-usage.rst b/docs/search-usage.rst new file mode 100644 index 000000000000..e87025e94636 --- /dev/null +++ b/docs/search-usage.rst @@ -0,0 +1,314 @@ +Using the API +============= + +Connection / Authorization +-------------------------- + +Implicitly use the default client: + +.. doctest:: + + >>> from gcloud import search + >>> # The search module has the same methods as a client, using the default. + >>> search.list_indexes() # API request + [] + +Configure the default client: + +.. doctest:: + + >>> from gcloud import search + >>> search.set_project_id('project-id') + >>> search.set_credentials(credentials) + >>> search.list_indexes() # API request + [] + +Explicitly use the default client: + +.. doctest:: + + >>> from gcloud.search import default_client as client + >>> # The default_client is equivalent to search.Client() + >>> client.list_indexes() # API request + [] + +Explicitly configure a client: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client(project_id='project-id', credentials=credentials) + >>> client.list_indexes() # API request + [] + +Manage indexes for a project +---------------------------- + +Create a new index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.create_index('index_id') # API request + >>> index.id + 'index_id' + +Create a new index with a name: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.create_index('index_id', name='Name') # API request + >>> index.name + 'Name' + +Get or create an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_or_create_index('index_id') # API request + >>> index.id + 'index_id' + +List the indexes: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> [index.id for index in client.list_indexes()] # API request + ['index_id'] + +Retrieve an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('missing_index_id') # API request + >>> index is None + True + >>> index = client.get_index('index_id') # API request + >>> index.id + 'index_id' + +Get an index without making an API request + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id', check=False) + >>> index.id + 'index_id' + +Update an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> index.name = 'Name' + >>> client.update_index(index) + +Delete an index by ID: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> client.delete_index('index_id') # API request + +Delete an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> index.id + 'index_id' + >>> client.delete_index(index) # API request + +Manage documents and fields +--------------------------- + +Create a document + +.. doctest:: + + >>> from gcloud import search + >>> document = search.Document('document_id', rank=0) + >>> document.id + 'document_id' + +Add a field to a document + +.. doctest:: + + >>> from gcloud import search + >>> document = search.Document('document_id') + >>> document.add_field(search.Field('fieldname')) + +Add values to a field + +.. doctest:: + + >>> from datetime import datetime + >>> from gcloud import search + >>> field = search.Field('fieldname') + >>> field.add_value('string') + >>> # Tokenization field ignored for non-string values. + >>> field.add_value('

string

', tokenization='html') + >>> field.add_value('string', tokenization='atom') + >>> field.add_value('string', tokenization='text') + >>> field.add_value(1234) + >>> field.add_value(datetime.now()) + >>> len(field.values) + 9 + +Add values to a field at initialization time + +.. doctest:: + + >>> from gcloud import search + >>> field = search.Field('fieldname', values=[ + 'string', + search.Value('

string2

', tokenization='html') + search.Value('string', tokenization='atom')]) + +Add a single document to an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> document = search.Document('document_id', rank=0) + >>> index.add_document(document) # API request + +Add multiple documents to an index: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> documents = [search.Document('document_id')] + >>> index.add_documents(documents) # API request + +Get a single document by ID: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> document = index.get_document('missing_document_id') # API request + >>> document is None + True + >>> document = index.get_document('document_id') # API request + >>> document.fields + [] + +Delete a document by ID: + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> index.delete_document('document_id') # API request + >>> index.delete_document('missing_document_id') # API request + +Searching +--------- + +Create a query + +.. doctest:: + + >>> from gcloud import search + >>> query = search.Query('query text here') + >>> query.query + 'query text here' + +Specify the fields to return in a query + +.. doctest:: + + >>> from gcloud import search + >>> query = search.Query('query text here', fields=['field1', 'field2']) + >>> query.fields + ['field1', 'field2'] + +Set the sort order of a query + +.. doctest:: + + >>> from gcloud import search + >>> query = search.Query('query text here', order_by='field1') + >>> query.order_by + 'field1' + >>> query2 = search.Query('query text here', order_by=['field2', 'field3']) + >>> query2.order_by + ['field2', 'field3'] + >>> # Order descending by field1 and ascending by field2 + >>> query4 = search.Query('query text here', order_by=['-field1', 'field2']) + >>> query3.order_by + ['-field1', 'field2'] + +Set custom field expressions on a query + +.. doctest:: + + >>> from gcloud import search + >>> query = search.Query('query text here') + >>> query.add_field_expression('total_price', '(price + tax)') + >>> # We don't do any checks on the expression. These are checked at query time. + >>> query.add_field_expression('invalid', 'is_prime(num)') + >>> query.add_field_expression('bad_field', '(missing_field + tax)') + +Set custom field expressions at initialization time + +.. doctest:: + + >>> from gcloud import search + >>> query = search.Query('query text here', field_expressions={ + 'total_price': '(price + tax)'}) + +Search an index + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> matching = index.search(search.Query('query text here')) # API request + >>> for document in matching: + ... print document.id + +Search an index with a limit on number of results + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> matching = index.search(search.Query('query text here'), + ... limit=42) # API request + +Search an index with a custom page size (advanced) + +.. doctest:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('index_id') # API request + >>> matching = index.search(search.Query('query text here'), + ... page_size=20) # API request diff --git a/gcloud/search/__init__.py b/gcloud/search/__init__.py new file mode 100644 index 000000000000..4616e6075b6c --- /dev/null +++ b/gcloud/search/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Search API wrapper. + +The main concepts with this API are: + +- :class:`gcloud.pubsub.topic.Topic` represents an endpoint to which messages + can be published using the Cloud Storage Pubsub API. + +- :class:`gcloud.pubsub.subscription.Subscription` represents a named + subscription (either pull or push) to a topic. +""" + +from gcloud.search.client import Client +from gcloud.search.connection import SCOPE diff --git a/gcloud/search/client.py b/gcloud/search/client.py new file mode 100644 index 000000000000..e4d17ac59b40 --- /dev/null +++ b/gcloud/search/client.py @@ -0,0 +1,95 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Client for interacting with the Cloud Search API.""" + + +from gcloud.client import JSONClient +from gcloud.exceptions import NotFound +from gcloud.iterator import Iterator +from gcloud.search.connection import Connection +from gcloud.search.index import Index + + +class Client(JSONClient): + """Client to bundle configuration needed for API requests.""" + + _connection_class = Connection + + def list_indexes(self, page_size=None): + """List topics for the project associated with this client. + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/list + + :type page_size: int + :param page_size: maximum number of indexes to return, If not passed, + defaults to a value set by the API. + + :rtype: :class:`gcloud.iterator.Iterator` + :returns: an :class:`gcloud.iterator.Iterator` of + :class:`gcloud.search.index.Index` + """ + params = {} + + if page_size is not None: + params['pageSize'] = page_size + + path = '/projects/%s/indexes' % (self.project,) + + client = self + + class IndexIterator(Iterator): + """An iterator over a list of Index resources.""" + + def get_items_from_response(self, response): + """Get :class:`gcloud.search.index.Index` items from response. + + :type response: dict + :param response: The JSON API response for a page of indexes. + """ + for resource in response.get('indexes', []): + item = Index.from_api_repr(resource, client=client) + yield item + + return IndexIterator(connection=self.connection, extra_params=params, + path=path) + + def index(self, index_id): + """Creates an index bound to the current client. + + :type index_id: string + :param index_id: the ID of the index to be constructed. + + :rtype: :class:`gcloud.search.index.Index` + :returns: the index created with the current client. + """ + return Index(index_id=index_id, client=self) + + def get_index(self, index_id): + """Retrieves an index from the Cloud Search API. + + :type index_id: string + :param index_id: the ID of the index to be retrieved. + + :rtype: :class:`gcloud.search.index.Index` + :returns: the index retrieved via the current client or ``None`` if + the index with that ID doesn't exist. + """ + try: + index = self.index(index_id) + index.reload() + except NotFound: + index = None + return index diff --git a/gcloud/search/connection.py b/gcloud/search/connection.py new file mode 100644 index 000000000000..e26bf4430348 --- /dev/null +++ b/gcloud/search/connection.py @@ -0,0 +1,39 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Connection information for the Cloud Search API.""" + +from gcloud import connection as base_connection + + +SCOPE = ('https://www.googleapis.com/auth/cloudsearch', + 'https://www.googleapis.com/auth/cloud-platform') +"""The scopes required for authenticating as a Cloud Search consumer.""" + + +class Connection(base_connection.JSONConnection): + """A connection to Google Cloud Search via the JSON REST API.""" + + API_BASE_URL = 'https://cloudsearch.googleapis.com' + """The base of the API call URL.""" + + API_VERSION = 'v1' + """The version of the API, used in building the API call's URL.""" + + API_URL_TEMPLATE = '{api_base_url}/{api_version}{path}' + """A template for the URL of a particular API call.""" + + def __init__(self, credentials=None, http=None): + credentials = self._create_scoped_credentials(credentials, SCOPE) + super(Connection, self).__init__(credentials=credentials, http=http) diff --git a/gcloud/search/document.py b/gcloud/search/document.py new file mode 100644 index 000000000000..de145bb0c66e --- /dev/null +++ b/gcloud/search/document.py @@ -0,0 +1,131 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define a Cloud Search Document.""" + + +class Document(object): + """Documents are the entities that are searchable via the Cloud Search API + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/documents#resource_representation.google.cloudsearch.v1.Document + + :type document_id: string + :param document_id: the id of the document + + :type fields: list of :class:`gcloud.search.field.Field` + :param fields: the fields attached to the document + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration. + + :type index: :class:`gcloud.search.index.Index` + :param index: the index that the document belongs to. + """ + def __init__(self, document_id, fields=None, client=None, index=None): + self.document_id = document_id + self.fields = fields or [] + self._client = client + self.index = index + + @classmethod + def from_api_repr(cls, resource, client=None): + """Factory: construct a document given its API representation + + :type resource: dict + :param resource: the resource representation returned from the API + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration. + + :rtype: :class:`gcloud.search.document.Document` + :returns: a document parsed from ``resource``. + """ + return cls(document_id=resource['id'], client=client) + + @property + def client(self): + """The client associated with this document. + + This first checks the client set directly, then checks the client set + on the index. If no client is found, returns ``None``. + + :rtype: :class:`gcloud.search.client.Client` or ``None`` + :returns: The client associated with this document. + """ + if self._client: + return self._client + elif self._index: + return self._index.client + + @property + def path(self): + """URL path for the document's API calls""" + if not self.index: + raise ValueError('Missing Index.') + return '%s/documents/%s' % (self.index.path, self.document_id) + + def _require_client(self, client=None): + """Get either a client or raise an exception. + + We need to use this as the various methods could accept a client as a + parameter, which we need to evaluate. If the client provided is empty + and there is no client set as an instance variable, we'll raise a + :class:`ValueError`. + + :type client: :class:`gcloud.search.client.Client` + :param client: An optional client to test for existence. + """ + client = client or self.client + if not client: + raise ValueError('Missing client. You can set the client ' + 'directly on the document, or indirectly on the ' + 'index.') + return client + + def add_field(self, field): + """Add a field to this document. + + :type field: :class:`gcloud.search.field.Field` + :param field: The field to add to the document. + """ + pass + + def reload(self, client=None): + """API call: sync this document's data via a GET request. + + See + https://cloud.google.com/search/reference/rest/v1/projects/indexes/documents + + :type client: :class:`gcloud.search.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current document's index. + """ + # client = self._require_client(client) + # data = client.connection.api_request(method='GET', path=self.path) + + def delete(self, client=None): + """API call: delete the document via a DELETE request. + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/documents/delete + + :type client: :class:`gcloud.search.client.Client` or ``None`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored. + """ + client = self._require_client(client) + client.connection.api_request(method='DELETE', path=self.path) diff --git a/gcloud/search/field.py b/gcloud/search/field.py new file mode 100644 index 000000000000..2d811422af7e --- /dev/null +++ b/gcloud/search/field.py @@ -0,0 +1,59 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define Search Field.""" + + +from gcloud.search.value import Value + + +class Field(object): + """Fields store data which makes up a document. + + See: + https://cloud.google.com/search/reference/rest/google/cloudsearch/v1/FieldValueList + + :type name: string + :param name: the name of the field + + :type values: iterable of string or ``None`` + :param values: the list of values to be associated with this field. + """ + def __init__(self, name, values=None): + self.name = name + self.values = values or [] + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct a field given its API representation + + :type resource: dict + :param resource: the resource representation returned from the API + + :rtype: :class:`gcloud.search.field.Field` + :returns: Field parsed from ``resource``. + """ + return cls(name=resource['name']) + + def add_value(self, value, tokenization=None): + """Add a value to this field. + + :type value: string + :param value: The value to add to the field. + + :type tokenization: string + :param tokenization: The tokenization type of the value. + """ + self.values.append(Value(value=value, + tokenization=tokenization)) diff --git a/gcloud/search/index.py b/gcloud/search/index.py new file mode 100644 index 000000000000..f005d4f9be4f --- /dev/null +++ b/gcloud/search/index.py @@ -0,0 +1,326 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define Indexes for the Cloud Search API.""" + +from gcloud.exceptions import NotFound + + +class Index(object): + """Indexes are sets of documents that you can search over. + + To create a new index:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.index('my-new-index') + >>> index.create() + + To get an existing index:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('my-existing-index') + >>> print index + + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/list#google.cloudsearch.v1.IndexInfo + + :type index_id: string + :param index_id: the unique ID of the index. + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration. + """ + def __init__(self, client, index_id): + self.client = client + self.index_id = index_id + + def __repr__(self): + return '' % (self.index_id,) + + @classmethod + def from_api_repr(cls, resource, client): + """Factory: construct an index given its API representation. + + :type resource: dict + :param resource: the resource representation returned from the API + + :type client: :class:`gcloud.pubsub.client.Client` + :param client: Client which holds credentials and project + configuration for the index. + + :rtype: :class:`gcloud.search.index.Index` + """ + index = cls(index_id=resource['indexId'], client=client) + index.set_properties_from_api_repr(resource) + return index + + def set_properties_from_api_repr(self, resource): + """Update specific properties from its API representation.""" + pass + + @property + def project(self): + """Project bound to the index.""" + return self.client.project + + @property + def full_name(self): + """Fully-qualified name.""" + if not self.project: + raise ValueError('Missing project ID!') + return 'projects/%s/indexes/%s' % (self.project, self.index_id) + + @property + def path(self): + """URL for the index.""" + return '/%s' % (self.full_name) + + def _require_client(self, client=None): + """Get either a client or raise an exception. + + We need to use this as the various methods could accept a client as a + parameter, which we need to evaluate. If the client provided is empty + and there is no client set as an instance variable, we'll raise a + ValueError. + + :type client: :class:`gcloud.search.client.Client` + :param client: An optional client to test for existence. + """ + client = client or self.client + if not client: + raise ValueError('Missing client.') + return client + + def create(self, client=None): + """API call: create the index via a ``POST`` request. + + Example:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.index('my-new-index') + >>> index.create() + + See + https://cloud.google.com/search/reference/rest/v1/projects/indexes/create + + :type client: :class:`gcloud.search.client.Client` or None + :param client: the client to use. If not passed, falls back to + the ``client`` attribute. + """ + # Right now this is a no-op as indexes are implicitly created. + # Later, this will be a PUT request to the API. + # client = self._require_client(client=client) + # resp = client.connection.api_request(method='POST', path=self.path) + # self.set_properties_from_api_repr(resource=resp) + + def reload(self, client=None): + """API call: reload the index via a ``GET`` request. + + This method will reload the freshest metadata for an index. + + .. warning:: + + This will overwrite any local changes you've made and not saved! + + Example:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('my-existing-index') + >>> index.name = 'Locally changed name' + >>> print index + + >>> index.reload() + >>> print index + + + See + https://cloud.google.com/search/reference/rest/v1/projects/indexes/get + + :type client: :class:`gcloud.search.client.Client` or None + :param client: the client to use. If not passed, falls back to + the ``client`` attribute. + """ + # client = self._require_client(client=client) + + # We assume the index exists. If it doesn't it will raise a NotFound + # exception. + # Right now this is a no-op, as there's no extra data with an index. + # resp = client.connection.api_request(method='GET', path=self.path) + # self.set_properties_from_api_repr(resource=resp) + + def update(self, client=None): + """API call: update the index via a ``PUT`` request. + + Example:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('my-existing-index') + >>> index.name = 'New Purple Spaceship' + >>> index.labels['environment'] = 'prod' + >>> index.update() + + See + https://cloud.google.com/search/reference/rest/v1/projects/indexes/update + + :type client: :class:`gcloud.search.client.Client` or None + :param client: the client to use. If not passed, falls back to + the ``client`` attribute. + """ + client = self._require_client(client=client) + + # Right now this is a no-op, as indexes have no extra data. + # data = {'name': self.name, 'labels': self.labels} + # resp = client.connection.api_request(method='PUT', path=self.path, + # data=data) + # self.set_properties_from_api_repr(resp) + + def exists(self, client=None): + """API call: test the existence of an index via a ``GET`` request. + + Example:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.index('missing-index-id') + >>> index.exists() + False + + You can also use the + :func:`gcloud.search.client.Client.get_index` + method to check whether an index exists, as it will return ``None`` + if the index doesn't exist:: + + >>> from gcloud import search + >>> client = search.Client() + >>> print client.get_index('missing-index-id') + None + + See + https://cloud.google.com/search/reference/rest/v1/projects/indexes/get + + :type client: :class:`gcloud.search.client.Client` or None + :param client: the client to use. If not passed, falls back to + the ``client`` attribute. + """ + # Currently there is no way to do this, so this is a no-op. + # client = self._require_client(client=client) + # + # try: + # client.connection.api_request(method='GET', path=self.path) + # except NotFound: + # return False + # else: + # return True + + def delete(self, client=None, reload=True): + """API call: delete the index via a ``DELETE`` request. + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/delete + + This actually changes the status (````) from ``ACTIVE`` + to ``DELETING``. + Later (it's not specified when), the index will move into the + ``DELETE_IN_PROGRESS`` state, which means the deleting has actually + begun. + + Example:: + + >>> from gcloud import search + >>> client = search.Client() + >>> index = client.get_index('existing-index-id') + >>> index.delete() + + :type client: :class:`gcloud.search.client.Client` or None + :param client: the client to use. If not passed, + falls back to the ``client`` attribute. + + :type reload: bool + :param reload: Whether to reload the index with the latest state. + Default: ``True``. + """ + # This currently is not possible. + # client = self._require_client(client) + # client.connection.api_request(method='DELETE', path=self.path) + # + # if reload: + # self.reload() + + def document(self, document_id): + """Get a document instance without making an API call. + + :type document_id: string + :param document_id: The unique ID for the document. + + :rtype: :class:`gcloud.search.document.Document` + """ + pass + + def get_document(self, document_id, client=None): + """API call: Retrieve a document via a ``GET`` request. + + See: + https://cloud.google.com/search/reference/rest/v1/projects/indexes/documents/get + + :type document_id: string + :param document_id: The unique ID for the document. + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration for the index. + + :rtype: :class:`gcloud.search.document.Document` + """ + client = self._require_client(client=client) + document = self.document(document_id) + try: + document.reload() + except NotFound: + document = None + return document + + def add_document(self, document, client=None): + """API call: Add a document to this index. + + :type document: :class:`gcloud.search.document.Document` + :param document: The document to add to the index. + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration for the index. + """ + self.add_document([document], client=client) + + def add_documents(self, documents, client=None): + """API call: Add a list of documents to this index. + + :type documents: iterable of :class:`gcloud.search.document.Document` + :param documents: The documents to add to the index. + + :type client: :class:`gcloud.search.client.Client` + :param client: Client which holds credentials and project + configuration for the index. + """ + pass + + def query(self): + """Execute a query over this index.""" + pass diff --git a/gcloud/search/test_client.py b/gcloud/search/test_client.py new file mode 100644 index 000000000000..8b546ea8ef0e --- /dev/null +++ b/gcloud/search/test_client.py @@ -0,0 +1,136 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest2 + + +class TestClient(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.search.client import Client + return Client + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor_explicit(self): + from gcloud.search import SCOPE + from gcloud.search.connection import Connection + + PROJECT = 'PROJECT' + CREDS = _Credentials() + + client_obj = self._makeOne(project=PROJECT, credentials=CREDS) + + self.assertEqual(client_obj.project, PROJECT) + self.assertTrue(isinstance(client_obj.connection, Connection)) + self.assertTrue(client_obj.connection._credentials is CREDS) + self.assertEqual(CREDS._scopes, SCOPE) + + def test_list_indexes_no_paging(self): + from gcloud.search.index import Index + PROJECT = 'PROJECT' + CREDS = _Credentials() + + CLIENT_OBJ = self._makeOne(project=PROJECT, credentials=CREDS) + INDEX_ID = 'my-index-id' + INDEX_PATH = '/projects/%s/indexes/%s' % (PROJECT, INDEX_ID) + + RETURNED = {'indexes': [{'indexId': INDEX_ID}]} + # Replace the connection on the client with one of our own. + CLIENT_OBJ.connection = _Connection(RETURNED) + + # Execute request. + indexes = list(CLIENT_OBJ.list_indexes()) + # Test values are correct. + self.assertEqual(len(indexes), 1) + self.assertTrue(isinstance(indexes[0], Index)) + self.assertEqual(indexes[0].path, INDEX_PATH) + self.assertEqual(len(CLIENT_OBJ.connection._requested), 1) + req = CLIENT_OBJ.connection._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/projects/%s/indexes' % PROJECT) + self.assertEqual(req['query_params'], {}) + + def test_list_indexes_with_paging(self): + from gcloud.search.index import Index + PROJECT = 'PROJECT' + CREDS = _Credentials() + + CLIENT_OBJ = self._makeOne(project=PROJECT, credentials=CREDS) + + INDEX_ID1 = 'index-id-1' + INDEX_ID2 = 'index-id-2' + TOKEN = 'TOKEN' + SIZE = 1 + RETURNED = [{'indexes': [{'indexId': INDEX_ID1}], + 'nextPageToken': TOKEN}, + {'indexes': [{'indexId': INDEX_ID2}]}] + # Replace the connection on the client with one of our own. + CLIENT_OBJ.connection = _Connection(*RETURNED) + + # Execute request. + indexes = list(CLIENT_OBJ.list_indexes(page_size=SIZE)) + # Test values are correct. + self.assertEqual(len(indexes), 2) + self.assertTrue(isinstance(indexes[0], Index)) + self.assertEqual(indexes[0].index_id, INDEX_ID1) + self.assertEqual(len(CLIENT_OBJ.connection._requested), 2) + req = CLIENT_OBJ.connection._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/projects/%s/indexes' % PROJECT) + self.assertEqual(req['query_params'], {'pageSize': SIZE}) + req2 = CLIENT_OBJ.connection._requested[1] + self.assertEqual(req2['method'], 'GET') + self.assertEqual(req2['path'], '/projects/%s/indexes' % PROJECT) + self.assertEqual(req2['query_params'], + {'pageSize': SIZE, 'pageToken': TOKEN}) + + def test_index(self): + PROJECT = 'PROJECT' + INDEX_ID = 'index-id' + CREDS = _Credentials() + + client_obj = self._makeOne(project=PROJECT, credentials=CREDS) + new_index = client_obj.index(INDEX_ID) + self.assertEqual(new_index.index_id, INDEX_ID) + self.assertTrue(new_index.client is client_obj) + self.assertEqual(new_index.project, PROJECT) + self.assertEqual(new_index.full_name, + 'projects/%s/indexes/%s' % (PROJECT, INDEX_ID)) + + +class _Credentials(object): + + _scopes = None + + @staticmethod + def create_scoped_required(): + return True + + def create_scoped(self, scope): + self._scopes = scope + return self + + +class _Connection(object): + + def __init__(self, *responses): + self._responses = responses + self._requested = [] + + def api_request(self, **kw): + self._requested.append(kw) + response, self._responses = self._responses[0], self._responses[1:] + return response diff --git a/gcloud/search/test_connection.py b/gcloud/search/test_connection.py new file mode 100644 index 000000000000..4a8618388e4e --- /dev/null +++ b/gcloud/search/test_connection.py @@ -0,0 +1,46 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest2 + + +class TestConnection(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.pubsub.connection import Connection + return Connection + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_build_api_url_no_extra_query_params(self): + conn = self._makeOne() + URI = '/'.join([ + conn.API_BASE_URL, + conn.API_VERSION, + 'foo', + ]) + self.assertEqual(conn.build_api_url('/foo'), URI) + + def test_build_api_url_w_extra_query_params(self): + from six.moves.urllib.parse import parse_qsl + from six.moves.urllib.parse import urlsplit + conn = self._makeOne() + uri = conn.build_api_url('/foo', {'bar': 'baz'}) + scheme, netloc, path, qs, _ = urlsplit(uri) + self.assertEqual('%s://%s' % (scheme, netloc), conn.API_BASE_URL) + self.assertEqual(path, + '/'.join(['', conn.API_VERSION, 'foo'])) + parms = dict(parse_qsl(qs)) + self.assertEqual(parms['bar'], 'baz') diff --git a/gcloud/search/value.py b/gcloud/search/value.py new file mode 100644 index 000000000000..5efc56c328f3 --- /dev/null +++ b/gcloud/search/value.py @@ -0,0 +1,44 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define Cloud Search Values.""" + + +class Value(object): + """Value objects hold search values and tokenization parameters. + + See: + https://cloud.google.com/pubsub/reference/rest/v1beta2/projects/subscriptions + + :type value: string + :param value: the string value + + :type tokenization: string + :param tokenization: the tokenization format for string values. + """ + def __init__(self, value, tokenization=None): + self.value = value + self.tokenization = tokenization + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct a topic given its API representation + + :type resource: dict + :param resource: topic resource representation returned from the API + + :rtype: :class:`gcloud.search.value.Value` + :returns: Value parsed from ``resource``. + """ + return cls(value=resource['value'])