diff --git a/docs/bigtable-client-intro.rst b/docs/bigtable-client-intro.rst new file mode 100644 index 000000000000..6c7e48790f3d --- /dev/null +++ b/docs/bigtable-client-intro.rst @@ -0,0 +1,91 @@ +Base for Everything +=================== + +To use the API, the :class:`Client ` +class defines a high-level interface which handles authorization +and creating other objects: + +.. code:: python + + from gcloud.bigtable.client import Client + client = Client() + +Long-lived Defaults +------------------- + +When creating a :class:`Client `, the +``user_agent`` and ``timeout_seconds`` arguments have sensible +defaults +(:data:`DEFAULT_USER_AGENT ` and +:data:`DEFAULT_TIMEOUT_SECONDS `). +However, you may over-ride them and these will be used throughout all API +requests made with the ``client`` you create. + +Authorization +------------- + +- For an overview of authentication in ``gcloud-python``, + see :doc:`gcloud-auth`. + +- In addition to any authentication configuration, you can also set the + :envvar:`GCLOUD_PROJECT` environment variable for the project you'd like + to interact with. If you are Google App Engine or Google Compute Engine + this will be detected automatically. (Setting this environment + variable is not required, you may instead pass the ``project`` explicitly + when constructing a :class:`Client `). + +- After configuring your environment, create a + :class:`Client ` + + .. code:: + + >>> from gcloud import bigtable + >>> client = bigtable.Client() + + or pass in ``credentials`` and ``project`` explicitly + + .. code:: + + >>> from gcloud import bigtable + >>> client = bigtable.Client(project='my-project', credentials=creds) + +.. tip:: + + Be sure to use the **Project ID**, not the **Project Number**. + +Admin API Access +---------------- + +If you'll be using your client to make `Cluster Admin`_ and `Table Admin`_ +API requests, you'll need to pass the ``admin`` argument: + +.. code:: python + + client = bigtable.Client(admin=True) + +Read-Only Mode +-------------- + +If on the other hand, you only have (or want) read access to the data, +you can pass the ``read_only`` argument: + +.. code:: python + + client = bigtable.Client(read_only=True) + +This will ensure that the +:data:`READ_ONLY_SCOPE ` is used +for API requests (so any accidental requests that would modify data will +fail). + +Next Step +--------- + +After a :class:`Client `, the next highest-level +object is a :class:`Cluster `. You'll need +one before you can interact with tables or data. + +Head next to learn about the :doc:`bigtable-cluster-api`. + +.. _Cluster Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1 +.. _Table Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1 diff --git a/docs/bigtable-client.rst b/docs/bigtable-client.rst new file mode 100644 index 000000000000..b765144a160d --- /dev/null +++ b/docs/bigtable-client.rst @@ -0,0 +1,7 @@ +Client +~~~~~~ + +.. automodule:: gcloud.bigtable.client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-cluster-api.rst b/docs/bigtable-cluster-api.rst new file mode 100644 index 000000000000..bedf581fbf44 --- /dev/null +++ b/docs/bigtable-cluster-api.rst @@ -0,0 +1,179 @@ +Cluster Admin API +================= + +After creating a :class:`Client `, you can +interact with individual clusters, groups of clusters or available +zones for a project. + +List Clusters +------------- + +If you want a comprehensive list of all existing clusters, make a +`ListClusters`_ API request with +:meth:`Client.list_clusters() `: + +.. code:: python + + clusters = client.list_clusters() + +List Zones +---------- + +If you aren't sure which ``zone`` to create a cluster in, find out +which zones your project has access to with a `ListZones`_ API request +with :meth:`Client.list_zones() `: + +.. code:: python + + zones = client.list_zones() + +You can choose a :class:`string ` from among the result to pass to +the :class:`Cluster ` constructor. + +The available zones (as of February 2016) are + +.. code:: python + + >>> zones + [u'asia-east1-b', u'europe-west1-c', u'us-central1-c', u'us-central1-b'] + +Cluster Factory +--------------- + +To create a :class:`Cluster ` object: + +.. code:: python + + cluster = client.cluster(zone, cluster_id, + display_name=display_name, + serve_nodes=3) + +Both ``display_name`` and ``serve_nodes`` are optional. When not provided, +``display_name`` defaults to the ``cluster_id`` value and ``serve_nodes`` +defaults to the minimum allowed: +:data:`DEFAULT_SERVE_NODES `. + +Even if this :class:`Cluster ` already +has been created with the API, you'll want this object to use as a +parent of a :class:`Table ` just as the +:class:`Client ` is used as the parent of +a :class:`Cluster `. + +Create a new Cluster +-------------------- + +After creating the cluster object, make a `CreateCluster`_ API request +with :meth:`create() `: + +.. code:: python + + cluster.display_name = 'My very own cluster' + cluster.create() + +If you would like more than the minimum number of nodes +(:data:`DEFAULT_SERVE_NODES `) +in your cluster: + +.. code:: python + + cluster.serve_nodes = 10 + cluster.create() + +Check on Current Operation +-------------------------- + +.. note:: + + When modifying a cluster (via a `CreateCluster`_, `UpdateCluster`_ or + `UndeleteCluster`_ request), the Bigtable API will return a long-running + `Operation`_. This will be stored on the object after each of + :meth:`create() `, + :meth:`update() ` and + :meth:`undelete() ` are called. + +You can check if a long-running operation (for a +:meth:`create() `, +:meth:`update() ` or +:meth:`undelete() `) has finished +by making a `GetOperation`_ request with +:meth:`Operation.finished() `: + +.. code:: python + + >>> operation = cluster.create() + >>> operation.finished() + True + +.. note:: + + Once an :class:`Operation ` object + has returned :data:`True` from + :meth:`finished() `, the + object should not be re-used. Subsequent calls to + :meth:`finished() ` + will result in a :class:`ValueError `. + +Get metadata for an existing Cluster +------------------------------------ + +After creating the cluster object, make a `GetCluster`_ API request +with :meth:`reload() `: + +.. code:: python + + cluster.reload() + +This will load ``serve_nodes`` and ``display_name`` for the existing +``cluster`` in addition to the ``cluster_id``, ``zone`` and ``project`` +already set on the :class:`Cluster ` object. + +Update an existing Cluster +-------------------------- + +After creating the cluster object, make an `UpdateCluster`_ API request +with :meth:`update() `: + +.. code:: python + + client.display_name = 'New display_name' + cluster.update() + +Delete an existing Cluster +-------------------------- + +Make a `DeleteCluster`_ API request with +:meth:`delete() `: + +.. code:: python + + cluster.delete() + +Undelete a deleted Cluster +-------------------------- + +Make an `UndeleteCluster`_ API request with +:meth:`undelete() `: + +.. code:: python + + cluster.undelete() + +Next Step +--------- + +Now we go down the hierarchy from +:class:`Cluster ` to a +:class:`Table `. + +Head next to learn about the :doc:`bigtable-table-api`. + +.. _Cluster Admin API: https://cloud.google.com/bigtable/docs/creating-cluster +.. _CreateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L66-L68 +.. _GetCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L38-L40 +.. _UpdateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L93-L95 +.. _DeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L109-L111 +.. _ListZones: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L33-L35 +.. _ListClusters: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L44-L46 +.. _GetOperation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L43-L45 +.. _UndeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L126-L128 +.. _Operation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L73-L102 diff --git a/docs/bigtable-cluster.rst b/docs/bigtable-cluster.rst new file mode 100644 index 000000000000..9b88f2059d14 --- /dev/null +++ b/docs/bigtable-cluster.rst @@ -0,0 +1,7 @@ +Cluster +~~~~~~~ + +.. automodule:: gcloud.bigtable.cluster + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-column-family.rst b/docs/bigtable-column-family.rst new file mode 100644 index 000000000000..b51d7a18e4ee --- /dev/null +++ b/docs/bigtable-column-family.rst @@ -0,0 +1,50 @@ +Column Families +=============== + +When creating a +:class:`ColumnFamily `, it is +possible to set garbage collection rules for expired data. + +By setting a rule, cells in the table matching the rule will be deleted +during periodic garbage collection (which executes opportunistically in the +background). + +The types +:class:`MaxAgeGCRule `, +:class:`MaxVersionsGCRule `, +:class:`GarbageCollectionRuleUnion ` and +:class:`GarbageCollectionRuleIntersection ` +can all be used as the optional ``gc_rule`` argument in the +:class:`ColumnFamily ` +constructor. This value is then used in the +:meth:`create() ` and +:meth:`update() ` methods. + +These rules can be nested arbitrarily, with a +:class:`MaxAgeGCRule ` or +:class:`MaxVersionsGCRule ` +at the lowest level of the nesting: + +.. code:: python + + import datetime + + max_age = datetime.timedelta(days=3) + rule1 = MaxAgeGCRule(max_age) + rule2 = MaxVersionsGCRule(1) + + # Make a composite that matches anything older than 3 days **AND** + # with more than 1 version. + rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) + + # Make another composite that matches our previous intersection + # **OR** anything that has more than 3 versions. + rule4 = GarbageCollectionRule(max_num_versions=3) + rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) + +---- + +.. automodule:: gcloud.bigtable.column_family + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-data-api.rst b/docs/bigtable-data-api.rst new file mode 100644 index 000000000000..776fe4a8429f --- /dev/null +++ b/docs/bigtable-data-api.rst @@ -0,0 +1,354 @@ +Data API +======== + +After creating a :class:`Table ` and some +column families, you are ready to store and retrieve data. + +Cells vs. Columns vs. Column Families ++++++++++++++++++++++++++++++++++++++ + +* As explained in the :doc:`table overview `, tables can + have many column families. +* As described below, a table can also have many rows which are + specified by row keys. +* Within a row, data is stored in a cell. A cell simply has a value (as + bytes) and a timestamp. The number of cells in each row can be + different, depending on what was stored in each row. +* Each cell lies in a column (**not** a column family). A column is really + just a more **specific** modifier within a column family. A column + can be present in every column family, in only one or anywhere in between. +* Within a column family there can be many columns. For example within + the column family ``foo`` we could have columns ``bar`` and ``baz``. + These would typically be represented as ``foo:bar`` and ``foo:baz``. + +Modifying Data +++++++++++++++ + +Since data is stored in cells, which are stored in rows, the +:class:`Row ` class is the only class used to +modify (write, update, delete) data in a +:class:`Table `. + +Row Factory +----------- + +To create a :class:`Row ` object + +.. code:: python + + row = table.row(row_key) + +Unlike the previous string values we've used before, the row key must +be ``bytes``. + +Direct vs. Conditional vs. Append +--------------------------------- + +There are three ways to modify data in a table, described by the +`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API +methods. + +* The **direct** way is via `MutateRow`_ which involves simply + adding, overwriting or deleting cells. +* The **conditional** way is via `CheckAndMutateRow`_. This method + first checks if some filter is matched in a a given row, then + applies one of two sets of mutations, depending on if a match + occurred or not. (These mutation sets are called the "true + mutations" and "false mutations".) +* The **append** way is via `ReadModifyWriteRow`_. This simply + appends (as bytes) or increments (as an integer) data in a presumed + existing cell in a row. + +Building Up Mutations +--------------------- + +In all three cases, a set of mutations (or two sets) are built up +on a :class:`Row ` before they are sent of +in a batch via :meth:`commit() `: + +.. code:: python + + row.commit() + +To send **append** mutations in batch, use +:meth:`commit_modifications() `: + +.. code:: python + + row.commit_modifications() + +We have a small set of methods on the :class:`Row ` +to build these mutations up. + +Direct Mutations +---------------- + +Direct mutations can be added via one of four methods + +* :meth:`set_cell() ` allows a + single value to be written to a column + + .. code:: python + + row.set_cell(column_family_id, column, value, + timestamp=timestamp) + + If the ``timestamp`` is omitted, the current time on the Google Cloud + Bigtable server will be used when the cell is stored. + + The value can either by bytes or an integer (which will be converted to + bytes as an unsigned 64-bit integer). + +* :meth:`delete_cell() ` deletes + all cells (i.e. for all timestamps) in a given column + + .. code:: python + + row.delete_cell(column_family_id, column) + + Remember, this only happens in the ``row`` we are using. + + If we only want to delete cells from a limited range of time, a + :class:`TimestampRange ` can + be used + + .. code:: python + + row.delete_cell(column_family_id, column, + time_range=time_range) + +* :meth:`delete_cells() ` does + the same thing as :meth:`delete_cell() ` + but accepts a list of columns in a column family rather than a single one. + + .. code:: python + + row.delete_cells(column_family_id, [column1, column2], + time_range=time_range) + + In addition, if we want to delete cells from every column in a column family, + the special :attr:`ALL_COLUMNS ` value + can be used + + .. code:: python + + row.delete_cells(column_family_id, Row.ALL_COLUMNS, + time_range=time_range) + +* :meth:`delete() ` will delete the entire row + + .. code:: python + + row.delete() + +Conditional Mutations +--------------------- + +Making **conditional** modifications is essentially identical +to **direct** modifications, but we need to specify a filter to match +against in the row: + +.. code:: python + + row = table.row(row_key, filter_=filter_val) + +See the :class:`Row ` class for more information +about acceptable values for ``filter_``. + +The only other difference from **direct** modifications are that each mutation +added must specify a ``state``: will the mutation be applied if the filter +matches or if it fails to match. + +For example: + +.. code:: python + + row.set_cell(column_family_id, column, value, + timestamp=timestamp, state=True) + +will add to the set of true mutations. + +.. note:: + + If ``state`` is passed when no ``filter_`` is set on a + :class:`Row `, adding the mutation will fail. + Similarly, if no ``state`` is passed when a ``filter_`` has been set, + adding the mutation will fail. + +Append Mutations +---------------- + +Append mutations can be added via one of two methods + +* :meth:`append_cell_value() ` + appends a bytes value to an existing cell: + + .. code:: python + + row.append_cell_value(column_family_id, column, bytes_value) + +* :meth:`increment_cell_value() ` + increments an integer value in an existing cell: + + .. code:: python + + row.increment_cell_value(column_family_id, column, int_value) + + Since only bytes are stored in a cell, the cell value is decoded as + an unsigned 64-bit integer before being incremented. (This happens on + the Google Cloud Bigtable server, not in the library.) + +Notice that no timestamp was specified. This is because **append** mutations +operate on the latest value of the specified column. + +If there are no cells in the specified column, then the empty string (bytes +case) or zero (integer case) are the assumed values. + +Starting Fresh +-------------- + +If accumulated mutations need to be dropped, use +:meth:`clear_mutations() ` + +.. code:: python + + row.clear_mutations() + +To clear **append** mutations, use +:meth:`clear_modification_rules() ` + +.. code:: python + + row.clear_modification_rules() + +Reading Data +++++++++++++ + +Read Single Row from a Table +---------------------------- + +To make a `ReadRows`_ API request for a single row key, use +:meth:`Table.read_row() `: + +.. code:: python + + >>> row_data = table.read_row(row_key) + >>> row_data.cells + { + u'fam1': { + b'col1': [ + , + , + ], + b'col2': [ + , + ], + }, + u'fam2': { + b'col3': [ + , + , + , + ], + }, + } + >>> cell = row_data.cells[u'fam1'][b'col1'][0] + >>> cell + + >>> cell.value + b'val1' + >>> cell.timestamp + datetime.datetime(2016, 2, 27, 3, 41, 18, 122823, tzinfo=) + +Rather than returning a :class:`Row `, this method +returns a :class:`PartialRowData ` +instance. This class is used for reading and parsing data rather than for +modifying data (as :class:`Row ` is). + +A filter can also be applied to the + +.. code:: python + + row_data = table.read_row(row_key, filter_=filter_val) + +The allowable ``filter_`` values are the same as those used for a +:class:`Row ` with **conditional** mutations. For +more information, see the +:meth:`Table.read_row() ` documentation. + +Stream Many Rows from a Table +----------------------------- + +To make a `ReadRows`_ API request for a stream of rows, use +:meth:`Table.read_rows() `: + +.. code:: python + + row_data = table.read_rows() + +Using gRPC over HTTP/2, a continual stream of responses will be delivered. +In particular + +* :meth:`consume_next() ` + pulls the next result from the stream, parses it and stores it on the + :class:`PartialRowsData ` instance +* :meth:`consume_all() ` + pulls results from the stream until there are no more +* :meth:`cancel() ` closes + the stream + +See the :class:`PartialRowsData ` +documentation for more information. + +As with +:meth:`Table.read_row() `, an optional +``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` +can be supplied for the stream, a ``limit`` can be set and a boolean +``allow_row_interleaving`` can be specified to allow faster streamed results +at the potential cost of non-sequential reads. + +See the :meth:`Table.read_rows() ` +documentation for more information on the optional arguments. + +Sample Keys in a Table +---------------------- + +Make a `SampleRowKeys`_ API request with +:meth:`Table.sample_row_keys() `: + +.. code:: python + + keys_iterator = table.sample_row_keys() + +The returned row keys will delimit contiguous sections of the table of +approximately equal size, which can be used to break up the data for +distributed tasks like mapreduces. + +As with +:meth:`Table.read_rows() `, the +returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. + +The next key in the result can be accessed via + +.. code:: python + + next_key = keys_iterator.next() + +or all keys can be iterated over via + +.. code:: python + + for curr_key in keys_iterator: + do_something(curr_key) + +Just as with reading, the stream can be canceled: + +.. code:: python + + keys_iterator.cancel() + +.. _ReadRows: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L38 +.. _SampleRowKeys: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L44-L46 +.. _MutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L50-L52 +.. _CheckAndMutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L62-L64 +.. _ReadModifyWriteRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L70-L72 diff --git a/docs/bigtable-row-data.rst b/docs/bigtable-row-data.rst new file mode 100644 index 000000000000..5ec98f932d1d --- /dev/null +++ b/docs/bigtable-row-data.rst @@ -0,0 +1,7 @@ +Row Data +~~~~~~~~ + +.. automodule:: gcloud.bigtable.row_data + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-row.rst b/docs/bigtable-row.rst new file mode 100644 index 000000000000..08ae1d4beaae --- /dev/null +++ b/docs/bigtable-row.rst @@ -0,0 +1,66 @@ +Bigtable Row +============ + +It is possible to use a :class:`RowFilter ` +when adding mutations to a :class:`Row ` and when +reading row data with :meth:`read_row() ` +:meth:`read_rows() `. + +As laid out in the `RowFilter definition`_, the following basic filters +are provided: + +* :class:`SinkFilter <.row.SinkFilter>` +* :class:`PassAllFilter <.row.PassAllFilter>` +* :class:`BlockAllFilter <.row.BlockAllFilter>` +* :class:`RowKeyRegexFilter <.row.RowKeyRegexFilter>` +* :class:`RowSampleFilter <.row.RowSampleFilter>` +* :class:`FamilyNameRegexFilter <.row.FamilyNameRegexFilter>` +* :class:`ColumnQualifierRegexFilter <.row.ColumnQualifierRegexFilter>` +* :class:`TimestampRangeFilter <.row.TimestampRangeFilter>` +* :class:`ColumnRangeFilter <.row.ColumnRangeFilter>` +* :class:`ValueRegexFilter <.row.ValueRegexFilter>` +* :class:`ValueRangeFilter <.row.ValueRangeFilter>` +* :class:`CellsRowOffsetFilter <.row.CellsRowOffsetFilter>` +* :class:`CellsRowLimitFilter <.row.CellsRowLimitFilter>` +* :class:`CellsColumnLimitFilter <.row.CellsColumnLimitFilter>` +* :class:`StripValueTransformerFilter <.row.StripValueTransformerFilter>` +* :class:`ApplyLabelFilter <.row.ApplyLabelFilter>` + +In addition, these filters can be combined into composite filters with + +* :class:`RowFilterChain <.row.RowFilterChain>` +* :class:`RowFilterUnion <.row.RowFilterUnion>` +* :class:`ConditionalRowFilter <.row.ConditionalRowFilter>` + +These rules can be nested arbitrarily, with a basic filter at the lowest +level. For example: + +.. code:: python + + # Filter in a specified column (matching any column family). + col1_filter = ColumnQualifierRegexFilter(b'columnbia') + + # Create a filter to label results. + label1 = u'label-red' + label1_filter = ApplyLabelFilter(label1) + + # Combine the filters to label all the cells in columnbia. + chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) + + # Create a similar filter to label cells blue. + col2_filter = ColumnQualifierRegexFilter(b'columnseeya') + label2 = u'label-blue' + label2_filter = ApplyLabelFilter(label2) + chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) + + # Bring our two labeled columns together. + row_filter = RowFilterUnion(filters=[chain1, chain2]) + +---- + +.. automodule:: gcloud.bigtable.row + :members: + :undoc-members: + :show-inheritance: + +.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/bigtable-table-api.rst b/docs/bigtable-table-api.rst new file mode 100644 index 000000000000..b3108da75a1b --- /dev/null +++ b/docs/bigtable-table-api.rst @@ -0,0 +1,165 @@ +Table Admin API +=============== + +After creating a :class:`Cluster `, you can +interact with individual tables, groups of tables or column families within +a table. + +List Tables +----------- + +If you want a comprehensive list of all existing tables in a cluster, make a +`ListTables`_ API request with +:meth:`Cluster.list_tables() `: + +.. code:: python + + >>> cluster.list_tables() + [, + ] + +Table Factory +------------- + +To create a :class:`Table ` object: + +.. code:: python + + table = cluster.table(table_id) + +Even if this :class:`Table ` already +has been created with the API, you'll want this object to use as a +parent of a :class:`ColumnFamily ` +or :class:`Row `. + +Create a new Table +------------------ + +After creating the table object, make a `CreateTable`_ API request +with :meth:`create() `: + +.. code:: python + + table.create() + +If you would to initially split the table into several tablets (Tablets are +similar to HBase regions): + +.. code:: python + + table.create(initial_split_keys=['s1', 's2']) + +Delete an existing Table +------------------------ + +Make a `DeleteTable`_ API request with +:meth:`delete() `: + +.. code:: python + + table.delete() + +Rename an existing Table +------------------------ + +Though the `RenameTable`_ API request is listed in the service +definition, requests to that method return:: + + BigtableTableService.RenameTable is not yet implemented + +We have implemented :meth:`rename() ` +but it will not work unless the backend supports the method. + +List Column Families in a Table +------------------------------- + +Though there is no **official** method for retrieving `column families`_ +associated with a table, the `GetTable`_ API method returns a +table object with the names of the column families. + +To retrieve the list of column families use +:meth:`list_column_families() `: + +.. code:: python + + column_families = table.list_column_families() + +Column Family Factory +--------------------- + +To create a +:class:`ColumnFamily ` object: + +.. code:: python + + column_family = table.column_family(column_family_id) + +There is no real reason to use this factory unless you intend to +create or delete a column family. + +In addition, you can specify an optional ``gc_rule`` (a +:class:`GarbageCollectionRule ` +or similar): + +.. code:: python + + column_family = table.column_family(column_family_id, + gc_rule=gc_rule) + +This rule helps the backend determine when and how to clean up old cells +in the column family. + +See :doc:`bigtable-column-family` for more information about +:class:`GarbageCollectionRule ` +and related classes. + +Create a new Column Family +-------------------------- + +After creating the column family object, make a `CreateColumnFamily`_ API +request with +:meth:`ColumnFamily.create() ` + +.. code:: python + + column_family.create() + +Delete an existing Column Family +-------------------------------- + +Make a `DeleteColumnFamily`_ API request with +:meth:`ColumnFamily.delete() ` + +.. code:: python + + column_family.delete() + +Update an existing Column Family +-------------------------------- + +Make an `UpdateColumnFamily`_ API request with +:meth:`ColumnFamily.delete() ` + +.. code:: python + + column_family.update() + +Next Step +--------- + +Now we go down the final step of the hierarchy from +:class:`Table ` to +:class:`Row ` as well as streaming +data directly via a :class:`Table `. + +Head next to learn about the :doc:`bigtable-data-api`. + +.. _ListTables: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L40-L42 +.. _CreateTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L35-L37 +.. _DeleteTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L50-L52 +.. _RenameTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L56-L58 +.. _GetTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L45-L47 +.. _CreateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L61-L63 +.. _UpdateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L66-L68 +.. _DeleteColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L71-L73 +.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers diff --git a/docs/bigtable-table.rst b/docs/bigtable-table.rst new file mode 100644 index 000000000000..03ca332f9c9a --- /dev/null +++ b/docs/bigtable-table.rst @@ -0,0 +1,7 @@ +Table +~~~~~ + +.. automodule:: gcloud.bigtable.table + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-usage.rst b/docs/bigtable-usage.rst new file mode 100644 index 000000000000..95faba854e69 --- /dev/null +++ b/docs/bigtable-usage.rst @@ -0,0 +1,25 @@ +Using the API +============= + +API requests are sent to the `Google Cloud Bigtable`_ API via RPC over HTTP/2. +In order to support this, we'll rely on `gRPC`_. We are working with the gRPC +team to rapidly make the install story more user-friendly. + +Get started by learning about the +:class:`Client ` on the +:doc:`bigtable-client-intro` page. + +In the hierarchy of API concepts + +* a :class:`Client ` owns a + :class:`Cluster ` +* a :class:`Cluster ` owns a + :class:`Table ` +* a :class:`Table ` owns a + :class:`ColumnFamily ` +* a :class:`Table ` owns a + :class:`Row ` + (and all the cells in the row) + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ +.. _gRPC: http://www.grpc.io/ diff --git a/docs/happybase-batch.rst b/docs/happybase-batch.rst new file mode 100644 index 000000000000..c1fc86b9d6e0 --- /dev/null +++ b/docs/happybase-batch.rst @@ -0,0 +1,7 @@ +HappyBase Batch +~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.batch + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-connection.rst b/docs/happybase-connection.rst new file mode 100644 index 000000000000..01485bbdbde0 --- /dev/null +++ b/docs/happybase-connection.rst @@ -0,0 +1,7 @@ +HappyBase Connection +~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-package.rst b/docs/happybase-package.rst new file mode 100644 index 000000000000..22e6134f0fa5 --- /dev/null +++ b/docs/happybase-package.rst @@ -0,0 +1,7 @@ +HappyBase Package +~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.__init__ + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-pool.rst b/docs/happybase-pool.rst new file mode 100644 index 000000000000..9390fd41c01d --- /dev/null +++ b/docs/happybase-pool.rst @@ -0,0 +1,7 @@ +HappyBase Connection Pool +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.pool + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-table.rst b/docs/happybase-table.rst new file mode 100644 index 000000000000..b5f477d8058d --- /dev/null +++ b/docs/happybase-table.rst @@ -0,0 +1,7 @@ +HappyBase Table +~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.table + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index c08588dfa8b1..84afe9522fc8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,6 +53,28 @@ bigquery-table bigquery-query +.. toctree:: + :maxdepth: 0 + :hidden: + :caption: Cloud Bigtable + + bigtable-usage + HappyBase + bigtable-client-intro + bigtable-cluster-api + bigtable-table-api + bigtable-data-api + Client + bigtable-cluster + bigtable-table + bigtable-column-family + bigtable-row + bigtable-row-data + happybase-connection + happybase-pool + happybase-table + happybase-batch + .. toctree:: :maxdepth: 0 :hidden: diff --git a/gcloud/bigtable/client.py b/gcloud/bigtable/client.py index 2913feaa68c3..c67ce2b7057f 100644 --- a/gcloud/bigtable/client.py +++ b/gcloud/bigtable/client.py @@ -27,6 +27,8 @@ """ +from pkg_resources import get_distribution + from grpc.beta import implementations from gcloud.bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 @@ -75,7 +77,8 @@ DEFAULT_TIMEOUT_SECONDS = 10 """The default timeout to use for API requests.""" -DEFAULT_USER_AGENT = 'gcloud-bigtable-python' +DEFAULT_USER_AGENT = 'gcloud-python/{0}'.format( + get_distribution('gcloud').version) """The default user agent for API requests.""" diff --git a/gcloud/bigtable/cluster.py b/gcloud/bigtable/cluster.py index c04175a3a16c..95be153ad474 100644 --- a/gcloud/bigtable/cluster.py +++ b/gcloud/bigtable/cluster.py @@ -34,7 +34,6 @@ _OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/zones/([^/]+)/' r'clusters/([a-z][-a-z0-9]*)/operations/' r'(?P\d+)$') -_DEFAULT_SERVE_NODES = 3 _TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' _ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' _CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' @@ -46,6 +45,9 @@ _UNDELETE_CREATE_METADATA: messages_pb2.UndeleteClusterMetadata, } +DEFAULT_SERVE_NODES = 3 +"""Default number of nodes to use when creating a cluster.""" + def _prepare_create_request(cluster): """Creates a protobuf request for a CreateCluster request. @@ -204,7 +206,7 @@ class Cluster(object): :type cluster_id: str :param cluster_id: The ID of the cluster. - :type client: :class:`.client.Client` + :type client: :class:`Client ` :param client: The client that owns the cluster. Provides authorization and a project ID. @@ -216,11 +218,11 @@ class Cluster(object): :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3 (``_DEFAULT_SERVE_NODES``). + Defaults to :data:`DEFAULT_SERVE_NODES`. """ def __init__(self, zone, cluster_id, client, - display_name=None, serve_nodes=_DEFAULT_SERVE_NODES): + display_name=None, serve_nodes=DEFAULT_SERVE_NODES): self.zone = zone self.cluster_id = cluster_id self.display_name = display_name or cluster_id @@ -253,14 +255,16 @@ def from_pb(cls, cluster_pb, client): :type cluster_pb: :class:`bigtable_cluster_data_pb2.Cluster` :param cluster_pb: A cluster protobuf object. - :type client: :class:`.client.Client` + :type client: :class:`Client ` :param client: The client that owns the cluster. :rtype: :class:`Cluster` :returns: The cluster parsed from the protobuf response. :raises: :class:`ValueError ` if the cluster - name does not match :data:`_CLUSTER_NAME_RE` or if the parsed - project ID does not match the project ID on the client. + name does not match + ``projects/{project}/zones/{zone}/clusters/{cluster_id}`` + or if the parsed project ID does not match the project ID + on the client. """ match = _CLUSTER_NAME_RE.match(cluster_pb.name) if match is None: @@ -277,9 +281,8 @@ def from_pb(cls, cluster_pb, client): def copy(self): """Make a copy of this cluster. - Copies the local data stored as simple types but does not copy the - current state of any operations with the Cloud Bigtable API. Also - copies the client attached to this instance. + Copies the local data stored as simple types and copies the client + attached to this instance. :rtype: :class:`.Cluster` :returns: A copy of the current cluster. diff --git a/gcloud/bigtable/column_family.py b/gcloud/bigtable/column_family.py index 0d3aeda8c508..c0d9060316a4 100644 --- a/gcloud/bigtable/column_family.py +++ b/gcloud/bigtable/column_family.py @@ -157,7 +157,7 @@ def __eq__(self, other): return other.rules == self.rules def to_pb(self): - """Converts the union into a single gc rule as a protobuf. + """Converts the union into a single GC rule as a protobuf. :rtype: :class:`.data_pb2.GcRule` :returns: The converted current object. @@ -183,7 +183,7 @@ def __eq__(self, other): return other.rules == self.rules def to_pb(self): - """Converts the intersection into a single gc rule as a protobuf. + """Converts the intersection into a single GC rule as a protobuf. :rtype: :class:`.data_pb2.GcRule` :returns: The converted current object. diff --git a/gcloud/bigtable/happybase/__init__.py b/gcloud/bigtable/happybase/__init__.py index 4c5b52f80924..03e4d9215ff1 100644 --- a/gcloud/bigtable/happybase/__init__.py +++ b/gcloud/bigtable/happybase/__init__.py @@ -24,10 +24,18 @@ Bigtable API. As a result, the following instance methods and functions could not be implemented: -* :meth:`.Connection.enable_table` - no concept of enabled/disabled -* :meth:`.Connection.disable_table` - no concept of enabled/disabled -* :meth:`.Connection.is_table_enabled` - no concept of enabled/disabled -* :meth:`.Connection.compact_table` - table storage is opaque to user +* :meth:`Connection.enable_table() \ + ` - no + concept of enabled/disabled +* :meth:`Connection.disable_table() \ + ` - no + concept of enabled/disabled +* :meth:`Connection.is_table_enabled() \ + ` + - no concept of enabled/disabled +* :meth:`Connection.compact_table() \ + ` - + table storage is opaque to user * :func:`make_row() ` - helper needed for Thrift library * :func:`make_ordered_row() ` @@ -41,8 +49,9 @@ However, it's worth nothing this implementation was based off HappyBase 0.9. -In addition, many of the constants from :mod:`.connection` are specific -to HBase and are defined as :data:`None` in our module: +In addition, many of the constants from +:mod:`connection ` +are specific to HBase and are defined as :data:`None` in our module: * ``COMPAT_MODES`` * ``THRIFT_TRANSPORTS`` @@ -63,9 +72,12 @@ -------------------- * Since there is no concept of an enabled / disabled table, calling - :meth:`.Connection.delete_table` with ``disable=True`` can't be supported. + :meth:`Connection.delete_table() \ + ` + with ``disable=True`` can't be supported. Using that argument will result in a warning. -* The :class:`.Connection` constructor **disables** the use of several +* The :class:`Connection ` + constructor **disables** the use of several arguments and will print a warning if any of them are passed in as keyword arguments. The arguments are: @@ -74,9 +86,12 @@ * ``compat`` * ``transport`` * ``protocol`` -* In order to make :class:`.Connection` compatible with Cloud Bigtable, we - add a ``cluster`` keyword argument to allow user's to pass in their own - :class:`.Cluster` (which they can construct beforehand). +* In order to make + :class:`Connection ` + compatible with Cloud Bigtable, we add a ``cluster`` keyword argument to + allow users to pass in their own + :class:`Cluster ` (which they can + construct beforehand). For example: @@ -93,14 +108,16 @@ * Any uses of the ``wal`` (Write Ahead Log) argument will result in a warning as well. This includes uses in: - * :class:`.Batch` constructor - * :meth:`.Batch.put` - * :meth:`.Batch.delete` + * :class:`Batch ` + * :meth:`Batch.put() ` + * :meth:`Batch.delete() ` * :meth:`Table.put() ` * :meth:`Table.delete() ` * :meth:`Table.batch() ` factory -* When calling :meth:`.Connection.create_table`, the majority of HBase column - family options cannot be used. Among +* When calling + :meth:`Connection.create_table() \ + `, the + majority of HBase column family options cannot be used. Among * ``max_versions`` * ``compression`` @@ -113,9 +130,9 @@ Only ``max_versions`` and ``time_to_live`` are availabe in Cloud Bigtable (as - :class:`MaxVersionsGCRule ` - and - `MaxAgeGCRule `). + :class:`MaxVersionsGCRule ` + and + :class:`MaxAgeGCRule `). In addition to using a dictionary for specifying column family options, we also accept instances of :class:`.GarbageCollectionRule` or subclasses. @@ -131,7 +148,8 @@ not possible with Cloud Bigtable and will result in a :class:`TypeError `. However, the method now accepts instances of :class:`.RowFilter` and subclasses. -* :meth:`.Batch.delete` (and hence +* :meth:`Batch.delete() ` (and + hence :meth:`Table.delete() `) will fail with a :class:`ValueError ` when either a row or column family delete is attempted with a ``timestamp``. This is diff --git a/gcloud/bigtable/happybase/batch.py b/gcloud/bigtable/happybase/batch.py index 310b0ae582cc..663b7f892c77 100644 --- a/gcloud/bigtable/happybase/batch.py +++ b/gcloud/bigtable/happybase/batch.py @@ -179,7 +179,7 @@ def _delete_columns(self, columns, row_object): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type row_object: :class:`Row ` :param row_object: The row which will hold the delete mutations. @@ -213,7 +213,7 @@ def delete(self, row, columns=None, wal=_WAL_SENTINEL): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` If not used, will delete the entire row. @@ -223,7 +223,7 @@ def delete(self, row, columns=None, wal=_WAL_SENTINEL): irrelevant for Cloud Bigtable since it does not have a Write Ahead Log. - :raises: If if the delete timestamp range is set on the + :raises: If the delete timestamp range is set on the current batch, but a full row delete is attempted. """ if wal is not _WAL_SENTINEL: @@ -287,7 +287,7 @@ def _get_column_pairs(columns, require_qualifier=False): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type require_qualifier: bool :param require_qualifier: Boolean indicating if the columns should diff --git a/gcloud/bigtable/happybase/connection.py b/gcloud/bigtable/happybase/connection.py index b3abad5da841..c6ba0b2f8f96 100644 --- a/gcloud/bigtable/happybase/connection.py +++ b/gcloud/bigtable/happybase/connection.py @@ -50,7 +50,8 @@ def _get_cluster(timeout=None): """Gets cluster for the default project. Creates a client with the inferred credentials and project ID from - the local environment. Then uses :meth:`.Client.list_clusters` to + the local environment. Then uses + :meth:`.bigtable.client.Client.list_clusters` to get the unique cluster owned by the project. If the request fails for any reason, or if there isn't exactly one cluster @@ -94,7 +95,8 @@ class Connection(object): If you pass a ``cluster``, it will be :meth:`.Cluster.copy`-ed before being stored on the new connection. This also copies the - :class:`.Client` that created the :class:`.Cluster` instance and the + :class:`Client ` that created the + :class:`Cluster ` instance and the :class:`Credentials ` stored on the client. @@ -117,12 +119,13 @@ class Connection(object): :param table_prefix_separator: (Optional) Separator used with ``table_prefix``. Defaults to ``_``. - :type cluster: :class:`gcloud.bigtable.cluster.Cluster` + :type cluster: :class:`Cluster ` :param cluster: (Optional) A Cloud Bigtable cluster. The instance also owns a client for making gRPC requests to the Cloud Bigtable API. If not passed in, defaults to creating client with ``admin=True`` and using the ``timeout`` here for the - ``timeout_seconds`` argument to the :class:`.Client`` + ``timeout_seconds`` argument to the + :class:`Client ` constructor. The credentials for the client will be the implicit ones loaded from the environment. Then that client is used to retrieve all the clusters @@ -191,7 +194,8 @@ def open(self): """Open the underlying transport to Cloud Bigtable. This method opens the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by + :class:`Client ` bound to the + :class:`Cluster ` owned by this connection. """ self._cluster._client.start() @@ -200,7 +204,8 @@ def close(self): """Close the underlying transport to Cloud Bigtable. This method closes the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by + :class:`Client ` bound to the + :class:`Cluster ` owned by this connection. """ self._cluster._client.stop() @@ -233,7 +238,7 @@ def table(self, name, use_prefix=True): :type use_prefix: bool :param use_prefix: Whether to use the table prefix (if any). - :rtype: `Table ` + :rtype: :class:`Table ` :returns: Table instance owned by this connection. """ if use_prefix: @@ -362,8 +367,10 @@ def delete_table(self, name, disable=False): def enable_table(self, name): """Enable the specified table. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -374,8 +381,10 @@ def enable_table(self, name): def disable_table(self, name): """Disable the specified table. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -386,8 +395,10 @@ def disable_table(self, name): def is_table_enabled(self, name): """Return whether the specified table is enabled. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -398,8 +409,10 @@ def is_table_enabled(self, name): def compact_table(self, name, major=False): """Compact the specified table. - Cloud Bigtable does not support compacting a table, so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable does not support compacting a table, so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always diff --git a/gcloud/bigtable/happybase/pool.py b/gcloud/bigtable/happybase/pool.py index e2d897155adb..ab84724740a2 100644 --- a/gcloud/bigtable/happybase/pool.py +++ b/gcloud/bigtable/happybase/pool.py @@ -42,17 +42,19 @@ class ConnectionPool(object): .. note:: All keyword arguments are passed unmodified to the - :class:`.Connection` constructor **except** for ``autoconnect``. - This is because the ``open`` / ``closed`` status of a connection - is managed by the pool. In addition, if ``cluster`` is not passed, - the default / inferred cluster is determined by the pool and then - passed to each :class:`.Connection` that is created. + :class:`Connection <.happybase.connection.Connection>` constructor + **except** for ``autoconnect``. This is because the ``open`` / + ``closed`` status of a connection is managed by the pool. In addition, + if ``cluster`` is not passed, the default / inferred cluster is + determined by the pool and then passed to each + :class:`Connection <.happybase.connection.Connection>` that is created. :type size: int :param size: The maximum number of concurrently open connections. :type kwargs: dict - :param kwargs: Keyword arguments passed to :class:`.Connection` + :param kwargs: Keyword arguments passed to + :class:`Connection <.happybase.Connection>` constructor. :raises: :class:`TypeError ` if ``size`` @@ -88,7 +90,7 @@ def _acquire_connection(self, timeout=None): :param timeout: (Optional) Time (in seconds) to wait for a connection to open. - :rtype: :class:`.Connection` + :rtype: :class:`Connection <.happybase.Connection>` :returns: An active connection from the queue stored on the pool. :raises: :class:`NoConnectionsAvailable` if ``Queue.get`` fails before the ``timeout`` (only if a timeout is specified). @@ -109,13 +111,13 @@ def connection(self, timeout=None): pass # do something with the connection If ``timeout`` is omitted, this method waits forever for a connection - to become available. + to become available from the local queue. :type timeout: int :param timeout: (Optional) Time (in seconds) to wait for a connection to open. - :rtype: :class:`.Connection` + :rtype: :class:`Connection <.happybase.connection.Connection>` :returns: An active connection from the pool. :raises: :class:`NoConnectionsAvailable` if no connection can be retrieved from the pool before the ``timeout`` (only if diff --git a/gcloud/bigtable/happybase/table.py b/gcloud/bigtable/happybase/table.py index 342fb04a15ff..39fea0648d4a 100644 --- a/gcloud/bigtable/happybase/table.py +++ b/gcloud/bigtable/happybase/table.py @@ -49,7 +49,7 @@ def make_row(cell_map, include_timestamp): """Make a row dict for a Thrift cell mapping. - .. note:: + .. warning:: This method is only provided for HappyBase compatibility, but does not actually work. @@ -74,7 +74,7 @@ def make_row(cell_map, include_timestamp): def make_ordered_row(sorted_columns, include_timestamp): """Make a row dict for sorted Thrift column results from scans. - .. note:: + .. warning:: This method is only provided for HappyBase compatibility, but does not actually work. @@ -103,7 +103,7 @@ class Table(object): :type name: str :param name: The name of the table. - :type connection: :class:`.Connection` + :type connection: :class:`Connection <.happybase.connection.Connection>` :param connection: The connection which has access to the table. """ @@ -136,9 +136,11 @@ def families(self): def regions(self): """Retrieve the regions for this table. - Cloud Bigtable does not give information about how a table is laid - out in memory, so regions so this method does not work. It is - provided simply for compatibility. + .. warning:: + + Cloud Bigtable does not give information about how a table is laid + out in memory, so this method does not work. It is + provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -161,7 +163,7 @@ def row(self, row, columns=None, timestamp=None, include_timestamp=False): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -206,7 +208,7 @@ def rows(self, rows, columns=None, timestamp=None, strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -319,9 +321,9 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, results to retrieve per request. The HBase scanner defaults to reading one record at a time, so this argument allows HappyBase to increase that number. However, the Cloud Bigtable API uses HTTP/2 streaming so - there is no concept of a batch. The ``sorted_columns`` flag tells - HBase to return columns in order, but Cloud Bigtable doesn't have - this feature.) + there is no concept of a batched scan. The ``sorted_columns`` flag + tells HBase to return columns in order, but Cloud Bigtable doesn't + have this feature.) :type row_start: str :param row_start: (Optional) Row key where the scanner should start @@ -344,9 +346,9 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` - :type filter: :class:`.RowFilter` + :type filter: :class:`RowFilter ` :param filter: (Optional) An additional filter (beyond column and row range filters supported here). HappyBase / HBase users will have used this as an HBase filter string. See @@ -369,7 +371,7 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, :param kwargs: Remaining keyword arguments. Provided for HappyBase compatibility. - :raises: If ``limit`` is set but non-positive, or if row prefix is + :raises: If ``limit`` is set but non-positive, or if ``row_prefix`` is used with row start/stop, :class:`TypeError ` if a string ``filter`` is used. @@ -479,7 +481,7 @@ def delete(self, row, columns=None, timestamp=None, wal=_WAL_SENTINEL): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -498,8 +500,9 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, wal=_WAL_SENTINEL): """Create a new batch operation for this table. - This method returns a new :class:`.Batch` instance that can be used - for mass data manipulation. + This method returns a new + :class:`Batch <.happybase.batch.Batch>` instance that can be + used for mass data manipulation. :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -512,10 +515,11 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, :type transaction: bool :param transaction: Flag indicating if the mutations should be sent transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. + an error occurs while a + :class:`Batch <.happybase.batch.Batch>` is + active, then none of the accumulated mutations will + be committed. If ``batch_size`` is set, the + mutation can't be transactional. :type wal: object :param wal: Unused parameter (to be passed to the created batch). @@ -523,7 +527,7 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, for Cloud Bigtable since it does not have a Write Ahead Log. - :rtype: :class:`gcloud.bigtable.happybase.batch.Batch` + :rtype: :class:`Batch ` :returns: A batch bound to this table. """ return Batch(self, timestamp=timestamp, batch_size=batch_size, @@ -635,10 +639,12 @@ def _gc_rule_to_dict(gc_rule): Only does this if the garbage collection rule is: - * :class:`.MaxAgeGCRule` - * :class:`.MaxVersionsGCRule` - * Composite :class:`.GCRuleIntersection` with two rules, one each - of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` + * :class:`gcloud.bigtable.column_family.MaxAgeGCRule` + * :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` + * Composite :class:`gcloud.bigtable.column_family.GCRuleIntersection` + with two rules, one each of type + :class:`gcloud.bigtable.column_family.MaxAgeGCRule` and + :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` Otherwise, just returns the input without change. @@ -647,7 +653,8 @@ def _gc_rule_to_dict(gc_rule): :param gc_rule: A garbage collection rule to convert to a dictionary (if possible). - :rtype: dict or :class:`.GarbageCollectionRule` + :rtype: dict or + :class:`gcloud.bigtable.column_family.GarbageCollectionRule` :returns: The converted garbage collection rule. """ result = gc_rule @@ -734,7 +741,8 @@ def _convert_to_time_range(timestamp=None): epoch). Intended to be used as the end of an HBase time range, which is exclusive. - :rtype: :class:`.TimestampRange`, :data:`NoneType ` + :rtype: :class:`gcloud.bigtable.row.TimestampRange`, + :data:`NoneType ` :returns: The timestamp range corresponding to the passed in ``timestamp``. """ @@ -760,7 +768,8 @@ def _cells_to_pairs(cells, include_timestamp=False): [(b'val1', 1456361486255), (b'val2', 1456361491927)] :type cells: list - :param cells: List of :class:`.Cell` returned from a read request. + :param cells: List of :class:`gcloud.bigtable.row_data.Cell` returned + from a read request. :type include_timestamp: bool :param include_timestamp: Flag to indicate if cell timestamps should be @@ -844,7 +853,7 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, :type filters: list :param filters: (Optional) List of existing filters to be extended. - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The chained filter created, or just a single filter if only one was needed. :raises: :class:`ValueError ` if there are no @@ -883,9 +892,9 @@ def _columns_filter_helper(columns): name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The union filter created containing all of the matched columns. :raises: :class:`ValueError ` if there are no filters to union. @@ -916,7 +925,7 @@ def _row_keys_filter_helper(row_keys): :type row_keys: list :param row_keys: Iterable containing row keys (as strings). - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The union filter created containing all of the row keys. :raises: :class:`ValueError ` if there are no filters to union. diff --git a/gcloud/bigtable/row.py b/gcloud/bigtable/row.py index d090cb071620..028c697d0c3d 100644 --- a/gcloud/bigtable/row.py +++ b/gcloud/bigtable/row.py @@ -27,9 +27,11 @@ bigtable_service_messages_pb2 as messages_pb2) -_MAX_MUTATIONS = 100000 _PACK_I64 = struct.Struct('>q').pack +MAX_MUTATIONS = 100000 +"""The maximum number of mutations that a row can accumulate.""" + class Row(object): """Representation of a Google Cloud Bigtable Row. @@ -112,8 +114,7 @@ def set_cell(self, column_family_id, column, value, timestamp=None, The cell is determined by the ``row_key`` of the :class:`Row` and the ``column``. The ``column`` must be in an existing - :class:`.column_family.ColumnFamily` (as determined by - ``column_family_id``). + :class:`.ColumnFamily` (as determined by ``column_family_id``). .. note:: @@ -352,15 +353,15 @@ def _commit_mutate(self): by :meth:`commit`. :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. + mutations exceeds the :data:`MAX_MUTATIONS`. """ mutations_list = self._get_mutations() num_mutations = len(mutations_list) if num_mutations == 0: return - if num_mutations > _MAX_MUTATIONS: + if num_mutations > MAX_MUTATIONS: raise ValueError('%d total mutations exceed the maximum allowable ' - '%d.' % (num_mutations, _MAX_MUTATIONS)) + '%d.' % (num_mutations, MAX_MUTATIONS)) request_pb = messages_pb2.MutateRowRequest( table_name=self._table.name, row_key=self._row_key, @@ -380,7 +381,7 @@ def _commit_check_and_mutate(self): :returns: Flag indicating if the filter was matched (which also indicates which set of mutations were applied by the server). :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. + mutations exceeds the :data:`MAX_MUTATIONS`. """ true_mutations = self._get_mutations(state=True) false_mutations = self._get_mutations(state=False) @@ -388,12 +389,12 @@ def _commit_check_and_mutate(self): num_false_mutations = len(false_mutations) if num_true_mutations == 0 and num_false_mutations == 0: return - if (num_true_mutations > _MAX_MUTATIONS or - num_false_mutations > _MAX_MUTATIONS): + if (num_true_mutations > MAX_MUTATIONS or + num_false_mutations > MAX_MUTATIONS): raise ValueError( 'Exceed the maximum allowable mutations (%d). Had %s true ' 'mutations and %d false mutations.' % ( - _MAX_MUTATIONS, num_true_mutations, num_false_mutations)) + MAX_MUTATIONS, num_true_mutations, num_false_mutations)) request_pb = messages_pb2.CheckAndMutateRowRequest( table_name=self._table.name, @@ -438,7 +439,7 @@ def commit(self): indicating if the filter was matched (which also indicates which set of mutations were applied by the server). :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. + mutations exceeds the :data:`MAX_MUTATIONS`. """ if self._filter is None: result = self._commit_mutate() @@ -467,31 +468,33 @@ def commit_modifications(self): or the highest timestamp of a cell in that column (if it exceeds the server time). + .. code:: python + + >>> row.commit_modifications() + { + u'col-fam-id': { + b'col-name1': [ + (b'cell-val', datetime.datetime(...)), + (b'cell-val-newer', datetime.datetime(...)), + ], + b'col-name2': [ + (b'altcol-cell-val', datetime.datetime(...)), + ], + }, + u'col-fam-id2': { + b'col-name3-but-other-fam': [ + (b'foo', datetime.datetime(...)), + ], + }, + } + :rtype: dict :returns: The new contents of all modified cells. Returned as a dictionary of column families, each of which holds a dictionary of columns. Each column contains a list of cells modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } + value (in bytes) and the timestamp for the cell. + """ if len(self._rule_pb_list) == 0: return {} @@ -1099,14 +1102,14 @@ class ApplyLabelFilter(RowFilter): """Filter to apply labels to cells. Intended to be used as an intermediate filter on a pre-existing filtered - result set. This was if two sets are combined, the label can tell where + result set. This way if two sets are combined, the label can tell where the cell(s) originated.This allows the client to determine which results were produced from which part of the filter. .. note:: - Due to a technical limitation, it is not currently possible to apply - multiple labels to a cell. + Due to a technical limitation of the backend, it is not currently + possible to apply multiple labels to a cell. :type label: str :param label: Label to apply to cells in the output row. Values must be diff --git a/gcloud/bigtable/table.py b/gcloud/bigtable/table.py index 2abee12bc9d0..81e967218a00 100644 --- a/gcloud/bigtable/table.py +++ b/gcloud/bigtable/table.py @@ -53,7 +53,7 @@ class Table(object): :type table_id: str :param table_id: The ID of the table. - :type cluster: :class:`.cluster.Cluster` + :type cluster: :class:`Cluster <.cluster.Cluster>` :param cluster: The cluster that owns the table. """ @@ -86,11 +86,11 @@ def column_family(self, column_family_id, gc_rule=None): :param column_family_id: The ID of the column family. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type gc_rule: :class:`.column_family.GarbageCollectionRule` + :type gc_rule: :class:`.GarbageCollectionRule` :param gc_rule: (Optional) The garbage collection settings for this column family. - :rtype: :class:`.column_family.ColumnFamily` + :rtype: :class:`.ColumnFamily` :returns: A column family owned by this table. """ return ColumnFamily(column_family_id, self, gc_rule=gc_rule) @@ -199,7 +199,7 @@ def list_column_families(self): :rtype: dict :returns: Dictionary of column families attached to this table. Keys are strings (column family names) and values are - :class:`.column_family.ColumnFamily` instances. + :class:`.ColumnFamily` instances. :raises: :class:`ValueError ` if the column family name from the response does not agree with the computed name from the column family ID. @@ -228,7 +228,7 @@ def read_row(self, row_key, filter_=None): :type row_key: bytes :param row_key: The key of the row to read from. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. @@ -291,7 +291,7 @@ def read_rows(self, start_key=None, end_key=None, more than N rows. However, only N ``commit_row`` chunks will be sent. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads every column in each row. @@ -368,7 +368,7 @@ def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. diff --git a/gcloud/bigtable/test_cluster.py b/gcloud/bigtable/test_cluster.py index eba227a72db0..427a4ec9126b 100644 --- a/gcloud/bigtable/test_cluster.py +++ b/gcloud/bigtable/test_cluster.py @@ -210,7 +210,7 @@ def test_table_factory(self): def test__update_from_pb_success(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES display_name = 'display_name' serve_nodes = 8 @@ -221,7 +221,7 @@ def test__update_from_pb_success(self): cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, display_name) self.assertEqual(cluster.serve_nodes, serve_nodes) @@ -229,30 +229,30 @@ def test__update_from_pb_success(self): def test__update_from_pb_no_display_name(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES cluster_pb = data_pb2.Cluster(serve_nodes=331) cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) with self.assertRaises(ValueError): cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) def test__update_from_pb_no_serve_nodes(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES cluster_pb = data_pb2.Cluster(display_name='name') cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) with self.assertRaises(ValueError): cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) def test_from_pb_success(self): from gcloud.bigtable._generated import ( @@ -353,7 +353,7 @@ def test_reload(self): from gcloud.bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES project = 'PROJECT' zone = 'zone' @@ -383,7 +383,7 @@ def test_reload(self): expected_result = None # reload() has no return value. # Check Cluster optional config values before. - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) self.assertEqual(cluster.display_name, cluster_id) # Perform the method and check the result. diff --git a/gcloud/bigtable/test_row.py b/gcloud/bigtable/test_row.py index 71a62763bd32..b2385be2901c 100644 --- a/gcloud/bigtable/test_row.py +++ b/gcloud/bigtable/test_row.py @@ -420,7 +420,7 @@ def test_commit_too_many_mutations(self): row = self._makeOne(row_key, table) row._pb_mutations = [1, 2, 3] num_mutations = len(row._pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): + with _Monkey(MUT, MAX_MUTATIONS=num_mutations - 1): with self.assertRaises(ValueError): row.commit() @@ -521,7 +521,7 @@ def test_commit_with_filter_too_many_mutations(self): row = self._makeOne(row_key, table, filter_=filter_) row._true_pb_mutations = [1, 2, 3] num_mutations = len(row._true_pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): + with _Monkey(MUT, MAX_MUTATIONS=num_mutations - 1): with self.assertRaises(ValueError): row.commit() diff --git a/scripts/verify_included_modules.py b/scripts/verify_included_modules.py index e290a020114e..a136e72c2d8a 100644 --- a/scripts/verify_included_modules.py +++ b/scripts/verify_included_modules.py @@ -30,20 +30,22 @@ OBJECT_INVENTORY_RELPATH = os.path.join('_build', 'html', 'objects.inv') IGNORED_PREFIXES = ('test_', '_') IGNORED_MODULES = frozenset([ - 'gcloud.bigtable.client', - 'gcloud.bigtable.cluster', - 'gcloud.bigtable.column_family', - 'gcloud.bigtable.happybase.batch', - 'gcloud.bigtable.happybase.connection', - 'gcloud.bigtable.happybase.pool', - 'gcloud.bigtable.happybase.table', - 'gcloud.bigtable.row', - 'gcloud.bigtable.row_data', - 'gcloud.bigtable.table', + 'gcloud.__init__', + 'gcloud.bigquery.__init__', + 'gcloud.bigtable.__init__', + 'gcloud.datastore.__init__', + 'gcloud.datastore.demo.__init__', 'gcloud.datastore.demo.demo', 'gcloud.demo', + 'gcloud.dns.__init__', 'gcloud.iterator', + 'gcloud.pubsub.__init__', + 'gcloud.resource_manager.__init__', + 'gcloud.search.__init__', + 'gcloud.storage.__init__', + 'gcloud.storage.demo.__init__', 'gcloud.storage.demo.demo', + 'gcloud.streaming.__init__', 'gcloud.streaming.buffered_stream', 'gcloud.streaming.exceptions', 'gcloud.streaming.http_wrapper', @@ -73,6 +75,8 @@ def is_valid_module(filename): """ if not filename.endswith('.py'): return False + if filename == '__init__.py': + return True for prefix in IGNORED_PREFIXES: if filename.startswith(prefix): return False