diff --git a/docs/bigtable-row-filters.rst b/docs/bigtable-row-filters.rst new file mode 100644 index 000000000000..b5e99eab6575 --- /dev/null +++ b/docs/bigtable-row-filters.rst @@ -0,0 +1,68 @@ +Bigtable Row Filters +==================== + +It is possible to use a +:class:`RowFilter ` +when adding mutations to a +:class:`ConditionalRow ` and when +reading row data with :meth:`read_row() ` +:meth:`read_rows() `. + +As laid out in the `RowFilter definition`_, the following basic filters +are provided: + +* :class:`SinkFilter <.row_filters.SinkFilter>` +* :class:`PassAllFilter <.row_filters.PassAllFilter>` +* :class:`BlockAllFilter <.row_filters.BlockAllFilter>` +* :class:`RowKeyRegexFilter <.row_filters.RowKeyRegexFilter>` +* :class:`RowSampleFilter <.row_filters.RowSampleFilter>` +* :class:`FamilyNameRegexFilter <.row_filters.FamilyNameRegexFilter>` +* :class:`ColumnQualifierRegexFilter <.row_filters.ColumnQualifierRegexFilter>` +* :class:`TimestampRangeFilter <.row_filters.TimestampRangeFilter>` +* :class:`ColumnRangeFilter <.row_filters.ColumnRangeFilter>` +* :class:`ValueRegexFilter <.row_filters.ValueRegexFilter>` +* :class:`ValueRangeFilter <.row_filters.ValueRangeFilter>` +* :class:`CellsRowOffsetFilter <.row_filters.CellsRowOffsetFilter>` +* :class:`CellsRowLimitFilter <.row_filters.CellsRowLimitFilter>` +* :class:`CellsColumnLimitFilter <.row_filters.CellsColumnLimitFilter>` +* :class:`StripValueTransformerFilter <.row_filters.StripValueTransformerFilter>` +* :class:`ApplyLabelFilter <.row_filters.ApplyLabelFilter>` + +In addition, these filters can be combined into composite filters with + +* :class:`RowFilterChain <.row_filters.RowFilterChain>` +* :class:`RowFilterUnion <.row_filters.RowFilterUnion>` +* :class:`ConditionalRowFilter <.row_filters.ConditionalRowFilter>` + +These rules can be nested arbitrarily, with a basic filter at the lowest +level. For example: + +.. code:: python + + # Filter in a specified column (matching any column family). + col1_filter = ColumnQualifierRegexFilter(b'columnbia') + + # Create a filter to label results. + label1 = u'label-red' + label1_filter = ApplyLabelFilter(label1) + + # Combine the filters to label all the cells in columnbia. + chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) + + # Create a similar filter to label cells blue. + col2_filter = ColumnQualifierRegexFilter(b'columnseeya') + label2 = u'label-blue' + label2_filter = ApplyLabelFilter(label2) + chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) + + # Bring our two labeled columns together. + row_filter = RowFilterUnion(filters=[chain1, chain2]) + +---- + +.. automodule:: gcloud.bigtable.row_filters + :members: + :undoc-members: + :show-inheritance: + +.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/bigtable-row.rst b/docs/bigtable-row.rst index fae56b76f3d5..fbd96352be24 100644 --- a/docs/bigtable-row.rst +++ b/docs/bigtable-row.rst @@ -1,67 +1,7 @@ Bigtable Row ============ -It is possible to use a :class:`RowFilter ` -when adding mutations to a -:class:`ConditionalRow ` and when -reading row data with :meth:`read_row() ` -:meth:`read_rows() `. - -As laid out in the `RowFilter definition`_, the following basic filters -are provided: - -* :class:`SinkFilter <.row.SinkFilter>` -* :class:`PassAllFilter <.row.PassAllFilter>` -* :class:`BlockAllFilter <.row.BlockAllFilter>` -* :class:`RowKeyRegexFilter <.row.RowKeyRegexFilter>` -* :class:`RowSampleFilter <.row.RowSampleFilter>` -* :class:`FamilyNameRegexFilter <.row.FamilyNameRegexFilter>` -* :class:`ColumnQualifierRegexFilter <.row.ColumnQualifierRegexFilter>` -* :class:`TimestampRangeFilter <.row.TimestampRangeFilter>` -* :class:`ColumnRangeFilter <.row.ColumnRangeFilter>` -* :class:`ValueRegexFilter <.row.ValueRegexFilter>` -* :class:`ValueRangeFilter <.row.ValueRangeFilter>` -* :class:`CellsRowOffsetFilter <.row.CellsRowOffsetFilter>` -* :class:`CellsRowLimitFilter <.row.CellsRowLimitFilter>` -* :class:`CellsColumnLimitFilter <.row.CellsColumnLimitFilter>` -* :class:`StripValueTransformerFilter <.row.StripValueTransformerFilter>` -* :class:`ApplyLabelFilter <.row.ApplyLabelFilter>` - -In addition, these filters can be combined into composite filters with - -* :class:`RowFilterChain <.row.RowFilterChain>` -* :class:`RowFilterUnion <.row.RowFilterUnion>` -* :class:`ConditionalRowFilter <.row.ConditionalRowFilter>` - -These rules can be nested arbitrarily, with a basic filter at the lowest -level. For example: - -.. code:: python - - # Filter in a specified column (matching any column family). - col1_filter = ColumnQualifierRegexFilter(b'columnbia') - - # Create a filter to label results. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - - # Combine the filters to label all the cells in columnbia. - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Create a similar filter to label cells blue. - col2_filter = ColumnQualifierRegexFilter(b'columnseeya') - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - ----- - .. automodule:: gcloud.bigtable.row :members: :undoc-members: :show-inheritance: - -.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/index.rst b/docs/index.rst index 84afe9522fc8..dd0e6e90bf70 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -69,6 +69,7 @@ bigtable-table bigtable-column-family bigtable-row + bigtable-row-filters bigtable-row-data happybase-connection happybase-pool diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index 58953dbc81d5..64660706be2e 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -1,4 +1,3 @@ -# pylint: disable=C0302 # Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/gcloud/bigtable/happybase/batch.py b/gcloud/bigtable/happybase/batch.py index 663b7f892c77..25e6d073fc66 100644 --- a/gcloud/bigtable/happybase/batch.py +++ b/gcloud/bigtable/happybase/batch.py @@ -21,7 +21,7 @@ import six from gcloud._helpers import _datetime_from_microseconds -from gcloud.bigtable.row import TimestampRange +from gcloud.bigtable.row_filters import TimestampRange _WAL_SENTINEL = object() diff --git a/gcloud/bigtable/happybase/table.py b/gcloud/bigtable/happybase/table.py index dcbf45163015..bf018ad1647f 100644 --- a/gcloud/bigtable/happybase/table.py +++ b/gcloud/bigtable/happybase/table.py @@ -30,14 +30,14 @@ from gcloud.bigtable.happybase.batch import _get_column_pairs from gcloud.bigtable.happybase.batch import _WAL_SENTINEL from gcloud.bigtable.happybase.batch import Batch -from gcloud.bigtable.row import CellsColumnLimitFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import FamilyNameRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion -from gcloud.bigtable.row import RowKeyRegexFilter -from gcloud.bigtable.row import TimestampRange -from gcloud.bigtable.row import TimestampRangeFilter +from gcloud.bigtable.row_filters import CellsColumnLimitFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import FamilyNameRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion +from gcloud.bigtable.row_filters import RowKeyRegexFilter +from gcloud.bigtable.row_filters import TimestampRange +from gcloud.bigtable.row_filters import TimestampRangeFilter from gcloud.bigtable.table import Table as _LowLevelTable diff --git a/gcloud/bigtable/happybase/test_batch.py b/gcloud/bigtable/happybase/test_batch.py index 40cc84ae4c67..cf2156f226b4 100644 --- a/gcloud/bigtable/happybase/test_batch.py +++ b/gcloud/bigtable/happybase/test_batch.py @@ -46,7 +46,7 @@ def test_constructor_defaults(self): def test_constructor_explicit(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange table = object() timestamp = 144185290431 diff --git a/gcloud/bigtable/happybase/test_table.py b/gcloud/bigtable/happybase/test_table.py index cf95896a7696..9c5382a9e7fe 100644 --- a/gcloud/bigtable/happybase/test_table.py +++ b/gcloud/bigtable/happybase/test_table.py @@ -1113,7 +1113,7 @@ def test_invalid_type(self): def test_success(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange timestamp = 1441928298571 ts_dt = _datetime_from_microseconds(1000 * timestamp) @@ -1219,7 +1219,7 @@ def test_no_filters(self): self._callFUT() def test_single_filter(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 1337 result = self._callFUT(versions=versions) @@ -1229,7 +1229,7 @@ def test_single_filter(self): self.assertEqual(result.num_cells, versions) def test_existing_filters(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter filters = [] versions = 1337 @@ -1244,9 +1244,9 @@ def test_existing_filters(self): def _column_helper(self, num_filters, versions=None, timestamp=None, column=None, col_fam=None, qual=None): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain if col_fam is None: col_fam = 'cf1' @@ -1282,7 +1282,7 @@ def test_column_unicode(self): col_fam=u'cfU', qual=u'qualN') def test_with_versions(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 11 result = self._column_helper(num_filters=3, versions=versions) @@ -1295,8 +1295,8 @@ def test_with_versions(self): def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange - from gcloud.bigtable.row import TimestampRangeFilter + from gcloud.bigtable.row_filters import TimestampRange + from gcloud.bigtable.row_filters import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) @@ -1330,7 +1330,7 @@ def test_no_columns(self): self._callFUT(columns) def test_single_column(self): - from gcloud.bigtable.row import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter col_fam = 'cf1' columns = [col_fam] @@ -1339,10 +1339,10 @@ def test_single_column(self): self.assertEqual(result, expected_result) def test_column_and_column_families(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain - from gcloud.bigtable.row import RowFilterUnion + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain + from gcloud.bigtable.row_filters import RowFilterUnion col_fam1 = 'cf1' col_fam2 = 'cf2' @@ -1378,7 +1378,7 @@ def test_no_rows(self): self._callFUT(row_keys) def test_single_row(self): - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key = b'row-key' row_keys = [row_key] @@ -1387,8 +1387,8 @@ def test_single_row(self): self.assertEqual(result, expected_result) def test_many_rows(self): - from gcloud.bigtable.row import RowFilterUnion - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowFilterUnion + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key1 = b'row-key1' row_key2 = b'row-key2' diff --git a/gcloud/bigtable/row.py b/gcloud/bigtable/row.py index 276f009b7bb1..6666a62e9d8a 100644 --- a/gcloud/bigtable/row.py +++ b/gcloud/bigtable/row.py @@ -397,7 +397,7 @@ class ConditionalRow(DirectRow): :type table: :class:`Table ` :param table: The table that owns the row. - :type filter_: :class:`RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: Filter to be used for conditional mutations. """ def __init__(self, row_key, table, filter_): @@ -771,750 +771,6 @@ def commit(self): return _parse_rmw_row_response(row_response) -class RowFilter(object): - """Basic filter to apply to cells in a row. - - These values can be combined via :class:`RowFilterChain`, - :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - - .. note:: - - This class is a do-nothing base class for all row filters. - """ - - def __ne__(self, other): - return not self.__eq__(other) - - -class _BoolFilter(RowFilter): - """Row filter that uses a boolean flag. - - :type flag: bool - :param flag: An indicator if a setting is turned on or off. - """ - - def __init__(self, flag): - self.flag = flag - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.flag == self.flag - - -class SinkFilter(_BoolFilter): - """Advanced row filter to skip parent filters. - - :type flag: bool - :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(sink=self.flag) - - -class PassAllFilter(_BoolFilter): - """Row filter equivalent to not filtering at all. - - :type flag: bool - :param flag: Matches all cells, regardless of input. Functionally - equivalent to leaving ``filter`` unset, but included for - completeness. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(pass_all_filter=self.flag) - - -class BlockAllFilter(_BoolFilter): - """Row filter that doesn't match any cells. - - :type flag: bool - :param flag: Does not match any cells, regardless of input. Useful for - temporarily disabling just part of a filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(block_all_filter=self.flag) - - -class _RegexFilter(RowFilter): - """Row filter that uses a regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: bytes or str - :param regex: A regular expression (RE2) for some row filter. - """ - - def __init__(self, regex): - self.regex = regex - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.regex == self.regex - - -class RowKeyRegexFilter(_RegexFilter): - """Row filter for a row key regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from rows with row - keys that satisfy this regex. For a - ``CheckAndMutateRowRequest``, this filter is unnecessary - since the row key is already specified. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_key_regex_filter=self.regex) - - -class RowSampleFilter(RowFilter): - """Matches all cells from a row with probability p. - - :type sample: float - :param sample: The probability of matching a cell (must be in the - interval ``[0, 1]``). - """ - - def __init__(self, sample): - self.sample = sample - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.sample == self.sample - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_sample_filter=self.sample) - - -class FamilyNameRegexFilter(_RegexFilter): - """Row filter for a family name regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: str - :param regex: A regular expression (RE2) to match cells from columns in a - given column family. For technical reasons, the regex must - not contain the ``':'`` character, even if it is not being - used as a literal. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(family_name_regex_filter=self.regex) - - -class ColumnQualifierRegexFilter(_RegexFilter): - """Row filter for a column qualifier regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from column that - match this regex (irrespective of column family). - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) - - -class TimestampRange(object): - """Range of time with inclusive lower and exclusive upper bounds. - - :type start: :class:`datetime.datetime` - :param start: (Optional) The (inclusive) lower bound of the timestamp - range. If omitted, defaults to Unix epoch. - - :type end: :class:`datetime.datetime` - :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, no upper bound is used. - """ - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start == self.start and - other.end == self.end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`TimestampRange` to a protobuf. - - :rtype: :class:`.data_pb2.TimestampRange` - :returns: The converted current object. - """ - timestamp_range_kwargs = {} - if self.start is not None: - timestamp_range_kwargs['start_timestamp_micros'] = ( - _microseconds_from_datetime(self.start)) - if self.end is not None: - timestamp_range_kwargs['end_timestamp_micros'] = ( - _microseconds_from_datetime(self.end)) - return data_pb2.TimestampRange(**timestamp_range_kwargs) - - -class TimestampRangeFilter(RowFilter): - """Row filter that limits cells to a range of time. - - :type range_: :class:`TimestampRange` - :param range_: Range of time that cells should match against. - """ - - def __init__(self, range_): - self.range_ = range_ - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.range_ == self.range_ - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts the ``range_`` on the current object to a protobuf and - then uses it in the ``timestamp_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) - - -class ColumnRangeFilter(RowFilter): - """A row filter to restrict to a range of columns. - - Both the start and end column can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must - be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is - used, the backend applies no upper bound to the - values. - - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_column`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_column`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_column`` is given or if ``inclusive_end`` - is set but no ``end_column`` is given - """ - - def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=None, inclusive_end=None): - self.column_family_id = column_family_id - - if inclusive_start is None: - inclusive_start = True - elif start_column is None: - raise ValueError('Inclusive start was specified but no ' - 'start column was given.') - self.start_column = start_column - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_column is None: - raise ValueError('Inclusive end was specified but no ' - 'end column was given.') - self.end_column = end_column - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.start_column == self.start_column and - other.end_column == self.end_column and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ColumnRange` and then uses it - in the ``column_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - column_range_kwargs = {'family_name': self.column_family_id} - if self.start_column is not None: - if self.inclusive_start: - key = 'start_qualifier_inclusive' - else: - key = 'start_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: - if self.inclusive_end: - key = 'end_qualifier_inclusive' - else: - key = 'end_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.end_column) - - column_range = data_pb2.ColumnRange(**column_range_kwargs) - return data_pb2.RowFilter(column_range_filter=column_range) - - -class ValueRegexFilter(_RegexFilter): - """Row filter for a value regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells with values that - match this regex. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(value_regex_filter=self.regex) - - -class ValueRangeFilter(RowFilter): - """A range of values to restrict to in a row filter. - - Will only match cells that have values in this range. - - Both the start and end value can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type start_value: bytes - :param start_value: The start of the range of values. If no value is used, - the backend applies no lower bound to the values. - - :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_value`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_value`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_value`` is given or if ``inclusive_end`` - is set but no ``end_value`` is given - """ - - def __init__(self, start_value=None, end_value=None, - inclusive_start=None, inclusive_end=None): - if inclusive_start is None: - inclusive_start = True - elif start_value is None: - raise ValueError('Inclusive start was specified but no ' - 'start value was given.') - self.start_value = start_value - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_value is None: - raise ValueError('Inclusive end was specified but no ' - 'end value was given.') - self.end_value = end_value - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start_value == self.start_value and - other.end_value == self.end_value and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ValueRange` and then uses - it to create a row filter protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - value_range_kwargs = {} - if self.start_value is not None: - if self.inclusive_start: - key = 'start_value_inclusive' - else: - key = 'start_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.start_value) - if self.end_value is not None: - if self.inclusive_end: - key = 'end_value_inclusive' - else: - key = 'end_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.end_value) - - value_range = data_pb2.ValueRange(**value_range_kwargs) - return data_pb2.RowFilter(value_range_filter=value_range) - - -class _CellCountFilter(RowFilter): - """Row filter that uses an integer count of cells. - - The cell count is used as an offset or a limit for the number - of results returned. - - :type num_cells: int - :param num_cells: An integer count / offset / limit. - """ - - def __init__(self, num_cells): - self.num_cells = num_cells - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.num_cells == self.num_cells - - -class CellsRowOffsetFilter(_CellCountFilter): - """Row filter to skip cells in a row. - - :type num_cells: int - :param num_cells: Skips the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) - - -class CellsRowLimitFilter(_CellCountFilter): - """Row filter to limit cells in a row. - - :type num_cells: int - :param num_cells: Matches only the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) - - -class CellsColumnLimitFilter(_CellCountFilter): - """Row filter to limit cells in a column. - - :type num_cells: int - :param num_cells: Matches only the most recent N cells within each column. - This filters a (family name, column) pair, based on - timestamps of each cell. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) - - -class StripValueTransformerFilter(_BoolFilter): - """Row filter that transforms cells into empty string (0 bytes). - - :type flag: bool - :param flag: If :data:`True`, replaces each cell's value with the empty - string. As the name indicates, this is more useful as a - transformer than a generic query / filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(strip_value_transformer=self.flag) - - -class ApplyLabelFilter(RowFilter): - """Filter to apply labels to cells. - - Intended to be used as an intermediate filter on a pre-existing filtered - result set. This way if two sets are combined, the label can tell where - the cell(s) originated.This allows the client to determine which results - were produced from which part of the filter. - - .. note:: - - Due to a technical limitation of the backend, it is not currently - possible to apply multiple labels to a cell. - - :type label: str - :param label: Label to apply to cells in the output row. Values must be - at most 15 characters long, and match the pattern - ``[a-z0-9\\-]+``. - """ - - def __init__(self, label): - self.label = label - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.label == self.label - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(apply_label_transformer=self.label) - - -class _FilterCombination(RowFilter): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def __init__(self, filters=None): - if filters is None: - filters = [] - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - -class RowFilterChain(_FilterCombination): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - chain = data_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(chain=chain) - - -class RowFilterUnion(_FilterCombination): - """Union of row filters. - - Sends rows through several filters simultaneously, then - merges / interleaves all the filtered results together. - - If multiple cells are produced with the same column and timestamp, - they will all appear in the output row in an unspecified mutual order. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - interleave = data_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(interleave=interleave) - - -class ConditionalRowFilter(RowFilter): - """Conditional row filter which exhibits ternary behavior. - - Executes one of two filters based on another filter. If the ``base_filter`` - returns any cells in the row, then ``true_filter`` is executed. If not, - then ``false_filter`` is executed. - - .. note:: - - The ``base_filter`` does not execute atomically with the true and false - filters, which may lead to inconsistent or unexpected results. - - Additionally, executing a :class:`ConditionalRowFilter` has poor - performance on the server, especially when ``false_filter`` is set. - - :type base_filter: :class:`RowFilter` - :param base_filter: The filter to condition on before executing the - true/false filters. - - :type true_filter: :class:`RowFilter` - :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results - will be returned in the true case. - - :type false_filter: :class:`RowFilter` - :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results - will be returned in the false case. - """ - - def __init__(self, base_filter, true_filter=None, false_filter=None): - self.base_filter = base_filter - self.true_filter = true_filter - self.false_filter = false_filter - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.base_filter == self.base_filter and - other.true_filter == self.true_filter and - other.false_filter == self.false_filter) - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} - if self.true_filter is not None: - condition_kwargs['true_filter'] = self.true_filter.to_pb() - if self.false_filter is not None: - condition_kwargs['false_filter'] = self.false_filter.to_pb() - condition = data_pb2.RowFilter.Condition(**condition_kwargs) - return data_pb2.RowFilter(condition=condition) - - def _parse_rmw_row_response(row_response): """Parses the response to a ``ReadModifyWriteRow`` request. diff --git a/gcloud/bigtable/row_filters.py b/gcloud/bigtable/row_filters.py new file mode 100644 index 000000000000..be220b6308f2 --- /dev/null +++ b/gcloud/bigtable/row_filters.py @@ -0,0 +1,764 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Filters for Google Cloud Bigtable Row classes.""" + + +from gcloud._helpers import _microseconds_from_datetime +from gcloud._helpers import _to_bytes +from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + +class RowFilter(object): + """Basic filter to apply to cells in a row. + + These values can be combined via :class:`RowFilterChain`, + :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. + + .. note:: + + This class is a do-nothing base class for all row filters. + """ + + def __ne__(self, other): + return not self.__eq__(other) + + +class _BoolFilter(RowFilter): + """Row filter that uses a boolean flag. + + :type flag: bool + :param flag: An indicator if a setting is turned on or off. + """ + + def __init__(self, flag): + self.flag = flag + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.flag == self.flag + + +class SinkFilter(_BoolFilter): + """Advanced row filter to skip parent filters. + + :type flag: bool + :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. + Outputs all cells directly to the output of the read rather + than to any parent filter. Cannot be used within the + ``predicate_filter``, ``true_filter``, or ``false_filter`` + of a :class:`ConditionalRowFilter`. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(sink=self.flag) + + +class PassAllFilter(_BoolFilter): + """Row filter equivalent to not filtering at all. + + :type flag: bool + :param flag: Matches all cells, regardless of input. Functionally + equivalent to leaving ``filter`` unset, but included for + completeness. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(pass_all_filter=self.flag) + + +class BlockAllFilter(_BoolFilter): + """Row filter that doesn't match any cells. + + :type flag: bool + :param flag: Does not match any cells, regardless of input. Useful for + temporarily disabling just part of a filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(block_all_filter=self.flag) + + +class _RegexFilter(RowFilter): + """Row filter that uses a regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: bytes or str + :param regex: A regular expression (RE2) for some row filter. + """ + + def __init__(self, regex): + self.regex = regex + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.regex == self.regex + + +class RowKeyRegexFilter(_RegexFilter): + """Row filter for a row key regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from rows with row + keys that satisfy this regex. For a + ``CheckAndMutateRowRequest``, this filter is unnecessary + since the row key is already specified. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_key_regex_filter=self.regex) + + +class RowSampleFilter(RowFilter): + """Matches all cells from a row with probability p. + + :type sample: float + :param sample: The probability of matching a cell (must be in the + interval ``[0, 1]``). + """ + + def __init__(self, sample): + self.sample = sample + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.sample == self.sample + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_sample_filter=self.sample) + + +class FamilyNameRegexFilter(_RegexFilter): + """Row filter for a family name regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: str + :param regex: A regular expression (RE2) to match cells from columns in a + given column family. For technical reasons, the regex must + not contain the ``':'`` character, even if it is not being + used as a literal. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(family_name_regex_filter=self.regex) + + +class ColumnQualifierRegexFilter(_RegexFilter): + """Row filter for a column qualifier regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from column that + match this regex (irrespective of column family). + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) + + +class TimestampRange(object): + """Range of time with inclusive lower and exclusive upper bounds. + + :type start: :class:`datetime.datetime` + :param start: (Optional) The (inclusive) lower bound of the timestamp + range. If omitted, defaults to Unix epoch. + + :type end: :class:`datetime.datetime` + :param end: (Optional) The (exclusive) upper bound of the timestamp + range. If omitted, no upper bound is used. + """ + + def __init__(self, start=None, end=None): + self.start = start + self.end = end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start == self.start and + other.end == self.end) + + def __ne__(self, other): + return not self.__eq__(other) + + def to_pb(self): + """Converts the :class:`TimestampRange` to a protobuf. + + :rtype: :class:`.data_pb2.TimestampRange` + :returns: The converted current object. + """ + timestamp_range_kwargs = {} + if self.start is not None: + timestamp_range_kwargs['start_timestamp_micros'] = ( + _microseconds_from_datetime(self.start)) + if self.end is not None: + timestamp_range_kwargs['end_timestamp_micros'] = ( + _microseconds_from_datetime(self.end)) + return data_pb2.TimestampRange(**timestamp_range_kwargs) + + +class TimestampRangeFilter(RowFilter): + """Row filter that limits cells to a range of time. + + :type range_: :class:`TimestampRange` + :param range_: Range of time that cells should match against. + """ + + def __init__(self, range_): + self.range_ = range_ + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.range_ == self.range_ + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts the ``range_`` on the current object to a protobuf and + then uses it in the ``timestamp_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + + +class ColumnRangeFilter(RowFilter): + """A row filter to restrict to a range of columns. + + Both the start and end column can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type column_family_id: str + :param column_family_id: The column family that contains the columns. Must + be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. + + :type start_column: bytes + :param start_column: The start of the range of columns. If no value is + used, the backend applies no upper bound to the + values. + + :type end_column: bytes + :param end_column: The end of the range of columns. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start column should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_column`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end column should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_column`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_column`` is given or if ``inclusive_end`` + is set but no ``end_column`` is given + """ + + def __init__(self, column_family_id, start_column=None, end_column=None, + inclusive_start=None, inclusive_end=None): + self.column_family_id = column_family_id + + if inclusive_start is None: + inclusive_start = True + elif start_column is None: + raise ValueError('Inclusive start was specified but no ' + 'start column was given.') + self.start_column = start_column + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_column is None: + raise ValueError('Inclusive end was specified but no ' + 'end column was given.') + self.end_column = end_column + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.column_family_id == self.column_family_id and + other.start_column == self.start_column and + other.end_column == self.end_column and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ColumnRange` and then uses it + in the ``column_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + column_range_kwargs = {'family_name': self.column_family_id} + if self.start_column is not None: + if self.inclusive_start: + key = 'start_qualifier_inclusive' + else: + key = 'start_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.start_column) + if self.end_column is not None: + if self.inclusive_end: + key = 'end_qualifier_inclusive' + else: + key = 'end_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.end_column) + + column_range = data_pb2.ColumnRange(**column_range_kwargs) + return data_pb2.RowFilter(column_range_filter=column_range) + + +class ValueRegexFilter(_RegexFilter): + """Row filter for a value regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells with values that + match this regex. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(value_regex_filter=self.regex) + + +class ValueRangeFilter(RowFilter): + """A range of values to restrict to in a row filter. + + Will only match cells that have values in this range. + + Both the start and end value can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type start_value: bytes + :param start_value: The start of the range of values. If no value is used, + the backend applies no lower bound to the values. + + :type end_value: bytes + :param end_value: The end of the range of values. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start value should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_value`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end value should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_value`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_value`` is given or if ``inclusive_end`` + is set but no ``end_value`` is given + """ + + def __init__(self, start_value=None, end_value=None, + inclusive_start=None, inclusive_end=None): + if inclusive_start is None: + inclusive_start = True + elif start_value is None: + raise ValueError('Inclusive start was specified but no ' + 'start value was given.') + self.start_value = start_value + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_value is None: + raise ValueError('Inclusive end was specified but no ' + 'end value was given.') + self.end_value = end_value + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start_value == self.start_value and + other.end_value == self.end_value and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ValueRange` and then uses + it to create a row filter protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + value_range_kwargs = {} + if self.start_value is not None: + if self.inclusive_start: + key = 'start_value_inclusive' + else: + key = 'start_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.start_value) + if self.end_value is not None: + if self.inclusive_end: + key = 'end_value_inclusive' + else: + key = 'end_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.end_value) + + value_range = data_pb2.ValueRange(**value_range_kwargs) + return data_pb2.RowFilter(value_range_filter=value_range) + + +class _CellCountFilter(RowFilter): + """Row filter that uses an integer count of cells. + + The cell count is used as an offset or a limit for the number + of results returned. + + :type num_cells: int + :param num_cells: An integer count / offset / limit. + """ + + def __init__(self, num_cells): + self.num_cells = num_cells + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.num_cells == self.num_cells + + +class CellsRowOffsetFilter(_CellCountFilter): + """Row filter to skip cells in a row. + + :type num_cells: int + :param num_cells: Skips the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + + +class CellsRowLimitFilter(_CellCountFilter): + """Row filter to limit cells in a row. + + :type num_cells: int + :param num_cells: Matches only the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + + +class CellsColumnLimitFilter(_CellCountFilter): + """Row filter to limit cells in a column. + + :type num_cells: int + :param num_cells: Matches only the most recent N cells within each column. + This filters a (family name, column) pair, based on + timestamps of each cell. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + + +class StripValueTransformerFilter(_BoolFilter): + """Row filter that transforms cells into empty string (0 bytes). + + :type flag: bool + :param flag: If :data:`True`, replaces each cell's value with the empty + string. As the name indicates, this is more useful as a + transformer than a generic query / filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(strip_value_transformer=self.flag) + + +class ApplyLabelFilter(RowFilter): + """Filter to apply labels to cells. + + Intended to be used as an intermediate filter on a pre-existing filtered + result set. This way if two sets are combined, the label can tell where + the cell(s) originated.This allows the client to determine which results + were produced from which part of the filter. + + .. note:: + + Due to a technical limitation of the backend, it is not currently + possible to apply multiple labels to a cell. + + :type label: str + :param label: Label to apply to cells in the output row. Values must be + at most 15 characters long, and match the pattern + ``[a-z0-9\\-]+``. + """ + + def __init__(self, label): + self.label = label + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.label == self.label + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(apply_label_transformer=self.label) + + +class _FilterCombination(RowFilter): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def __init__(self, filters=None): + if filters is None: + filters = [] + self.filters = filters + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.filters == self.filters + + +class RowFilterChain(_FilterCombination): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + chain = data_pb2.RowFilter.Chain( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(chain=chain) + + +class RowFilterUnion(_FilterCombination): + """Union of row filters. + + Sends rows through several filters simultaneously, then + merges / interleaves all the filtered results together. + + If multiple cells are produced with the same column and timestamp, + they will all appear in the output row in an unspecified mutual order. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + interleave = data_pb2.RowFilter.Interleave( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(interleave=interleave) + + +class ConditionalRowFilter(RowFilter): + """Conditional row filter which exhibits ternary behavior. + + Executes one of two filters based on another filter. If the ``base_filter`` + returns any cells in the row, then ``true_filter`` is executed. If not, + then ``false_filter`` is executed. + + .. note:: + + The ``base_filter`` does not execute atomically with the true and false + filters, which may lead to inconsistent or unexpected results. + + Additionally, executing a :class:`ConditionalRowFilter` has poor + performance on the server, especially when ``false_filter`` is set. + + :type base_filter: :class:`RowFilter` + :param base_filter: The filter to condition on before executing the + true/false filters. + + :type true_filter: :class:`RowFilter` + :param true_filter: (Optional) The filter to execute if there are any cells + matching ``base_filter``. If not provided, no results + will be returned in the true case. + + :type false_filter: :class:`RowFilter` + :param false_filter: (Optional) The filter to execute if there are no cells + matching ``base_filter``. If not provided, no results + will be returned in the false case. + """ + + def __init__(self, base_filter, true_filter=None, false_filter=None): + self.base_filter = base_filter + self.true_filter = true_filter + self.false_filter = false_filter + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.base_filter == self.base_filter and + other.true_filter == self.true_filter and + other.false_filter == self.false_filter) + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} + if self.true_filter is not None: + condition_kwargs['true_filter'] = self.true_filter.to_pb() + if self.false_filter is not None: + condition_kwargs['false_filter'] = self.false_filter.to_pb() + condition = data_pb2.RowFilter.Condition(**condition_kwargs) + return data_pb2.RowFilter(condition=condition) diff --git a/gcloud/bigtable/test_row.py b/gcloud/bigtable/test_row.py index 4caa6668e2b2..71efbb2cd57c 100644 --- a/gcloud/bigtable/test_row.py +++ b/gcloud/bigtable/test_row.py @@ -237,7 +237,7 @@ def test_delete_cells_no_time_range(self): def test_delete_cells_with_time_range(self): import datetime from gcloud._helpers import _EPOCH - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange microseconds = 30871000 # Makes sure already milliseconds granularity start = _EPOCH + datetime.timedelta(microseconds=microseconds) @@ -410,7 +410,7 @@ def test_commit(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter row_key = b'row_key' table_name = 'projects/more-stuff' @@ -670,995 +670,6 @@ def test_commit_too_many_mutations(self): row.commit() -class Test_BoolFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _BoolFilter - return _BoolFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - flag = object() - row_filter = self._makeOne(flag) - self.assertTrue(row_filter.flag is flag) - - def test___eq__type_differ(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestSinkFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import SinkFilter - return SinkFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(sink=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestPassAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import PassAllFilter - return PassAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(pass_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestBlockAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import BlockAllFilter - return BlockAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(block_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class Test_RegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _RegexFilter - return _RegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - regex = object() - row_filter = self._makeOne(regex) - self.assertTrue(row_filter.regex is regex) - - def test___eq__type_differ(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestRowKeyRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowKeyRegexFilter - return RowKeyRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'row-key-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestRowSampleFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowSampleFilter - return RowSampleFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - sample = object() - row_filter = self._makeOne(sample) - self.assertTrue(row_filter.sample is sample) - - def test___eq__type_differ(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = self._makeOne(sample) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - sample = 0.25 - row_filter = self._makeOne(sample) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_sample_filter=sample) - self.assertEqual(pb_val, expected_pb) - - -class TestFamilyNameRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import FamilyNameRegexFilter - return FamilyNameRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = u'family-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnQualifierRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - return ColumnQualifierRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'column-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestTimestampRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRange - return TimestampRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - start = object() - end = object() - time_range = self._makeOne(start=start, end=end) - self.assertTrue(time_range.start is start) - self.assertTrue(time_range.end is end) - - def test___eq__(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - self.assertEqual(time_range1, time_range2) - - def test___eq__type_differ(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = object() - self.assertNotEqual(time_range1, time_range2) - - def test___ne__same_value(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - comparison_val = (time_range1 != time_range2) - self.assertFalse(comparison_val) - - def _to_pb_helper(self, start_micros=None, end_micros=None): - import datetime - from gcloud._helpers import _EPOCH - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - pb_kwargs = {} - - start = None - if start_micros is not None: - start = _EPOCH + datetime.timedelta(microseconds=start_micros) - pb_kwargs['start_timestamp_micros'] = start_micros - end = None - if end_micros is not None: - end = _EPOCH + datetime.timedelta(microseconds=end_micros) - pb_kwargs['end_timestamp_micros'] = end_micros - time_range = self._makeOne(start=start, end=end) - - expected_pb = data_pb2.TimestampRange(**pb_kwargs) - self.assertEqual(time_range.to_pb(), expected_pb) - - def test_to_pb(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - end_micros = 12939371000 - self._to_pb_helper(start_micros=start_micros, - end_micros=end_micros) - - def test_to_pb_start_only(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - self._to_pb_helper(start_micros=start_micros) - - def test_to_pb_end_only(self): - # Makes sure already milliseconds granularity - end_micros = 12939371000 - self._to_pb_helper(end_micros=end_micros) - - -class TestTimestampRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRangeFilter - return TimestampRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - range_ = object() - row_filter = self._makeOne(range_) - self.assertTrue(row_filter.range_ is range_) - - def test___eq__type_differ(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = self._makeOne(range_) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import TimestampRange - - range_ = TimestampRange() - row_filter = self._makeOne(range_) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - timestamp_range_filter=data_pb2.TimestampRange()) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnRangeFilter - return ColumnRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - column_family_id = object() - row_filter = self._makeOne(column_family_id) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertEqual(row_filter.start_column, None) - self.assertEqual(row_filter.end_column, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertTrue(row_filter.start_column is start_column) - self.assertTrue(row_filter.end_column is end_column) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_start=True) - - def test_constructor_bad_end(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_end=True) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - column_family_id = object() - row_filter1 = self._makeOne(column_family_id) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - row_filter = self._makeOne(column_family_id) - col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column, - inclusive_start=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column, - inclusive_end=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class TestValueRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRegexFilter - return ValueRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'value-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestValueRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRangeFilter - return ValueRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.start_value, None) - self.assertEqual(row_filter.end_value, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.start_value is start_value) - self.assertTrue(row_filter.end_value is end_value) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - self.assertRaises(ValueError, self._makeOne, inclusive_start=True) - - def test_constructor_bad_end(self): - self.assertRaises(ValueError, self._makeOne, inclusive_end=True) - - def test___eq__(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - row_filter1 = self._makeOne() - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - row_filter = self._makeOne() - expected_pb = data_pb2.RowFilter( - value_range_filter=data_pb2.ValueRange()) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value) - val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value, inclusive_start=False) - val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value) - val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value, inclusive_end=False) - val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class Test_CellCountFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _CellCountFilter - return _CellCountFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - num_cells = object() - row_filter = self._makeOne(num_cells) - self.assertTrue(row_filter.num_cells is num_cells) - - def test___eq__type_differ(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestCellsRowOffsetFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowOffsetFilter - return CellsRowOffsetFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 76 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsRowLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowLimitFilter - return CellsRowLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 189 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsColumnLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsColumnLimitFilter - return CellsColumnLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 10 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - cells_per_column_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestStripValueTransformerFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import StripValueTransformerFilter - return StripValueTransformerFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestApplyLabelFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ApplyLabelFilter - return ApplyLabelFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - label = object() - row_filter = self._makeOne(label) - self.assertTrue(row_filter.label is label) - - def test___eq__type_differ(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = self._makeOne(label) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - label = u'label' - row_filter = self._makeOne(label) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(apply_label_transformer=label) - self.assertEqual(pb_val, expected_pb) - - -class Test_FilterCombination(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _FilterCombination - return _FilterCombination - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.filters, []) - - def test_constructor_explicit(self): - filters = object() - row_filter = self._makeOne(filters=filters) - self.assertTrue(row_filter.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - -class TestRowFilterChain(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterChain - return RowFilterChain - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestRowFilterUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterUnion - return RowFilterUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestConditionalRowFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ConditionalRowFilter - return ConditionalRowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertTrue(cond_filter.base_filter is base_filter) - self.assertTrue(cond_filter.true_filter is true_filter) - self.assertTrue(cond_filter.false_filter is false_filter) - - def test___eq__(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertEqual(cond_filter1, cond_filter2) - - def test___eq__type_differ(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = object() - self.assertNotEqual(cond_filter1, cond_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowOffsetFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = CellsRowOffsetFilter(11) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, - false_filter=row_filter3) - filter_pb = row_filter4.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - false_filter=row_filter3_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_true_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_false_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - false_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - class Test__parse_rmw_row_response(unittest2.TestCase): def _callFUT(self, row_response): diff --git a/gcloud/bigtable/test_row_filters.py b/gcloud/bigtable/test_row_filters.py new file mode 100644 index 000000000000..90a79dceb52d --- /dev/null +++ b/gcloud/bigtable/test_row_filters.py @@ -0,0 +1,1005 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest2 + + +class Test_BoolFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _BoolFilter + return _BoolFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + flag = object() + row_filter = self._makeOne(flag) + self.assertTrue(row_filter.flag is flag) + + def test___eq__type_differ(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestSinkFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import SinkFilter + return SinkFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(sink=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestPassAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import PassAllFilter + return PassAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(pass_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestBlockAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import BlockAllFilter + return BlockAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(block_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class Test_RegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _RegexFilter + return _RegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + regex = object() + row_filter = self._makeOne(regex) + self.assertTrue(row_filter.regex is regex) + + def test___eq__type_differ(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestRowKeyRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowKeyRegexFilter + return RowKeyRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'row-key-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestRowSampleFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowSampleFilter + return RowSampleFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + sample = object() + row_filter = self._makeOne(sample) + self.assertTrue(row_filter.sample is sample) + + def test___eq__type_differ(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = self._makeOne(sample) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + sample = 0.25 + row_filter = self._makeOne(sample) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_sample_filter=sample) + self.assertEqual(pb_val, expected_pb) + + +class TestFamilyNameRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + return FamilyNameRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = u'family-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnQualifierRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + return ColumnQualifierRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'column-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestTimestampRange(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRange + return TimestampRange + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + start = object() + end = object() + time_range = self._makeOne(start=start, end=end) + self.assertTrue(time_range.start is start) + self.assertTrue(time_range.end is end) + + def test___eq__(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + self.assertEqual(time_range1, time_range2) + + def test___eq__type_differ(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = object() + self.assertNotEqual(time_range1, time_range2) + + def test___ne__same_value(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + comparison_val = (time_range1 != time_range2) + self.assertFalse(comparison_val) + + def _to_pb_helper(self, start_micros=None, end_micros=None): + import datetime + from gcloud._helpers import _EPOCH + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + pb_kwargs = {} + + start = None + if start_micros is not None: + start = _EPOCH + datetime.timedelta(microseconds=start_micros) + pb_kwargs['start_timestamp_micros'] = start_micros + end = None + if end_micros is not None: + end = _EPOCH + datetime.timedelta(microseconds=end_micros) + pb_kwargs['end_timestamp_micros'] = end_micros + time_range = self._makeOne(start=start, end=end) + + expected_pb = data_pb2.TimestampRange(**pb_kwargs) + self.assertEqual(time_range.to_pb(), expected_pb) + + def test_to_pb(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + end_micros = 12939371000 + self._to_pb_helper(start_micros=start_micros, + end_micros=end_micros) + + def test_to_pb_start_only(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + self._to_pb_helper(start_micros=start_micros) + + def test_to_pb_end_only(self): + # Makes sure already milliseconds granularity + end_micros = 12939371000 + self._to_pb_helper(end_micros=end_micros) + + +class TestTimestampRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRangeFilter + return TimestampRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + range_ = object() + row_filter = self._makeOne(range_) + self.assertTrue(row_filter.range_ is range_) + + def test___eq__type_differ(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = self._makeOne(range_) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import TimestampRange + + range_ = TimestampRange() + row_filter = self._makeOne(range_) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + timestamp_range_filter=data_pb2.TimestampRange()) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnRangeFilter + return ColumnRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + column_family_id = object() + row_filter = self._makeOne(column_family_id) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertEqual(row_filter.start_column, None) + self.assertEqual(row_filter.end_column, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(column_family_id, start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertTrue(row_filter.start_column is start_column) + self.assertTrue(row_filter.end_column is end_column) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_start=True) + + def test_constructor_bad_end(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_end=True) + + def test___eq__(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + column_family_id = object() + row_filter1 = self._makeOne(column_family_id) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + row_filter = self._makeOne(column_family_id) + col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column, + inclusive_start=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column, + inclusive_end=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class TestValueRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRegexFilter + return ValueRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'value-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(value_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestValueRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRangeFilter + return ValueRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.start_value, None) + self.assertEqual(row_filter.end_value, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.start_value is start_value) + self.assertTrue(row_filter.end_value is end_value) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + self.assertRaises(ValueError, self._makeOne, inclusive_start=True) + + def test_constructor_bad_end(self): + self.assertRaises(ValueError, self._makeOne, inclusive_end=True) + + def test___eq__(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + row_filter1 = self._makeOne() + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + row_filter = self._makeOne() + expected_pb = data_pb2.RowFilter( + value_range_filter=data_pb2.ValueRange()) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value) + val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value, inclusive_start=False) + val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value) + val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value, inclusive_end=False) + val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class Test_CellCountFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _CellCountFilter + return _CellCountFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + num_cells = object() + row_filter = self._makeOne(num_cells) + self.assertTrue(row_filter.num_cells is num_cells) + + def test___eq__type_differ(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestCellsRowOffsetFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + return CellsRowOffsetFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 76 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsRowLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowLimitFilter + return CellsRowLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 189 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsColumnLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsColumnLimitFilter + return CellsColumnLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 10 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + cells_per_column_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestStripValueTransformerFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import StripValueTransformerFilter + return StripValueTransformerFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestApplyLabelFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ApplyLabelFilter + return ApplyLabelFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + label = object() + row_filter = self._makeOne(label) + self.assertTrue(row_filter.label is label) + + def test___eq__type_differ(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = self._makeOne(label) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + label = u'label' + row_filter = self._makeOne(label) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(apply_label_transformer=label) + self.assertEqual(pb_val, expected_pb) + + +class Test_FilterCombination(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _FilterCombination + return _FilterCombination + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.filters, []) + + def test_constructor_explicit(self): + filters = object() + row_filter = self._makeOne(filters=filters) + self.assertTrue(row_filter.filters is filters) + + def test___eq__(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = self._makeOne(filters=filters) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + +class TestRowFilterChain(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterChain + return RowFilterChain + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestRowFilterUnion(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterUnion + return RowFilterUnion + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestConditionalRowFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ConditionalRowFilter + return ConditionalRowFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertTrue(cond_filter.base_filter is base_filter) + self.assertTrue(cond_filter.true_filter is true_filter) + self.assertTrue(cond_filter.false_filter is false_filter) + + def test___eq__(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertEqual(cond_filter1, cond_filter2) + + def test___eq__type_differ(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = object() + self.assertNotEqual(cond_filter1, cond_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = CellsRowOffsetFilter(11) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, + false_filter=row_filter3) + filter_pb = row_filter4.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + false_filter=row_filter3_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_true_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_false_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + false_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) diff --git a/gcloud/bigtable/test_table.py b/gcloud/bigtable/test_table.py index 65cfe0d87fe2..9fcdf21593b0 100644 --- a/gcloud/bigtable/test_table.py +++ b/gcloud/bigtable/test_table.py @@ -582,7 +582,7 @@ def test_row_range_both_keys(self): def test_with_filter(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter table_name = 'table_name' row_filter = RowSampleFilter(0.33) diff --git a/scripts/pylintrc_default b/scripts/pylintrc_default index 4a76e193baae..a06fd338f61d 100644 --- a/scripts/pylintrc_default +++ b/scripts/pylintrc_default @@ -200,7 +200,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1593 +max-module-lines=1200 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 1e92422fc45c..7bc418b5cdac 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -66,7 +66,7 @@ } TEST_RC_REPLACEMENTS = { 'FORMAT': { - 'max-module-lines': 1825, + 'max-module-lines': 1700, }, } diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 5066e2e710de..39d60e0658f6 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -24,10 +24,10 @@ from gcloud._helpers import UTC from gcloud.bigtable.client import Client from gcloud.bigtable.column_family import MaxVersionsGCRule -from gcloud.bigtable.row import ApplyLabelFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion +from gcloud.bigtable.row_filters import ApplyLabelFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData from gcloud.environment_vars import TESTS_PROJECT