Jorwoods/query endpoint paging (#881)

* Automatically paginate in QuerySet endpoints
* Enable queryset operations for flows, groups, jobs, projects
* Keep only page_size in memory
* NULL result cache to ensure restarting from the first page
* Use modulus to simplify idx calculation
* Add support for slicing
* Minimum page number is 1
* Properly support negative steps

Author: jorwoods

tableauserverclient/models/workbook_item.py

@@ -16,6 +16,7 @@
 from .view_item import ViewItem
 from ..datetime_helpers import parse_datetime
 
+
 if TYPE_CHECKING:
     from .connection_item import ConnectionItem
     from .permissions_item import PermissionsRule

tableauserverclient/server/endpoint/__init__.py

@@ -3,7 +3,7 @@
 from .data_alert_endpoint import DataAlerts
 from .databases_endpoint import Databases
 from .datasources_endpoint import Datasources
-from .endpoint import Endpoint
+from .endpoint import Endpoint, QuerysetEndpoint
 from .exceptions import (
     ServerResponseError,
     MissingRequiredFieldError,

tableauserverclient/server/endpoint/endpoint.py

@@ -228,7 +228,9 @@ def all(self, *args, **kwargs):
         return queryset
 
     @api(version="2.0")
-    def filter(self, *args, **kwargs):
+    def filter(self, *_, **kwargs):
+        if _:
+            raise RuntimeError("Only keyword arguments accepted.")
         queryset = QuerySet(self).filter(**kwargs)
         return queryset
 

tableauserverclient/server/endpoint/flows_endpoint.py

@@ -6,7 +6,7 @@
 from typing import Iterable, List, Optional, TYPE_CHECKING, Tuple, Union
 
 from .dqw_endpoint import _DataQualityWarningEndpoint
-from .endpoint import Endpoint, api
+from .endpoint import Endpoint, QuerysetEndpoint, api
 from .exceptions import InternalServerError, MissingRequiredFieldError
 from .permissions_endpoint import _PermissionsEndpoint
 from .resource_tagger import _ResourceTagger
@@ -30,7 +30,7 @@
 FilePath = Union[str, os.PathLike]
 
 
-class Flows(Endpoint):
+class Flows(QuerysetEndpoint):
     def __init__(self, parent_srv):
         super(Flows, self).__init__(parent_srv)
         self._resource_tagger = _ResourceTagger(parent_srv)

tableauserverclient/server/endpoint/groups_endpoint.py

@@ -1,6 +1,6 @@
 import logging
 
-from .endpoint import Endpoint, api
+from .endpoint import QuerysetEndpoint, api
 from .exceptions import MissingRequiredFieldError
 from .. import RequestFactory, GroupItem, UserItem, PaginationItem, JobItem
 from ..pager import Pager
@@ -13,7 +13,7 @@
     from ..request_options import RequestOptions
 
 
-class Groups(Endpoint):
+class Groups(QuerysetEndpoint):
     @property
     def baseurl(self) -> str:
         return "{0}/sites/{1}/groups".format(self.parent_srv.baseurl, self.parent_srv.site_id)

tableauserverclient/server/endpoint/jobs_endpoint.py

@@ -1,6 +1,6 @@
 import logging
 
-from .endpoint import Endpoint, api
+from .endpoint import QuerysetEndpoint, api
 from .exceptions import JobCancelledException, JobFailedException
 from .. import JobItem, BackgroundJobItem, PaginationItem
 from ..request_options import RequestOptionsBase
@@ -11,7 +11,7 @@
 from typing import List, Optional, Tuple, Union
 
 
-class Jobs(Endpoint):
+class Jobs(QuerysetEndpoint):
     @property
     def baseurl(self):
         return "{0}/sites/{1}/jobs".format(self.parent_srv.baseurl, self.parent_srv.site_id)

tableauserverclient/server/endpoint/projects_endpoint.py

@@ -1,7 +1,7 @@
 import logging
 
 from .default_permissions_endpoint import _DefaultPermissionsEndpoint
-from .endpoint import api, Endpoint, XML_CONTENT_TYPE
+from .endpoint import QuerysetEndpoint, api, XML_CONTENT_TYPE
 from .exceptions import MissingRequiredFieldError
 from .permissions_endpoint import _PermissionsEndpoint
 from .. import RequestFactory, RequestOptions, ProjectItem, PaginationItem, Permission
@@ -15,7 +15,7 @@
     from ..request_options import RequestOptions
 
 
-class Projects(Endpoint):
+class Projects(QuerysetEndpoint):
     def __init__(self, parent_srv: "Server") -> None:
         super(Projects, self).__init__(parent_srv)
 

tableauserverclient/server/query.py

@@ -1,6 +1,7 @@
 from .filter import Filter
 from .request_options import RequestOptions
 from .sort import Sort
+import math
 
 
 def to_camel_case(word):
@@ -15,11 +16,54 @@ def __init__(self, model):
         self._pagination_item = None
 
     def __iter__(self):
-        self._fetch_all()
-        return iter(self._result_cache)
+        self.request_options.pagenumber = 1
+        self._result_cache = None
+        total = self.total_available
+        size = self.page_size
+        yield from self._result_cache
+        for page in range(1, math.ceil(total / size)):
+            self.request_options.pagenumber = page + 1
+            self._result_cache = None
+            self._fetch_all()
+            yield from self._result_cache
 
     def __getitem__(self, k):
-        return list(self)[k]
+        page = self.page_number
+        size = self.page_size
+
+        page_range = range((page - 1) * size, page * size)
+
+        if isinstance(k, slice):
+            step = k.step if k.step is not None else 1
+            start = k.start if k.start is not None else 0
+            stop = k.stop if k.stop is not None else self.total_available
+            if start < 0:
+                start += self.total_available
+            if stop < 0:
+                stop += self.total_available
+            if start < stop and step < 0:
+                # Since slicing is left inclusive and right exclusive, shift
+                # the start and stop values by 1 to keep that behavior
+                start, stop = stop - 1, start - 1
+                slice_stop = stop if stop > 0 else None
+                k = slice(start, slice_stop, step)
+
+            k_range = range(start, stop, step)
+            if all(i in page_range for i in k_range):
+                return self._result_cache[k]
+            return [self[i] for i in k_range]
+
+        if k < 0:
+            k += self.total_available
+
+        if k in page_range:
+            return self._result_cache[k % size]
+        elif k in range(self.total_available):
+            self._result_cache = None
+            self.request_options.pagenumber = max(1, math.ceil(k / size))
+            return self[k]
+        else:
+            raise IndexError
 
     def _fetch_all(self):
         """
@@ -43,7 +87,9 @@ def page_size(self):
         self._fetch_all()
         return self._pagination_item.page_size
 
-    def filter(self, **kwargs):
+    def filter(self, *invalid, **kwargs):
+        if invalid:
+            raise RuntimeError(f"Only accepts keyword arguments.")
         for kwarg_key, value in kwargs.items():
             field_name, operator = self._parse_shorthand_filter(kwarg_key)
             self.request_options.filter.add(Filter(field_name, operator, value))

test/assets/request_option_slicing_queryset.xml

@@ -0,0 +1,46 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
+    <pagination pageNumber="1" pageSize="10" totalAvailable="10" />
+    <views>
+        <view id="d79634e1-6063-4ec9-95ff-50acbf609ff5" name="ENDANGERED SAFARI" contentUrl="SafariSample/sheets/ENDANGEREDSAFARI">
+            <workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="fd252f73-593c-4c4e-8584-c032b8022adc" name="Overview" contentUrl="Superstore/sheets/Overview">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="7ddf1576-4b15-4df5-98cb-03066f279059" name="Product" contentUrl="Superstore/sheets/Product">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="c75f90a9-f10e-46be-8f01-07403780386a" name="Customers" contentUrl="Superstore/sheets/Customers">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="880cd185-3f15-4063-816c-4a600883f4ed" name="Shipping" contentUrl="Superstore/sheets/Shipping">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="29c73e36-94ed-474a-9533-e310d1e6a787" name="Performance" contentUrl="Superstore/sheets/Performance">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="48ff745e-aeda-4d76-a162-2bc740b4f478" name="Commission Model" contentUrl="Superstore/sheets/CommissionModel">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="2df55de2-3a2d-4e34-b515-6d4e70b830e9" name="Order Details" contentUrl="Superstore/sheets/OrderDetails">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="c1513b2f-cb6f-4ae8-841f-4128fdd2a7dd" name="Forecast" contentUrl="Superstore/sheets/Forecast">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+        <view id="2e6d6c81-da71-4b41-892c-ba80d4e7a6d0" name="What If Forecast" contentUrl="Superstore/sheets/WhatIfForecast">
+            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+        </view>
+    </views>
+</tsResponse>

test/test_request_option.py

@@ -14,6 +14,7 @@
 FILTER_EQUALS = os.path.join(TEST_ASSET_DIR, "request_option_filter_equals.xml")
 FILTER_TAGS_IN = os.path.join(TEST_ASSET_DIR, "request_option_filter_tags_in.xml")
 FILTER_MULTIPLE = os.path.join(TEST_ASSET_DIR, "request_option_filter_tags_in.xml")
+SLICING_QUERYSET = os.path.join(TEST_ASSET_DIR, "request_option_slicing_queryset.xml")
 
 
 class RequestOptionTests(unittest.TestCase):
@@ -244,3 +245,31 @@ def test_multiple_filter_options_shorthand(self):
             for _ in range(100):
                 matching_workbooks = self.server.workbooks.filter(tags__in=["sample", "safari", "weather"], name="foo")
                 self.assertEqual(3, matching_workbooks.total_available)
+
+    def test_slicing_queryset(self):
+        with open(SLICING_QUERYSET, "rb") as f:
+            response_xml = f.read().decode("utf-8")
+        with requests_mock.mock() as m:
+            m.get(self.baseurl + "/views?pageNumber=1", text=response_xml)
+            all_views = self.server.views.all()
+
+            self.assertEqual(10, len(all_views[::]))
+            self.assertEqual(5, len(all_views[::2]))
+            self.assertEqual(8, len(all_views[2:]))
+            self.assertEqual(2, len(all_views[:2]))
+            self.assertEqual(3, len(all_views[2:5]))
+            self.assertEqual(3, len(all_views[-3:]))
+            self.assertEqual(3, len(all_views[-6:-3]))
+            self.assertEqual(3, len(all_views[3:6:-1]))
+            self.assertEqual(3, len(all_views[6:3:-1]))
+            self.assertEqual(10, len(all_views[::-1]))
+            self.assertEqual(all_views[3:6], list(reversed(all_views[3:6:-1])))
+
+            self.assertEqual(all_views[-3].id, "2df55de2-3a2d-4e34-b515-6d4e70b830e9")
+
+        with self.assertRaises(IndexError):
+            all_views[100]
+
+    def test_queryset_filter_args_error(self):
+        with self.assertRaises(RuntimeError):
+            workbooks = self.server.workbooks.filter("argument")