Skip to content

fix(bigquery): fix start index with page size for list rows #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
HemangChothani wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

fix(bigquery): fix start index with page size for list rows #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a352a0
fix(bigquery): fix start index with page size for list rows
HemangChothani Jan 29, 2020
b1b3f95
fix(bigquery): cosmetic changes
HemangChothani Jan 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bigquery/google/cloud/bigquery/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1362,6 +1362,8 @@ def _get_next_page_response(self):
"""
params = self._get_query_params()
if self._page_size is not None:
if self.page_number and "startIndex" in params:
del params["startIndex"]
Copy link
Collaborator

@IlyaFaer IlyaFaer Jan 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think, we could write if self.page_number instead of if self.page_number != 0. Page number is not gonna be negative, so this if-statement will be giving True in all cases except self.page_number == 0. And it's little bit shorter.

Copy link
Collaborator Author

@HemangChothani HemangChothani Jan 31, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here I need to delete startIndex from query parameter after first request so when first request hit at that time page_num is 0 and after it increments so except page 0 need to delete for every request.

params["maxResults"] = self._page_size
return self.api_request(
method=self._HTTP_METHOD, path=self.path, query_params=params
Expand Down
67 changes: 67 additions & 0 deletions bigquery/tests/unit/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5610,6 +5610,73 @@ def _bigquery_timestamp_float_repr(ts_float):
method="GET", path="/%s" % PATH, query_params={}, timeout=7.5
)

def test_list_rows_w_start_index_w_page_size(self):
from google.cloud.bigquery.schema import SchemaField
from google.cloud.bigquery.table import Table
from google.cloud.bigquery.table import Row

PATH = "projects/%s/datasets/%s/tables/%s/data" % (
self.PROJECT,
self.DS_ID,
self.TABLE_ID,
)

page_1 = {
"totalRows": 4,
"pageToken": "some-page-token",
"rows": [
{"f": [{"v": "Phred Phlyntstone"}]},
{"f": [{"v": "Bharney Rhubble"}]},
],
}
page_2 = {
"totalRows": 4,
"rows": [
{"f": [{"v": "Wylma Phlyntstone"}]},
{"f": [{"v": "Bhettye Rhubble"}]},
],
}
Copy link
Collaborator

@IlyaFaer IlyaFaer Jan 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

page_1 and page_2 are constants, I believe

Copy link
Collaborator Author

@HemangChothani HemangChothani Jan 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes this is the response with pages.

creds = _make_credentials()
http = object()
client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
conn = client._connection = make_connection(page_1, page_2)
full_name = SchemaField("full_name", "STRING", mode="REQUIRED")

table = Table(self.TABLE_REF, schema=[full_name])

iterator = client.list_rows(table, max_results=4, page_size=2, start_index=1)
pages = iterator.pages
rows = list(six.next(pages))
extra_params = iterator.extra_params
f2i = {"full_name": 0}
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0], Row(("Phred Phlyntstone",), f2i))
self.assertEqual(rows[1], Row(("Bharney Rhubble",), f2i))

rows = list(six.next(pages))

self.assertEqual(len(rows), 2)
self.assertEqual(rows[0], Row(("Wylma Phlyntstone",), f2i))
self.assertEqual(rows[1], Row(("Bhettye Rhubble",), f2i))
self.assertEqual(extra_params, {"startIndex": 1})

conn.api_request.assert_has_calls(
[
mock.call(
method="GET",
path="/%s" % PATH,
query_params={"startIndex": 1, "maxResults": 2},
timeout=None,
),
mock.call(
method="GET",
path="/%s" % PATH,
query_params={"pageToken": "some-page-token", "maxResults": 2},
timeout=None,
),
]
)

def test_list_rows_empty_table(self):
response = {"totalRows": "0", "rows": []}
creds = _make_credentials()
Expand Down