Skip to content

Commit 1b946ba

Browse files
plamutplamut
authored
fix: remove DB-API dependency on pyarrow with decimal query parameters (#551)
* fix: DB API pyarrow dependency with decimal values DB API should gracefully handle the case when the optional pyarrow dependency is not installed. * Blacken DB API helpers tests * Refine the logic for recognizing NUMERIC Decimals
1 parent a460f93 commit 1b946ba

File tree

2 files changed

+60
-29
lines changed

2 files changed

+60
-29
lines changed

‎google/cloud/bigquery/dbapi/_helpers.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,15 @@
1919
import functools
2020
import numbers
2121

22-
try:
23-
import pyarrow
24-
except ImportError: # pragma: NO COVER
25-
pyarrow = None
26-
2722
from google.cloud import bigquery
2823
from google.cloud.bigquery import table
2924
from google.cloud.bigquery.dbapi import exceptions
3025

3126

27+
_NUMERIC_SERVER_MIN = decimal.Decimal("-9.9999999999999999999999999999999999999E+28")
28+
_NUMERIC_SERVER_MAX = decimal.Decimal("9.9999999999999999999999999999999999999E+28")
29+
30+
3231
def scalar_to_query_parameter(value, name=None):
3332
"""Convert a scalar value into a query parameter.
3433
@@ -189,12 +188,20 @@ def bigquery_scalar_type(value):
189188
elif isinstance(value, numbers.Real):
190189
return "FLOAT64"
191190
elif isinstance(value, decimal.Decimal):
192-
# We check for NUMERIC before BIGNUMERIC in order to support pyarrow < 3.0.
193-
scalar_object = pyarrow.scalar(value)
194-
if isinstance(scalar_object, pyarrow.Decimal128Scalar):
191+
vtuple = value.as_tuple()
192+
# NUMERIC values have precision of 38 (number of digits) and scale of 9 (number
193+
# of fractional digits), and their max absolute value must be strictly smaller
194+
# than 1.0E+29.
195+
# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types
196+
if (
197+
len(vtuple.digits) <= 38 # max precision: 38
198+
and vtuple.exponent >= -9 # max scale: 9
199+
and _NUMERIC_SERVER_MIN <= value <= _NUMERIC_SERVER_MAX
200+
):
195201
return "NUMERIC"
196202
else:
197203
return "BIGNUMERIC"
204+
198205
elif isinstance(value, str):
199206
return "STRING"
200207
elif isinstance(value, bytes):

‎tests/unit/test_dbapi__helpers.py‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525

2626
import google.cloud._helpers
2727
from google.cloud.bigquery import table
28-
from google.cloud.bigquery._pandas_helpers import _BIGNUMERIC_SUPPORT
2928
from google.cloud.bigquery.dbapi import _helpers
3029
from google.cloud.bigquery.dbapi import exceptions
3130
from tests.unit.helpers import _to_pyarrow
@@ -39,9 +38,8 @@ def test_scalar_to_query_parameter(self):
3938
(123, "INT64"),
4039
(-123456789, "INT64"),
4140
(1.25, "FLOAT64"),
42-
(decimal.Decimal("1.25"), "NUMERIC"),
4341
(b"I am some bytes", "BYTES"),
44-
(u"I am a string", "STRING"),
42+
("I am a string", "STRING"),
4543
(datetime.date(2017, 4, 1), "DATE"),
4644
(datetime.time(12, 34, 56), "TIME"),
4745
(datetime.datetime(2012, 3, 4, 5, 6, 7), "DATETIME"),
@@ -51,14 +49,17 @@ def test_scalar_to_query_parameter(self):
5149
),
5250
"TIMESTAMP",
5351
),
52+
(decimal.Decimal("1.25"), "NUMERIC"),
53+
(decimal.Decimal("9.9999999999999999999999999999999999999E+28"), "NUMERIC"),
54+
(decimal.Decimal("1.0E+29"), "BIGNUMERIC"), # more than max NUMERIC value
55+
(decimal.Decimal("1.123456789"), "NUMERIC"),
56+
(decimal.Decimal("1.1234567891"), "BIGNUMERIC"), # scale > 9
57+
(decimal.Decimal("12345678901234567890123456789.012345678"), "NUMERIC"),
58+
(
59+
decimal.Decimal("12345678901234567890123456789012345678"),
60+
"BIGNUMERIC", # larger than max NUMERIC value, despite precision <=38
61+
),
5462
]
55-
if _BIGNUMERIC_SUPPORT:
56-
expected_types.append(
57-
(
58-
decimal.Decimal("1.1234567890123456789012345678901234567890"),
59-
"BIGNUMERIC",
60-
)
61-
)
6263

6364
for value, expected_type in expected_types:
6465
msg = "value: {} expected_type: {}".format(value, expected_type)
@@ -71,6 +72,33 @@ def test_scalar_to_query_parameter(self):
7172
self.assertEqual(named_parameter.type_, expected_type, msg=msg)
7273
self.assertEqual(named_parameter.value, value, msg=msg)
7374

75+
def test_decimal_to_query_parameter(self): # TODO: merge with previous test
76+
77+
expected_types = [
78+
(decimal.Decimal("9.9999999999999999999999999999999999999E+28"), "NUMERIC"),
79+
(decimal.Decimal("1.0E+29"), "BIGNUMERIC"), # more than max value
80+
(decimal.Decimal("1.123456789"), "NUMERIC"),
81+
(decimal.Decimal("1.1234567891"), "BIGNUMERIC"), # scale > 9
82+
(decimal.Decimal("12345678901234567890123456789.012345678"), "NUMERIC"),
83+
(
84+
decimal.Decimal("12345678901234567890123456789012345678"),
85+
"BIGNUMERIC", # larger than max size, even if precision <=38
86+
),
87+
]
88+
89+
for value, expected_type in expected_types:
90+
msg = f"value: {value} expected_type: {expected_type}"
91+
92+
parameter = _helpers.scalar_to_query_parameter(value)
93+
self.assertIsNone(parameter.name, msg=msg)
94+
self.assertEqual(parameter.type_, expected_type, msg=msg)
95+
self.assertEqual(parameter.value, value, msg=msg)
96+
97+
named_parameter = _helpers.scalar_to_query_parameter(value, name="myvar")
98+
self.assertEqual(named_parameter.name, "myvar", msg=msg)
99+
self.assertEqual(named_parameter.type_, expected_type, msg=msg)
100+
self.assertEqual(named_parameter.value, value, msg=msg)
101+
74102
def test_scalar_to_query_parameter_w_unexpected_type(self):
75103
with self.assertRaises(exceptions.ProgrammingError):
76104
_helpers.scalar_to_query_parameter(value={"a": "dictionary"})
@@ -89,8 +117,9 @@ def test_array_to_query_parameter_valid_argument(self):
89117
([123, -456, 0], "INT64"),
90118
([1.25, 2.50], "FLOAT64"),
91119
([decimal.Decimal("1.25")], "NUMERIC"),
120+
([decimal.Decimal("{d38}.{d38}".format(d38="9" * 38))], "BIGNUMERIC"),
92121
([b"foo", b"bar"], "BYTES"),
93-
([u"foo", u"bar"], "STRING"),
122+
(["foo", "bar"], "STRING"),
94123
([datetime.date(2017, 4, 1), datetime.date(2018, 4, 1)], "DATE"),
95124
([datetime.time(12, 34, 56), datetime.time(10, 20, 30)], "TIME"),
96125
(
@@ -113,11 +142,6 @@ def test_array_to_query_parameter_valid_argument(self):
113142
),
114143
]
115144

116-
if _BIGNUMERIC_SUPPORT:
117-
expected_types.append(
118-
([decimal.Decimal("{d38}.{d38}".format(d38="9" * 38))], "BIGNUMERIC")
119-
)
120-
121145
for values, expected_type in expected_types:
122146
msg = "value: {} expected_type: {}".format(values, expected_type)
123147
parameter = _helpers.array_to_query_parameter(values)
@@ -134,7 +158,7 @@ def test_array_to_query_parameter_empty_argument(self):
134158
_helpers.array_to_query_parameter([])
135159

136160
def test_array_to_query_parameter_unsupported_sequence(self):
137-
unsupported_iterables = [{10, 20, 30}, u"foo", b"bar", bytearray([65, 75, 85])]
161+
unsupported_iterables = [{10, 20, 30}, "foo", b"bar", bytearray([65, 75, 85])]
138162
for iterable in unsupported_iterables:
139163
with self.assertRaises(exceptions.ProgrammingError):
140164
_helpers.array_to_query_parameter(iterable)
@@ -144,7 +168,7 @@ def test_array_to_query_parameter_sequence_w_invalid_elements(self):
144168
_helpers.array_to_query_parameter([object(), 2, 7])
145169

146170
def test_to_query_parameters_w_dict(self):
147-
parameters = {"somebool": True, "somestring": u"a-string-value"}
171+
parameters = {"somebool": True, "somestring": "a-string-value"}
148172
query_parameters = _helpers.to_query_parameters(parameters)
149173
query_parameter_tuples = []
150174
for param in query_parameters:
@@ -154,7 +178,7 @@ def test_to_query_parameters_w_dict(self):
154178
sorted(
155179
[
156180
("somebool", "BOOL", True),
157-
("somestring", "STRING", u"a-string-value"),
181+
("somestring", "STRING", "a-string-value"),
158182
]
159183
),
160184
)
@@ -177,14 +201,14 @@ def test_to_query_parameters_w_dict_dict_param(self):
177201
_helpers.to_query_parameters(parameters)
178202

179203
def test_to_query_parameters_w_list(self):
180-
parameters = [True, u"a-string-value"]
204+
parameters = [True, "a-string-value"]
181205
query_parameters = _helpers.to_query_parameters(parameters)
182206
query_parameter_tuples = []
183207
for param in query_parameters:
184208
query_parameter_tuples.append((param.name, param.type_, param.value))
185209
self.assertSequenceEqual(
186210
sorted(query_parameter_tuples),
187-
sorted([(None, "BOOL", True), (None, "STRING", u"a-string-value")]),
211+
sorted([(None, "BOOL", True), (None, "STRING", "a-string-value")]),
188212
)
189213

190214
def test_to_query_parameters_w_list_array_param(self):

0 commit comments

Comments
 (0)