From 42dd7ae8bb99da07546a698612b9f358d7570dbf Mon Sep 17 00:00:00 2001 From: takbab Date: Thu, 21 Jul 2016 10:29:31 +0900 Subject: [PATCH 1/4] Add configuration.query.userDefinedFunctionResources to bigquery.jobs. --- gcloud/bigquery/job.py | 11 +++++++++++ gcloud/bigquery/test_job.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/gcloud/bigquery/job.py b/gcloud/bigquery/job.py index 7febfd06c60b..2399b1dde05e 100644 --- a/gcloud/bigquery/job.py +++ b/gcloud/bigquery/job.py @@ -873,6 +873,7 @@ class _AsyncQueryConfiguration(object): _priority = None _use_query_cache = None _use_legacy_sql = None + _udf_resources = None _write_disposition = None @@ -937,6 +938,12 @@ def __init__(self, name, query, client): reference/v2/jobs#configuration.query.useLegacySql """ + udf_resources = _TypedProperty( + 'udf_resources', list) + """See: + https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.userDefinedFunctionResources + """ + write_disposition = WriteDisposition('write_disposition') """See: https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.writeDisposition @@ -977,6 +984,10 @@ def _populate_config_resource(self, configuration): configuration['useQueryCache'] = self.use_query_cache if self.use_legacy_sql is not None: configuration['useLegacySql'] = self.use_legacy_sql + if self.udf_resources is not None: + configuration['userDefinedFunctionResources'] = ( + self.udf_resources + ) if self.write_disposition is not None: configuration['writeDisposition'] = self.write_disposition diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index aade86379493..7647916c7795 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -1219,6 +1219,7 @@ class TestQueryJob(unittest2.TestCase, _Base): JOB_TYPE = 'query' QUERY = 'select count(*) from persons' DESTINATION_TABLE = 'destination_table' + UDF = {"resourceUri": "gs://backet/functions.js", "inlineCode": ""} def _getTargetClass(self): from gcloud.bigquery.job import QueryJob @@ -1427,6 +1428,7 @@ def test_begin_w_alternate_client(self): 'priority': 'INTERACTIVE', 'useQueryCache': True, 'useLegacySql': True, + 'userDefinedFunctionResources': [self.UDF], 'writeDisposition': 'WRITE_TRUNCATE', } RESOURCE['configuration']['query'] = QUERY_CONFIGURATION @@ -1447,6 +1449,7 @@ def test_begin_w_alternate_client(self): job.priority = 'INTERACTIVE' job.use_query_cache = True job.use_legacy_sql = True + job.udf_resources = [self.UDF] job.write_disposition = 'WRITE_TRUNCATE' job.begin(client=client2) From 383b490066833e0b7f6912f00323c94edf8095b9 Mon Sep 17 00:00:00 2001 From: takbab Date: Fri, 22 Jul 2016 17:40:07 +0900 Subject: [PATCH 2/4] add udf resource class for BigQuery jobs API. --- gcloud/bigquery/job.py | 20 +++++++++++++++++--- gcloud/bigquery/test_job.py | 10 ++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/gcloud/bigquery/job.py b/gcloud/bigquery/job.py index 2399b1dde05e..6c3bb532c786 100644 --- a/gcloud/bigquery/job.py +++ b/gcloud/bigquery/job.py @@ -877,6 +877,14 @@ class _AsyncQueryConfiguration(object): _write_disposition = None +class UdfResource(object): + """UDF resource for QueryJob.""" + + def __init__(self, uri=None, code=None): + self.uri = uri + self.code = code + + class QueryJob(_AsyncJob): """Asynchronous job: query tables. @@ -938,8 +946,7 @@ def __init__(self, name, query, client): reference/v2/jobs#configuration.query.useLegacySql """ - udf_resources = _TypedProperty( - 'udf_resources', list) + udf_resources = _TypedProperty('udf_resources', list) """See: https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.userDefinedFunctionResources """ @@ -985,8 +992,15 @@ def _populate_config_resource(self, configuration): if self.use_legacy_sql is not None: configuration['useLegacySql'] = self.use_legacy_sql if self.udf_resources is not None: + udf_list = [] + for udf_resource in self.udf_resources: + udf = { + 'resourceUri': udf_resource.uri, + 'inlineCode': udf_resource.code + } + udf_list.append(udf) configuration['userDefinedFunctionResources'] = ( - self.udf_resources + udf_list ) if self.write_disposition is not None: configuration['writeDisposition'] = self.write_disposition diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index 7647916c7795..123bdbf6e7f7 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest2 +from gcloud.bigquery.job import UdfResource class _Base(object): @@ -1219,7 +1220,7 @@ class TestQueryJob(unittest2.TestCase, _Base): JOB_TYPE = 'query' QUERY = 'select count(*) from persons' DESTINATION_TABLE = 'destination_table' - UDF = {"resourceUri": "gs://backet/functions.js", "inlineCode": ""} + UDF = UdfResource(uri="gs://backet/functions.js", code="") def _getTargetClass(self): from gcloud.bigquery.job import QueryJob @@ -1428,7 +1429,12 @@ def test_begin_w_alternate_client(self): 'priority': 'INTERACTIVE', 'useQueryCache': True, 'useLegacySql': True, - 'userDefinedFunctionResources': [self.UDF], + 'userDefinedFunctionResources': [ + { + "resourceUri": self.UDF.uri, + "inlineCode": self.UDF.code + } + ], 'writeDisposition': 'WRITE_TRUNCATE', } RESOURCE['configuration']['query'] = QUERY_CONFIGURATION From 5c7f7b0cf6ceabf2e153784d98999dcc26285c9c Mon Sep 17 00:00:00 2001 From: takbab Date: Sat, 23 Jul 2016 11:18:56 +0900 Subject: [PATCH 3/4] add a property of udf_resources in QueryJob. https://github.com/GoogleCloudPlatform/gcloud-python/pull/2007#issuecomment-234561095 --- gcloud/bigquery/job.py | 24 +++++++++++++++++++++++- gcloud/bigquery/test_job.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/gcloud/bigquery/job.py b/gcloud/bigquery/job.py index 6c3bb532c786..cb3b69164cda 100644 --- a/gcloud/bigquery/job.py +++ b/gcloud/bigquery/job.py @@ -946,7 +946,7 @@ def __init__(self, name, query, client): reference/v2/jobs#configuration.query.useLegacySql """ - udf_resources = _TypedProperty('udf_resources', list) + _udf_resources = None """See: https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.userDefinedFunctionResources """ @@ -956,6 +956,28 @@ def __init__(self, name, query, client): https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.writeDisposition """ + @property + def udf_resources(self): + """List of user-defined-function resources. + + :type: list of :class:`UdfResource` + :returns: the list, if set, or None + """ + return list(self._udf_resources) if self._udf_resources else None + + @udf_resources.setter + def udf_resources(self, value): + """Update list of user-defined-function resources. + + :type value: list of :class:`UdfResource` + :param value:the new list + + :raises: ValueError, if the items in the list are not all instance of :class:`UdfResource` + """ + if not all((isinstance(item, UdfResource) for item in value)): + raise ValueError("pass a list of UdfResource instances") + self._udf_resources = tuple(value) + def _destination_table_resource(self): """Create a JSON resource for the destination table. diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index 123bdbf6e7f7..63abdae43423 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -1220,7 +1220,8 @@ class TestQueryJob(unittest2.TestCase, _Base): JOB_TYPE = 'query' QUERY = 'select count(*) from persons' DESTINATION_TABLE = 'destination_table' - UDF = UdfResource(uri="gs://backet/functions.js", code="") + UDF1 = UdfResource(uri="gs://backet/functions.js") + UDF2 = UdfResource(code="function foo(row, emit) {...") def _getTargetClass(self): from gcloud.bigquery.job import QueryJob @@ -1291,6 +1292,18 @@ def _verifyResourceProperties(self, job, resource): config['priority']) else: self.assertTrue(job.priority is None) + if 'userDefinedFunctionResources' in config: + config_udf_list = config['userDefinedFunctionResources'] + udf_list = job.udf_resources + self.assertEquals(len(config_udf_list), len(udf_list)) + for config_udf, udf in zip(config_udf_list, udf_list): + udf_ref = { + 'inlineCode': udf.code, + 'resourceUri': udf.uri + } + self.assertEqual(udf_ref, config_udf) + else: + self.assertTrue(job.udf_resources is None) if 'writeDisposition' in config: self.assertEqual(job.write_disposition, config['writeDisposition']) @@ -1373,6 +1386,16 @@ def test_from_api_repr_w_properties(self): self.assertTrue(dataset._client is client) self._verifyResourceProperties(dataset, RESOURCE) + def test_set_udf_resource(self): + RESOURCE = self._makeResource() + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + job = self._makeOne(self.JOB_NAME, self.QUERY, client) + job.udf_resources = [self.UDF1] + job.udf_resources = [self.UDF1, self.UDF2] + with self.assertRaises(ValueError): + job.udf_resources = [self.UDF1, 'error'] + def test_begin_w_bound_client(self): PATH = 'projects/%s/jobs' % self.PROJECT RESOURCE = self._makeResource() @@ -1431,8 +1454,8 @@ def test_begin_w_alternate_client(self): 'useLegacySql': True, 'userDefinedFunctionResources': [ { - "resourceUri": self.UDF.uri, - "inlineCode": self.UDF.code + "resourceUri": self.UDF1.uri, + "inlineCode": self.UDF1.code } ], 'writeDisposition': 'WRITE_TRUNCATE', @@ -1455,7 +1478,7 @@ def test_begin_w_alternate_client(self): job.priority = 'INTERACTIVE' job.use_query_cache = True job.use_legacy_sql = True - job.udf_resources = [self.UDF] + job.udf_resources = [self.UDF1] job.write_disposition = 'WRITE_TRUNCATE' job.begin(client=client2) From c8d31c7328e66ad564b72f48479d54a275ffc847 Mon Sep 17 00:00:00 2001 From: takbab Date: Sat, 23 Jul 2016 11:34:14 +0900 Subject: [PATCH 4/4] wrap to < 79 characters for PEP8 --- gcloud/bigquery/job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gcloud/bigquery/job.py b/gcloud/bigquery/job.py index cb3b69164cda..4fea34f9a336 100644 --- a/gcloud/bigquery/job.py +++ b/gcloud/bigquery/job.py @@ -972,7 +972,8 @@ def udf_resources(self, value): :type value: list of :class:`UdfResource` :param value:the new list - :raises: ValueError, if the items in the list are not all instance of :class:`UdfResource` + :raises: ValueError, if the items in the list are not all instance of + :class:`UdfResource` """ if not all((isinstance(item, UdfResource) for item in value)): raise ValueError("pass a list of UdfResource instances")