Skip to content

bigquery: add config getters to LoadJob #4137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jba merged 1 commit into from
Oct 9, 2017
Merged

bigquery: add config getters to LoadJob #4137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 94 additions & 4 deletions bigquery/google/cloud/bigquery/job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -714,12 +714,102 @@ def __init__(self, job_id, source_uris, destination, client,
self._configuration = job_config

@property
def configuration(self):
"""Configuration for this job.
def allow_jagged_rows(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.allow_jagged_rows`.
"""
return self._configuration.allow_jagged_rows

@property
def allow_quoted_newlines(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.allow_quoted_newlines`.
"""
return self._configuration.allow_quoted_newlines

@property
def autodetect(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.autodetect`.
"""
return self._configuration.autodetect

@property
def create_disposition(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.create_disposition`.
"""
return self._configuration.create_disposition

@property
def encoding(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.encoding`.
"""
return self._configuration.encoding

@property
def field_delimiter(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.field_delimiter`.
"""
return self._configuration.field_delimiter

@property
def ignore_unknown_values(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.ignore_unknown_values`.
"""
return self._configuration.ignore_unknown_values

:rtype: :class:`~google.cloud.bigquery.job.LoadJobConfig`
@property
def max_bad_records(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.max_bad_records`.
"""
return self._configuration.max_bad_records

@property
def null_marker(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.null_marker`.
"""
return self._configuration.null_marker

@property
def quote_character(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.quote_character`.
"""
return self._configuration.quote_character

@property
def skip_leading_rows(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.skip_leading_rows`.
"""
return self._configuration.skip_leading_rows

@property
def source_format(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.source_format`.
"""
return self._configuration.source_format

@property
def write_disposition(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.write_disposition`.
"""
return self._configuration.write_disposition

@property
def schema(self):
"""See
:class:`~google.cloud.bigquery.job.LoadJobConfig.schema`.
"""
return self._configuration
return self._configuration.schema

@property
def input_file_bytes(self):
Expand Down
86 changes: 41 additions & 45 deletions bigquery/tests/unit/test_job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,53 +234,50 @@ def _makeResource(self, started=False, ended=False):
return resource

def _verifyBooleanConfigProperties(self, job, config):
jconfig = job.configuration
if 'allowJaggedRows' in config:
self.assertEqual(jconfig.allow_jagged_rows,
self.assertEqual(job.allow_jagged_rows,
config['allowJaggedRows'])
else:
self.assertIsNone(jconfig.allow_jagged_rows)
self.assertIsNone(job.allow_jagged_rows)
if 'allowQuotedNewlines' in config:
self.assertEqual(jconfig.allow_quoted_newlines,
self.assertEqual(job.allow_quoted_newlines,
config['allowQuotedNewlines'])
else:
self.assertIsNone(jconfig.allow_quoted_newlines)
self.assertIsNone(job.allow_quoted_newlines)
if 'autodetect' in config:
self.assertEqual(
jconfig.autodetect, config['autodetect'])
job.autodetect, config['autodetect'])
else:
self.assertIsNone(jconfig.autodetect)
self.assertIsNone(job.autodetect)
if 'ignoreUnknownValues' in config:
self.assertEqual(jconfig.ignore_unknown_values,
self.assertEqual(job.ignore_unknown_values,
config['ignoreUnknownValues'])
else:
self.assertIsNone(jconfig.ignore_unknown_values)
self.assertIsNone(job.ignore_unknown_values)

def _verifyEnumConfigProperties(self, job, config):
jconfig = job.configuration
if 'createDisposition' in config:
self.assertEqual(jconfig.create_disposition,
self.assertEqual(job.create_disposition,
config['createDisposition'])
else:
self.assertIsNone(jconfig.create_disposition)
self.assertIsNone(job.create_disposition)
if 'encoding' in config:
self.assertEqual(jconfig.encoding,
self.assertEqual(job.encoding,
config['encoding'])
else:
self.assertIsNone(jconfig.encoding)
self.assertIsNone(job.encoding)
if 'sourceFormat' in config:
self.assertEqual(jconfig.source_format,
self.assertEqual(job.source_format,
config['sourceFormat'])
else:
self.assertIsNone(jconfig.source_format)
self.assertIsNone(job.source_format)
if 'writeDisposition' in config:
self.assertEqual(jconfig.write_disposition,
self.assertEqual(job.write_disposition,
config['writeDisposition'])
else:
self.assertIsNone(jconfig.write_disposition)
self.assertIsNone(job.write_disposition)

def _verifyResourceProperties(self, job, resource):
jconfig = job.configuration
self._verifyReadonlyResourceProperties(job, resource)

config = resource.get('configuration', {}).get('load')
Expand All @@ -296,30 +293,30 @@ def _verifyResourceProperties(self, job, resource):
self.assertEqual(job.destination.table_id, table_ref['tableId'])

if 'fieldDelimiter' in config:
self.assertEqual(jconfig.field_delimiter,
self.assertEqual(job.field_delimiter,
config['fieldDelimiter'])
else:
self.assertIsNone(jconfig.field_delimiter)
self.assertIsNone(job.field_delimiter)
if 'maxBadRecords' in config:
self.assertEqual(jconfig.max_bad_records,
self.assertEqual(job.max_bad_records,
config['maxBadRecords'])
else:
self.assertIsNone(jconfig.max_bad_records)
self.assertIsNone(job.max_bad_records)
if 'nullMarker' in config:
self.assertEqual(jconfig.null_marker,
self.assertEqual(job.null_marker,
config['nullMarker'])
else:
self.assertIsNone(jconfig.null_marker)
self.assertIsNone(job.null_marker)
if 'quote' in config:
self.assertEqual(jconfig.quote_character,
self.assertEqual(job.quote_character,
config['quote'])
else:
self.assertIsNone(jconfig.quote_character)
self.assertIsNone(job.quote_character)
if 'skipLeadingRows' in config:
self.assertEqual(str(jconfig.skip_leading_rows),
self.assertEqual(str(job.skip_leading_rows),
config['skipLeadingRows'])
else:
self.assertIsNone(jconfig.skip_leading_rows)
self.assertIsNone(job.skip_leading_rows)

def test_ctor(self):
client = _Client(self.PROJECT)
Expand All @@ -332,7 +329,7 @@ def test_ctor(self):
self.assertEqual(
job.path,
'/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME))
self.assertEqual(job.configuration.schema, [])
self.assertEqual(job.schema, [])

self._verifyInitialReadonlyProperties(job)

Expand All @@ -343,20 +340,19 @@ def test_ctor(self):
self.assertIsNone(job.output_rows)

# set/read from resource['configuration']['load']
jconfig = job.configuration
self.assertIsNone(jconfig.allow_jagged_rows)
self.assertIsNone(jconfig.allow_quoted_newlines)
self.assertIsNone(jconfig.autodetect)
self.assertIsNone(jconfig.create_disposition)
self.assertIsNone(jconfig.encoding)
self.assertIsNone(jconfig.field_delimiter)
self.assertIsNone(jconfig.ignore_unknown_values)
self.assertIsNone(jconfig.max_bad_records)
self.assertIsNone(jconfig.null_marker)
self.assertIsNone(jconfig.quote_character)
self.assertIsNone(jconfig.skip_leading_rows)
self.assertIsNone(jconfig.source_format)
self.assertIsNone(jconfig.write_disposition)
self.assertIsNone(job.allow_jagged_rows)
self.assertIsNone(job.allow_quoted_newlines)
self.assertIsNone(job.autodetect)
self.assertIsNone(job.create_disposition)
self.assertIsNone(job.encoding)
self.assertIsNone(job.field_delimiter)
self.assertIsNone(job.ignore_unknown_values)
self.assertIsNone(job.max_bad_records)
self.assertIsNone(job.null_marker)
self.assertIsNone(job.quote_character)
self.assertIsNone(job.skip_leading_rows)
self.assertIsNone(job.source_format)
self.assertIsNone(job.write_disposition)

def test_ctor_w_config(self):
from google.cloud.bigquery.schema import SchemaField
Expand All @@ -368,7 +364,7 @@ def test_ctor_w_config(self):
config.schema = [full_name, age]
job = self._make_one(self.JOB_NAME, [self.SOURCE1], self.TABLE_REF,
client, config)
self.assertEqual(job.configuration.schema, [full_name, age])
self.assertEqual(job.schema, [full_name, age])

def test_done(self):
client = _Client(self.PROJECT)
Expand Down