-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Refactor BigQuery samples. #297
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,16 +30,14 @@ | |
| import java.util.Collection; | ||
|
|
||
| /** | ||
| * This class creates our Service to connect to Bigquery including auth. | ||
| * This class creates our Service to connect to BigQuery including auth. | ||
| */ | ||
| public final class BigqueryServiceFactory { | ||
| public final class BigQueryServiceFactory { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should the file name be changed?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sigh. Yes. I'm having trouble getting git to accept that the filename is changed (I think because I'm on my Mac) |
||
|
|
||
| /** | ||
| * Private constructor to disable creation of this utility Factory class. | ||
| */ | ||
| private BigqueryServiceFactory() { | ||
|
|
||
| } | ||
| private BigQueryServiceFactory() {} | ||
|
|
||
| /** | ||
| * Singleton service used through the app. | ||
|
|
@@ -52,9 +50,9 @@ private BigqueryServiceFactory() { | |
| private static Object serviceLock = new Object(); | ||
|
|
||
| /** | ||
| * Threadsafe Factory that provides an authorized Bigquery service. | ||
| * @return The Bigquery service | ||
| * @throws IOException Thronw if there is an error connecting to Bigquery. | ||
| * Threadsafe Factory that provides an authorized BigQuery service. | ||
| * @return The BigQuery service | ||
| * @throws IOException Throw if there is an error connecting to BigQuery. | ||
| */ | ||
| public static Bigquery getService() throws IOException { | ||
| if (service == null) { | ||
|
|
@@ -68,7 +66,7 @@ public static Bigquery getService() throws IOException { | |
| } | ||
|
|
||
| /** | ||
| * Creates an authorized client to Google Bigquery. | ||
| * Creates an authorized client to Google BigQuery. | ||
| * | ||
| * @return The BigQuery Service | ||
| * @throws IOException Thrown if there is an error connecting | ||
|
|
@@ -78,18 +76,19 @@ private static Bigquery createAuthorizedClient() throws IOException { | |
| // Create the credential | ||
| HttpTransport transport = new NetHttpTransport(); | ||
| JsonFactory jsonFactory = new JacksonFactory(); | ||
| GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); | ||
| GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); | ||
|
|
||
| // Depending on the environment that provides the default credentials (e.g. Compute Engine, App | ||
| // Engine), the credentials may require us to specify the scopes we need explicitly. | ||
| // Check for this case, and inject the Bigquery scope if required. | ||
| // Check for this case, and inject the BigQuery scope if required. | ||
| if (credential.createScopedRequired()) { | ||
| Collection<String> bigqueryScopes = BigqueryScopes.all(); | ||
| credential = credential.createScoped(bigqueryScopes); | ||
| } | ||
|
|
||
| return new Bigquery.Builder(transport, jsonFactory, credential) | ||
| .setApplicationName("BigQuery Samples").build(); | ||
| .setApplicationName("BigQuery Samples") | ||
| .build(); | ||
| } | ||
| // [END get_service] | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,7 +26,7 @@ | |
| /** | ||
| * Sample of how to Export Cloud Data. | ||
| */ | ||
| public class ExportDataCloudStorageSample { | ||
| public class ExportDataCloudStorageSample { | ||
| /** | ||
| * Protected constructor since this is a collection of static functions. | ||
| */ | ||
|
|
@@ -42,20 +42,17 @@ protected ExportDataCloudStorageSample() { | |
| * @throws InterruptedException Should never be thrown. | ||
| */ | ||
| // [START main] | ||
| public static void main(final String[] args) | ||
| throws IOException, InterruptedException { | ||
| public static void main(final String[] args) throws IOException, InterruptedException { | ||
| Scanner scanner = new Scanner(System.in); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm sure there is a good reason why we don't use command line args, w/ reasonable defaults. |
||
| System.out.println("Enter your project id: "); | ||
| String projectId = scanner.nextLine(); | ||
| System.out.println("Enter your dataset id: "); | ||
| String datasetId = scanner.nextLine(); | ||
| System.out.println("Enter your table id: "); | ||
| String tableId = scanner.nextLine(); | ||
| System.out.println("Enter the Google Cloud Storage Path to which you'd " | ||
| + "like to export: "); | ||
| System.out.println("Enter the Google Cloud Storage Path to which you'd " + "like to export: "); | ||
| String cloudStoragePath = scanner.nextLine(); | ||
| System.out.println("Enter how often to check if your job is complete " | ||
| + "(milliseconds): "); | ||
| System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); | ||
| long interval = scanner.nextLong(); | ||
| scanner.close(); | ||
|
|
||
|
|
@@ -79,30 +76,33 @@ public static void run( | |
| final String projectId, | ||
| final String datasetId, | ||
| final String tableId, | ||
| final long interval) throws IOException, InterruptedException { | ||
| final long interval) | ||
| throws IOException, InterruptedException { | ||
|
|
||
| Bigquery bigquery = BigqueryServiceFactory.getService(); | ||
| Bigquery bigquery = BigQueryServiceFactory.getService(); | ||
|
|
||
| Job extractJob = extractJob( | ||
| bigquery, | ||
| cloudStoragePath, | ||
| new TableReference() | ||
| .setProjectId(projectId) | ||
| .setDatasetId(datasetId) | ||
| .setTableId(tableId)); | ||
| Job extractJob = | ||
| extractJob( | ||
| bigquery, | ||
| cloudStoragePath, | ||
| new TableReference() | ||
| .setProjectId(projectId) | ||
| .setDatasetId(datasetId) | ||
| .setTableId(tableId)); | ||
|
|
||
| Bigquery.Jobs.Get getJob = bigquery.jobs().get( | ||
| extractJob.getJobReference().getProjectId(), | ||
| extractJob.getJobReference().getJobId()); | ||
| Bigquery.Jobs.Get getJob = | ||
| bigquery | ||
| .jobs() | ||
| .get( | ||
| extractJob.getJobReference().getProjectId(), | ||
| extractJob.getJobReference().getJobId()); | ||
|
|
||
| BigqueryUtils.pollJob(getJob, interval); | ||
| BigQueryUtils.pollJob(getJob, interval); | ||
|
|
||
| System.out.println("Export is Done!"); | ||
|
|
||
| } | ||
| // [END run] | ||
|
|
||
|
|
||
| /** | ||
| * A job that extracts data from a table. | ||
| * @param bigquery Bigquery service to use | ||
|
|
@@ -113,16 +113,17 @@ public static void run( | |
| */ | ||
| // [START extract_job] | ||
| public static Job extractJob( | ||
| final Bigquery bigquery, | ||
| final String cloudStoragePath, | ||
| final TableReference table) throws IOException { | ||
| final Bigquery bigquery, final String cloudStoragePath, final TableReference table) | ||
| throws IOException { | ||
|
|
||
| JobConfigurationExtract extract = new JobConfigurationExtract() | ||
| .setSourceTable(table) | ||
| .setDestinationUri(cloudStoragePath); | ||
| JobConfigurationExtract extract = | ||
| new JobConfigurationExtract().setSourceTable(table).setDestinationUri(cloudStoragePath); | ||
|
|
||
| return bigquery.jobs().insert(table.getProjectId(), | ||
| new Job().setConfiguration(new JobConfiguration().setExtract(extract))) | ||
| return bigquery | ||
| .jobs() | ||
| .insert( | ||
| table.getProjectId(), | ||
| new Job().setConfiguration(new JobConfiguration().setExtract(extract))) | ||
| .execute(); | ||
| } | ||
| // [END extract_job] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Some reason these aren't command line args, with defaults?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That was the way it was in the existing sample, and it didn't bother me as much as some of the other things I fixed. I can look at changing this, too.