diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java index b8c59214005..a7df34604da 100644 --- a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java @@ -54,7 +54,7 @@ public static void createEntry(String projectId, String entryGroupId, String ent .setDisplayName("My Fileset") .setDescription("This fileset consists of ....") .setGcsFilesetSpec( - GcsFilesetSpec.newBuilder().addFilePatterns("gs://my_bucket/*").build()) + GcsFilesetSpec.newBuilder().addFilePatterns("gs://cloud-samples-data/*").build()) .setSchema( Schema.newBuilder() .addColumns( diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java index 2651606c89c..04a312e9e7b 100644 --- a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java @@ -17,12 +17,15 @@ package com.example.datacatalog; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; import com.google.cloud.datacatalog.EntryGroupName; import com.google.cloud.datacatalog.EntryName; import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; import org.hamcrest.CoreMatchers; import org.junit.After; @@ -38,15 +41,12 @@ public class CreateEntryTests { private ByteArrayOutputStream bout; - private static String ENTRY_GROUP_ID_NO_CHILDREN = - "entry_group_no_children_" + UUID.randomUUID().toString().substring(0, 8); - private static String PARENT_ENTRY_GROUP_ID = - "fileset_entry_group_parent_" + UUID.randomUUID().toString().substring(0, 8); - private static String ENTRY_ID = - "fileset_entry_id_" + UUID.randomUUID().toString().substring(0, 8); private static String LOCATION = "us-central1"; private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT"); + private static List entryGroupsPendingDeletion = new ArrayList<>(); + private static List entriesPendingDeletion = new ArrayList<>(); + @Before public void setUp() { bout = new ByteArrayOutputStream(); @@ -62,13 +62,18 @@ public void tearDown() { @AfterClass public static void tearDownClass() { try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { - dataCatalogClient.deleteEntryGroup( - EntryGroupName.of(PROJECT_ID, LOCATION, ENTRY_GROUP_ID_NO_CHILDREN).toString()); - - dataCatalogClient.deleteEntry( - EntryName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID, ENTRY_ID).toString()); - dataCatalogClient.deleteEntryGroup( - EntryGroupName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID).toString()); + // Must delete Entries before deleting the Entry Group. + if (entriesPendingDeletion.isEmpty() || entryGroupsPendingDeletion.isEmpty()) { + fail("Something went wrong, no entries were generated"); + } + + for (String entryName : entriesPendingDeletion) { + dataCatalogClient.deleteEntry(entryName); + } + + for (String entryGroupName : entryGroupsPendingDeletion) { + dataCatalogClient.deleteEntryGroup(entryGroupName); + } } catch (Exception e) { System.out.println("Error in cleaning up test data:\n" + e.toString()); } @@ -76,31 +81,48 @@ public static void tearDownClass() { @Test public void testCreateFilesetEntry() { + String entryGroupId = "fileset_entry_group_parent_" + getUuid8Chars(); + String entryId = "fileset_entry_id_" + getUuid8Chars(); + // Must create a Entry Group before creating the entry. - CreateEntryGroup.createEntryGroup(PROJECT_ID, PARENT_ENTRY_GROUP_ID); - CreateFilesetEntry.createEntry(PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID); + CreateEntryGroup.createEntryGroup(PROJECT_ID, entryGroupId); + CreateFilesetEntry.createEntry(PROJECT_ID, entryGroupId, entryId); + + // Store names for clean up on teardown + String expectedEntryGroupName = + EntryGroupName.of(PROJECT_ID, LOCATION, entryGroupId).toString(); + entryGroupsPendingDeletion.add(expectedEntryGroupName); + + String expectedEntryName = EntryName.of(PROJECT_ID, LOCATION, entryGroupId, entryId).toString(); + entriesPendingDeletion.add(expectedEntryName); String output = bout.toString(); - String entryTemplate = - "Entry created with name: projects/%s/locations/us-central1/entryGroups/%s/entries/%s"; + String entryTemplate = "Entry created with name: %s"; assertThat( - output, - CoreMatchers.containsString( - String.format(entryTemplate, PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID))); + output, CoreMatchers.containsString(String.format(entryTemplate, expectedEntryName))); } @Test public void testCreateEntryGroup() { - CreateEntryGroup.createEntryGroup(PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN); + String entryGroupId = "entry_group_no_children_" + getUuid8Chars(); + + CreateEntryGroup.createEntryGroup(PROJECT_ID, entryGroupId); + + // Store names for clean up on teardown + String expectedEntryGroupName = + EntryGroupName.of(PROJECT_ID, LOCATION, entryGroupId).toString(); + entryGroupsPendingDeletion.add(expectedEntryGroupName); String output = bout.toString(); - String entryGroupTemplate = - "Entry Group created with name: projects/%s/locations/us-central1/entryGroups/%s"; + String entryGroupTemplate = "Entry Group created with name: %s"; assertThat( output, - CoreMatchers.containsString( - String.format(entryGroupTemplate, PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN))); + CoreMatchers.containsString(String.format(entryGroupTemplate, expectedEntryGroupName))); + } + + private String getUuid8Chars() { + return UUID.randomUUID().toString().substring(0, 8); } -} \ No newline at end of file +} diff --git a/datacatalog/quickstart/README.md b/datacatalog/quickstart/README.md new file mode 100644 index 00000000000..f4128e2c4b0 --- /dev/null +++ b/datacatalog/quickstart/README.md @@ -0,0 +1,31 @@ +# Getting Started with Data Catalog and the Google Cloud Client libraries + + +Open in Cloud Shell + +[Data Catalog][datacatalog] is a fully managed and scalable metadata management service that empowers organizations +to quickly discover, manage, and understand all their data in Google Cloud. +This sample Java application demonstrates how to access the Data Catalog API using +the [Google Cloud Client Library for Java][google-cloud-java]. + +[datacatalog]: https://cloud.google.com/data-catalog/ +[google-cloud-java]: https://github.com/GoogleCloudPlatform/google-cloud-java + +## Quickstart + +#### Setup +- Install [Maven](http://maven.apache.org/). +- [Enable](https://console.cloud.google.com/apis/api/datacatalog.googleapis.com/overview) Data Catalog API. +- Set up [authentication](https://cloud.google.com/docs/authentication/getting-started). + +#### Build +- Build your project with: +``` + mvn clean package -DskipTests +``` + +#### Testing +Run the test with Maven. +``` + mvn verify +``` diff --git a/datacatalog/quickstart/build.gradle b/datacatalog/quickstart/build.gradle new file mode 100644 index 00000000000..7c37191be55 --- /dev/null +++ b/datacatalog/quickstart/build.gradle @@ -0,0 +1,41 @@ +// Copyright 2019 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +apply plugin: 'java' + +repositories { + mavenCentral() +} + +dependencies { + compile group: 'com.google.cloud', name: 'google-cloud-datacatalog-quickstart', version:'0.28.0-alpha' + + testCompile group: 'com.google.truth', name: 'truth', version:'0.42' + testCompile group: 'junit', name: 'junit', version:'4.13-beta-2' +} + +test { + useJUnit() + testLogging.showStandardStreams = true + beforeTest { descriptor -> + logger.lifecycle("test: " + descriptor + " Running") + } + + onOutput { descriptor, event -> + logger.lifecycle("test: " + descriptor + ": " + event.message ) + } + afterTest { descriptor, result -> + logger.lifecycle("test: " + descriptor + ": " + result ) + } +} diff --git a/datacatalog/quickstart/pom.xml b/datacatalog/quickstart/pom.xml new file mode 100644 index 00000000000..7c5a38c93a5 --- /dev/null +++ b/datacatalog/quickstart/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + com.example.datacatalog + datacatalog-google-cloud-quickstart + jar + + + + com.google.cloud.samples + shared-configuration + 1.0.11 + + + + 1.8 + 1.8 + + + + + com.google.cloud + google-cloud-datacatalog + 0.29.0-alpha + + + + + junit + junit + 4.13-beta-3 + test + + + com.google.truth + truth + 1.0 + test + + + diff --git a/datacatalog/quickstart/src/main/java/com/example/datacatalog/CreateFilesetEntry.java b/datacatalog/quickstart/src/main/java/com/example/datacatalog/CreateFilesetEntry.java new file mode 100644 index 00000000000..90ac49add0e --- /dev/null +++ b/datacatalog/quickstart/src/main/java/com/example/datacatalog/CreateFilesetEntry.java @@ -0,0 +1,171 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +// [START datacatalog_create_fileset_quickstart_tag] + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.cloud.datacatalog.ColumnSchema; +import com.google.cloud.datacatalog.CreateEntryGroupRequest; +import com.google.cloud.datacatalog.CreateEntryRequest; +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.EntryGroup; +import com.google.cloud.datacatalog.EntryGroupName; +import com.google.cloud.datacatalog.EntryName; +import com.google.cloud.datacatalog.EntryType; +import com.google.cloud.datacatalog.GcsFilesetSpec; +import com.google.cloud.datacatalog.LocationName; +import com.google.cloud.datacatalog.Schema; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; +import java.io.IOException; + +public class CreateFilesetEntry { + + public static void createEntry() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String entryGroupId = "fileset_entry_group"; + String entryId = "fileset_entry_id"; + createEntry(projectId, entryGroupId, entryId); + } + + // Create Fileset Entry. + public static void createEntry(String projectId, String entryGroupId, String entryId) { + // Currently, Data Catalog stores metadata in the us-central1 region. + String location = "us-central1"; + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + + // 1. Environment cleanup: delete pre-existing data. + // Delete any pre-existing Entry with the same name + // that will be used in step 3. + try { + dataCatalogClient.deleteEntry( + EntryName.of(projectId, location, entryGroupId, entryId).toString()); + } catch (PermissionDeniedException | NotFoundException e) { + // PermissionDeniedException or NotFoundException are thrown if + // Entry does not exist. + System.out.println("Entry does not exist."); + } + + // Delete any pre-existing Entry Group with the same name + // that will be used in step 2. + try { + dataCatalogClient.deleteEntryGroup( + EntryGroupName.of(projectId, location, entryGroupId).toString()); + } catch (PermissionDeniedException | NotFoundException e) { + // PermissionDeniedException or NotFoundException are thrown if + // Entry Group does not exist. + System.out.println("Entry Group does not exist."); + } + + // 2. Create an Entry Group. + // Construct the EntryGroup for the EntryGroup request. + EntryGroup entryGroup = + EntryGroup.newBuilder() + .setDisplayName("My Fileset Entry Group") + .setDescription("This Entry Group consists of ....") + .build(); + + // Construct the EntryGroup request to be sent by the client. + CreateEntryGroupRequest entryGroupRequest = + CreateEntryGroupRequest.newBuilder() + .setParent(LocationName.of(projectId, location).toString()) + .setEntryGroupId(entryGroupId) + .setEntryGroup(entryGroup) + .build(); + + // Use the client to send the API request. + EntryGroup entryGroupResponse = dataCatalogClient.createEntryGroup(entryGroupRequest); + + System.out.printf("\nEntry Group created with name: %s\n", entryGroupResponse.getName()); + + // 3. Create a Fileset Entry. + // Construct the Entry for the Entry request. + Entry entry = + Entry.newBuilder() + .setDisplayName("My Fileset") + .setDescription("This fileset consists of ....") + .setGcsFilesetSpec( + GcsFilesetSpec.newBuilder().addFilePatterns("gs://cloud-samples-data/*").build()) + .setSchema( + Schema.newBuilder() + .addColumns( + ColumnSchema.newBuilder() + .setColumn("first_name") + .setDescription("First name") + .setMode("REQUIRED") + .setType("STRING") + .build()) + .addColumns( + ColumnSchema.newBuilder() + .setColumn("last_name") + .setDescription("Last name") + .setMode("REQUIRED") + .setType("STRING") + .build()) + .addColumns( + ColumnSchema.newBuilder() + .setColumn("addresses") + .setDescription("Addresses") + .setMode("REPEATED") + .setType("RECORD") + .addSubcolumns( + ColumnSchema.newBuilder() + .setColumn("city") + .setDescription("City") + .setMode("NULLABLE") + .setType("STRING") + .build()) + .addSubcolumns( + ColumnSchema.newBuilder() + .setColumn("state") + .setDescription("State") + .setMode("NULLABLE") + .setType("STRING") + .build()) + .build()) + .build()) + .setType(EntryType.FILESET) + .build(); + + // Construct the Entry request to be sent by the client. + CreateEntryRequest entryRequest = + CreateEntryRequest.newBuilder() + .setParent(entryGroupResponse.getName()) + .setEntryId(entryId) + .setEntry(entry) + .build(); + + // Use the client to send the API request. + Entry entryResponse = dataCatalogClient.createEntry(entryRequest); + + System.out.printf("\nEntry created with name: %s\n", entryResponse.getName()); + } catch (AlreadyExistsException | IOException e) { + // AlreadyExistsException's are thrown if EntryGroup or Entry already exists. + // IOException's are thrown when unable to create the DataCatalogClient, + // for example an invalid Service Account path. + System.out.println("Error in create entry process:\n" + e.toString()); + } + } +} +// [END datacatalog_create_fileset_quickstart_tag] diff --git a/datacatalog/quickstart/src/test/java/com/example/datacatalog/CreateFilesetEntryTests.java b/datacatalog/quickstart/src/test/java/com/example/datacatalog/CreateFilesetEntryTests.java new file mode 100644 index 00000000000..760f6ef24d9 --- /dev/null +++ b/datacatalog/quickstart/src/test/java/com/example/datacatalog/CreateFilesetEntryTests.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +import com.google.cloud.datacatalog.EntryGroupName; +import com.google.cloud.datacatalog.EntryName; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration (system) tests for {@link CreateFilesetEntry}. */ +@RunWith(JUnit4.class) +public class CreateFilesetEntryTests { + + private ByteArrayOutputStream bout; + + private static String LOCATION = "us-central1"; + private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT"); + + private static List entryGroupsPendingDeletion = new ArrayList<>(); + private static List entriesPendingDeletion = new ArrayList<>(); + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @After + public void tearDown() { + System.setOut(null); + bout.reset(); + } + + @AfterClass + public static void tearDownClass() { + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + // Must delete Entries before deleting the Entry Group. + if (entriesPendingDeletion.isEmpty() || entryGroupsPendingDeletion.isEmpty()) { + fail("Something went wrong, no entries were generated"); + } + + for (String entryName : entriesPendingDeletion) { + dataCatalogClient.deleteEntry(entryName); + } + + for (String entryGroupName : entryGroupsPendingDeletion) { + dataCatalogClient.deleteEntryGroup(entryGroupName); + } + } catch (Exception e) { + System.out.println("Error in cleaning up test data:\n" + e.toString()); + } + } + + @Test + public void testCreateEntryQuickStart() { + String entryGroupId = "fileset_entry_group_parent_" + getUuid8Chars(); + String entryId = "fileset_entry_id_" + getUuid8Chars(); + + CreateFilesetEntry.createEntry(PROJECT_ID, entryGroupId, entryId); + + // Store names for clean up on teardown + String expectedEntryGroupName = + EntryGroupName.of(PROJECT_ID, LOCATION, entryGroupId).toString(); + entryGroupsPendingDeletion.add(expectedEntryGroupName); + + String expectedEntryName = EntryName.of(PROJECT_ID, LOCATION, entryGroupId, entryId).toString(); + entriesPendingDeletion.add(expectedEntryName); + + String output = bout.toString(); + + String entryTemplate = "Entry created with name: %s"; + String entryGroupTemplate = "Entry Group created with name: %s"; + assertThat( + output, + CoreMatchers.containsString(String.format(entryGroupTemplate, expectedEntryGroupName))); + assertThat( + output, CoreMatchers.containsString(String.format(entryTemplate, expectedEntryName))); + } + + private String getUuid8Chars() { + return UUID.randomUUID().toString().substring(0, 8); + } +}