From baf7df5f77d17752e35860d186c88bb5c4dc139d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 5 Apr 2016 16:52:11 -0700 Subject: [PATCH] Add Cloud Natural Language API Java sample. This sample shows how to use the Cloud Natural Language API to do entity recognition. The client libraries are vendored in while we are in alpha. Change-Id: I037901017d0ffb7ffc73cc78c43badaff1dffd3c --- language/README.md | 39 ++++ language/analysis/README.md | 53 +++++ language/analysis/pom.xml | 93 ++++++++ .../cloud/language/samples/Analyze.java | 209 ++++++++++++++++++ .../cloud/language/samples/AnalyzeIT.java | 95 ++++++++ .../cloud/language/samples/AnalyzeTest.java | 123 +++++++++++ pom.xml | 1 + 7 files changed, 613 insertions(+) create mode 100644 language/README.md create mode 100644 language/analysis/README.md create mode 100644 language/analysis/pom.xml create mode 100644 language/analysis/src/main/java/com/google/cloud/language/samples/Analyze.java create mode 100644 language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeIT.java create mode 100644 language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeTest.java diff --git a/language/README.md b/language/README.md new file mode 100644 index 00000000000..2d20460d61c --- /dev/null +++ b/language/README.md @@ -0,0 +1,39 @@ +# Google Cloud Natural Language API Samples + +These samples demonstrate the use of the [Google Cloud Natural Language API][NL-Docs]. + +[NL-Docs]: https://cloud.google.com/language/docs/ + +## Prerequisites + +### Download Maven + +This sample uses the [Apache Maven][maven] build system. Before getting started, be +sure to [download][maven-download] and [install][maven-install] it. When you use +Maven as described here, it will automatically download the needed client +libraries. + +[maven]: https://maven.apache.org +[maven-download]: https://maven.apache.org/download.cgi +[maven-install]: https://maven.apache.org/install.html + +### Set Up to Authenticate With Your Project's Credentials + +Please follow the [Set Up Your Project](https://cloud.google.com/natural-language/docs/getting-started#set_up_your_project) +steps in the Quickstart doc to create a project and enable the +Cloud Natural Language API. Following those steps, make sure that you +[Set Up a Service Account](https://cloud.google.com/natural-language/docs/common/auth#set_up_a_service_account), +and export the following environment variable: + +``` +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your-project-credentials.json +``` + +[cloud-console]: https://console.cloud.google.com +[language-api]: https://console.cloud.google.com/apis/api/language.googleapis.com/overview?project=_ +[adc]: https://cloud.google.com/docs/authentication#developer_workflow + +## Samples + +- [Analyze](analysis) is a command line tool to show case the features of the API. + diff --git a/language/analysis/README.md b/language/analysis/README.md new file mode 100644 index 00000000000..f9c224faedb --- /dev/null +++ b/language/analysis/README.md @@ -0,0 +1,53 @@ +# Google Cloud Natural Language API Entity Recognition Sample + +This sample demonstrates the use of the [Google Cloud Natural Language API][NL-Docs] +for entity recognition. + +[NL-Docs]: https://cloud.google.com/language/docs/ + +## Java Version + +This sample requires you to have +[Java8](https://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html). + +## Download Maven + +This sample uses the [Apache Maven][maven] build system. Before getting started, be +sure to [download][maven-download] and [install][maven-install] it. When you use +Maven as described here, it will automatically download the needed client +libraries. + +[maven]: https://maven.apache.org +[maven-download]: https://maven.apache.org/download.cgi +[maven-install]: https://maven.apache.org/install.html + +## Run the sample + +To build the sample, we use Maven. + +```bash +mvn clean compile assembly:single +``` + +We can then run the assembled JAR file with the `java` command. The variable $COMMAND takes +three values `entities`, `sentiment` or `syntax`. + +``` +MAIN_CLASS=com.google.cloud.language.samples.Analyze +JAR_FILE=target/entities-1.0-SNAPSHOT-jar-with-dependencies.jar +java -cp $JAR_FILE $MAIN_CLASS +``` + +Example usage: + +``` +QUOTE="Larry Page, Google's co-founder, once described the 'perfect search + engine' as something that 'understands exactly what you mean and gives you + back exactly what you want.' Since he spoke those words Google has grown to + offer products beyond search, but the spirit of what he said remains." + +java -cp $JAR_FILE $MAIN_CLASS entities "$QUOTE" +java -cp $JAR_FILE $MAIN_CLASS sentiment "$QUOTE" +java -cp $JAR_FILE $MAIN_CLASS syntax "$QUOTE" +``` + diff --git a/language/analysis/pom.xml b/language/analysis/pom.xml new file mode 100644 index 00000000000..86cfa47fe75 --- /dev/null +++ b/language/analysis/pom.xml @@ -0,0 +1,93 @@ + + + 4.0.0 + jar + 1.0-SNAPSHOT + com.google.cloud.language.samples + entities + + + + + com.google.apis + google-api-services-language + v1beta1-rev1-1.22.0 + + + com.google.api-client + google-api-client + 1.21.0 + + + com.google.guava + guava + 19.0 + + + + + + junit + junit + 4.12 + + + com.google.truth + truth + 0.28 + + + + + + maven-assembly-plugin + + + + com.google.cloud.language.samples.entities.AnalyzeEntitiesApp + + + + jar-with-dependencies + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.18.1 + + + + integration-test + verify + + + + + + org.apache.maven.plugins + 3.3 + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/language/analysis/src/main/java/com/google/cloud/language/samples/Analyze.java b/language/analysis/src/main/java/com/google/cloud/language/samples/Analyze.java new file mode 100644 index 00000000000..104550bad22 --- /dev/null +++ b/language/analysis/src/main/java/com/google/cloud/language/samples/Analyze.java @@ -0,0 +1,209 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.language.samples; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.language.v1beta1.CloudNaturalLanguageAPI; +import com.google.api.services.language.v1beta1.CloudNaturalLanguageAPIScopes; +import com.google.api.services.language.v1beta1.model.AnalyzeEntitiesRequest; +import com.google.api.services.language.v1beta1.model.AnalyzeEntitiesResponse; +import com.google.api.services.language.v1beta1.model.AnalyzeSentimentRequest; +import com.google.api.services.language.v1beta1.model.AnalyzeSentimentResponse; +import com.google.api.services.language.v1beta1.model.AnnotateTextRequest; +import com.google.api.services.language.v1beta1.model.AnnotateTextResponse; +import com.google.api.services.language.v1beta1.model.Document; +import com.google.api.services.language.v1beta1.model.Entity; +import com.google.api.services.language.v1beta1.model.Features; +import com.google.api.services.language.v1beta1.model.Sentiment; +import com.google.api.services.language.v1beta1.model.Token; + +import java.io.IOException; +import java.io.PrintStream; +import java.security.GeneralSecurityException; +import java.util.List; +import java.util.Map; + +/** + * A sample application that uses the Natural Language API to perform + * entity, sentiment and syntax analysis. + */ +@SuppressWarnings("serial") +public class Analyze { + /** + * Be sure to specify the name of your application. If the application name is {@code null} or + * blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0". + */ + private static final String APPLICATION_NAME = "Google-LanguagAPISample/1.0"; + + private static final int MAX_RESULTS = 4; + + /** + * Detects entities,sentiment and syntax in a document using the Natural Language API. + */ + public static void main(String[] args) throws IOException, GeneralSecurityException { + if (args.length != 2) { + System.err.println("Usage:"); + System.err.printf( + "\tjava %s \"command\" \"text to analyze\"\n", + Analyze.class.getCanonicalName()); + System.exit(1); + } + String command = args[0]; + String text = args[1]; + + Analyze app = new Analyze(getLanguageService()); + + if (command.equals("entities")) { + printEntities(System.out, app.analyzeEntities(text)); + } else if (command.equals("sentiment")) { + printSentiment(System.out, app.analyzeSentiment(text)); + } else if (command.equals("syntax")) { + printSyntax(System.out, app.analyzeSyntax(text)); + } + } + + /** + * Print a list of {@code entities}. + */ + public static void printEntities(PrintStream out, List entities) { + if (entities == null || entities.size() == 0) { + out.println("No entities found."); + return; + } + out.printf("Found %d entit%s.\n", entities.size(), entities.size() == 1 ? "y" : "ies"); + for (Entity entity : entities) { + out.printf("%s\n", entity.getName()); + out.printf("\tSalience: %.3f\n", entity.getSalience()); + out.printf("\tType: %s\n", entity.getType()); + if (entity.getMetadata() != null) { + for (Map.Entry metadata : entity.getMetadata().entrySet()) { + out.printf("\tMetadata: %s = %s\n", metadata.getKey(), metadata.getValue()); + } + } + } + } + + /** + * Print the Sentiment {@code sentiment}. + */ + public static void printSentiment(PrintStream out, Sentiment sentiment) { + if (sentiment == null) { + out.println("No sentiment found"); + return; + } + out.println("Found sentiment."); + out.printf("\tMagnitude: %.3f\n", sentiment.getMagnitude()); + out.printf("\tPolarity: %.3f\n", sentiment.getPolarity()); + } + + public static void printSyntax(PrintStream out, List tokens) { + if (tokens == null || tokens.size() == 0) { + out.println("No syntax found"); + return; + } + out.printf("Found %d token%s.\n", tokens.size(), tokens.size() == 1 ? "" : "s"); + for (Token token : tokens) { + out.println("TextSpan"); + out.printf("\tText: %s\n", token.getText().getContent()); + out.printf("\tBeginOffset: %d\n", token.getText().getBeginOffset()); + out.printf("Lemma: %s\n", token.getLemma()); + out.printf("PartOfSpeechTag: %s\n", token.getPartOfSpeech().getTag()); + out.println("DependencyEdge"); + out.printf("\tHeadTokenIndex: %d\n", token.getDependencyEdge().getHeadTokenIndex()); + out.printf("\tLabel: %s\n", token.getDependencyEdge().getLabel()); + } + } + + /** + * Connects to the Natural Language API using Application Default Credentials. + */ + public static CloudNaturalLanguageAPI getLanguageService() + throws IOException, GeneralSecurityException { + GoogleCredential credential = + GoogleCredential.getApplicationDefault().createScoped(CloudNaturalLanguageAPIScopes.all()); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + return new CloudNaturalLanguageAPI.Builder( + GoogleNetHttpTransport.newTrustedTransport(), + jsonFactory, new HttpRequestInitializer() { + @Override + public void initialize(HttpRequest request) throws IOException { + credential.initialize(request); + } + }) + .setApplicationName(APPLICATION_NAME) + .build(); + } + + private final CloudNaturalLanguageAPI languageApi; + + /** + * Constructs a {@link Analyze} which connects to the Cloud Natural Language API. + */ + public Analyze(CloudNaturalLanguageAPI languageApi) { + this.languageApi = languageApi; + } + + /** + * Gets {@link Entity}s from the string {@code text}. + */ + public List analyzeEntities(String text) throws IOException { + AnalyzeEntitiesRequest request = + new AnalyzeEntitiesRequest() + .setDocument(new Document().setContent(text).setType("PLAIN_TEXT")) + .setEncodingType("UTF16"); + CloudNaturalLanguageAPI.Documents.AnalyzeEntities analyze = + languageApi.documents().analyzeEntities(request); + + AnalyzeEntitiesResponse response = analyze.execute(); + return response.getEntities(); + } + + /** + * Gets {@link Sentiment} from the string {@code text}. + */ + public Sentiment analyzeSentiment(String text) throws IOException { + AnalyzeSentimentRequest request = + new AnalyzeSentimentRequest() + .setDocument(new Document().setContent(text).setType("PLAIN_TEXT")); + CloudNaturalLanguageAPI.Documents.AnalyzeSentiment analyze = + languageApi.documents().analyzeSentiment(request); + + AnalyzeSentimentResponse response = analyze.execute(); + return response.getDocumentSentiment(); + } + + /** + * Gets {@link Token}s from the string {@code text}. + */ + public List analyzeSyntax(String text) throws IOException { + AnnotateTextRequest request = + new AnnotateTextRequest() + .setDocument(new Document().setContent(text).setType("PLAIN_TEXT")) + .setFeatures(new Features().setExtractSyntax(true)) + .setEncodingType("UTF16"); + CloudNaturalLanguageAPI.Documents.AnnotateText analyze = + languageApi.documents().annotateText(request); + + AnnotateTextResponse response = analyze.execute(); + return response.getTokens(); + } +} diff --git a/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeIT.java b/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeIT.java new file mode 100644 index 00000000000..adc97efe112 --- /dev/null +++ b/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeIT.java @@ -0,0 +1,95 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.language.samples; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.services.language.v1beta1.model.Entity; +import com.google.api.services.language.v1beta1.model.Sentiment; +import com.google.api.services.language.v1beta1.model.Token; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Integration (system) tests for {@link Analyze}. + */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class AnalyzeIT { + + private Analyze analyzeApp; + + @Before public void setup() throws Exception { + analyzeApp = new Analyze(Analyze.getLanguageService()); + } + + @Test public void analyzeEntities_withEntities_returnsLarryPage() throws Exception { + // Act + List entities = + analyzeApp.analyzeEntities( + "Larry Page, Google's co-founder, once described the 'perfect search engine' as" + + " something that 'understands exactly what you mean and gives you back exactly what" + + " you want.' Since he spoke those words Google has grown to offer products beyond" + + " search, but the spirit of what he said remains."); + List got = entities.stream().map(e -> e.getName()).collect(Collectors.toList()); + + // Assert + assertThat(got).named("entity names").contains("Larry Page"); + } + + @Test public void analyzeSentiment_returnPositive() throws Exception { + // Act + Sentiment sentiment = + analyzeApp.analyzeSentiment( + "Tom Cruise is one of the finest actors in hollywood and a great star!"); + + // Assert + assertThat((double)sentiment.getMagnitude()).isGreaterThan(0.0); + assertThat((double)sentiment.getPolarity()).isEqualTo(1.0); + } + + @Test public void analyzeSentiment_returnNegative() throws Exception { + // Act + Sentiment sentiment = + analyzeApp.analyzeSentiment( + "John was seriously injured in an accident"); + + // Assert + assertThat((double)sentiment.getMagnitude()).isGreaterThan(0.0); + assertThat((double)sentiment.getPolarity()).isEqualTo(-1.0); + } + + @Test public void analyzeSyntax_partOfSpeech() throws Exception { + // Act + List token = + analyzeApp.analyzeSyntax( + "President Obama was elected for the second term"); + + List got = token.stream().map(e -> e.getPartOfSpeech().getTag()) + .collect(Collectors.toList()); + + // Assert + assertThat(got).containsExactly("NOUN", "NOUN", "VERB", + "VERB", "ADP", "DET", "ADJ", "NOUN").inOrder(); + } +} diff --git a/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeTest.java b/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeTest.java new file mode 100644 index 00000000000..e38da64af06 --- /dev/null +++ b/language/analysis/src/test/java/com/google/cloud/language/samples/AnalyzeTest.java @@ -0,0 +1,123 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.language.samples; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.services.language.v1beta1.model.Entity; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + * Unit tests for {@link Analyze}. + */ +@RunWith(JUnit4.class) +public class AnalyzeTest { + private static final int MAX_RESULTS = 3; + + @Test public void printEntities_withNull_printsNoneFound() throws Exception { + // Arrange + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + + // Act + Analyze.printEntities(out, null); + + // Assert + String got = bout.toString(); + assertThat(got).contains("No entities found"); + } + + @Test public void printEntities_withEmpty_printsNoneFound() throws Exception { + // Arrange + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + ImmutableList entities = ImmutableList.of(); + + // Act + Analyze.printEntities(out, entities); + + // Assert + String got = bout.toString(); + assertThat(got).contains("No entities found"); + } + + @Test public void printEntities_withEntities_printsEntities() throws Exception { + // Arrange + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + ImmutableList entities = + ImmutableList.of( + new Entity().setName("Larry Page").setSalience(0.426f).setType("PERSON").setMetadata( + ImmutableMap.builder() + .put("knowledge_graph_mid", "/m/0gjpq") + .put("wikipedia_url", "http://en.wikipedia.org/wiki/index.html?curid=60903") + .build()), + new Entity().setName("search engine").setSalience(0.188f).setType("CONSUMER_GOOD"), + new Entity().setName("something")); + + // Act + Analyze.printEntities(out, entities); + + // Assert + String got = bout.toString(); + assertThat(got).contains("Found 3 entities."); + assertThat(got).contains("Larry Page"); + assertThat(got).contains("Salience: 0.426"); + assertThat(got).contains("Type: PERSON"); + assertThat(got).contains("Metadata: knowledge_graph_mid = /m/0gjpq"); + assertThat(got) + .contains("Metadata: wikipedia_url = http://en.wikipedia.org/wiki/index.html?curid=60903"); + assertThat(got).contains("search engine"); + assertThat(got).contains("Salience: 0.188"); + assertThat(got).contains("Type: CONSUMER_GOOD"); + assertThat(got).contains("something"); + } + + @Test public void printSentiment_withNull_printsNoneFound() throws Exception { + // Arrange + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + + // Act + Analyze.printSentiment(out, null); + + // Assert + String got = bout.toString(); + assertThat(got).contains("No sentiment found"); + } + + @Test public void printSyntax_withNull_printsNoneFound() throws Exception { + // Arrange + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + + // Act + Analyze.printSyntax(out, null); + + // Assert + String got = bout.toString(); + assertThat(got).contains("No syntax found"); + } +} diff --git a/pom.xml b/pom.xml index fa9f427db02..faaab723c76 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,7 @@ storage/xml-api/cmdline-sample storage/xml-api/serviceaccount-appengine-sample taskqueue/deferred + language/analysis unittests vision/face-detection vision/label