From 6596705e3a025b0e387fee7395e67c87adeb772c Mon Sep 17 00:00:00 2001 From: nnegrey Date: Thu, 5 Mar 2020 16:03:35 -0700 Subject: [PATCH 1/3] automl: move samples into beta set --- automl/beta/batch_predict.py | 51 ++++++++++++++++++++++++++++++ automl/beta/batch_predict_test.py | 47 +++++++++++++++++++++++++++ automl/beta/delete_dataset.py | 29 +++++++++++++++++ automl/beta/delete_dataset_test.py | 46 +++++++++++++++++++++++++++ 4 files changed, 173 insertions(+) create mode 100644 automl/beta/batch_predict.py create mode 100644 automl/beta/batch_predict_test.py create mode 100644 automl/beta/delete_dataset.py create mode 100644 automl/beta/delete_dataset_test.py diff --git a/automl/beta/batch_predict.py b/automl/beta/batch_predict.py new file mode 100644 index 00000000000..af3c443871f --- /dev/null +++ b/automl/beta/batch_predict.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START automl_batch_predict_beta] +from google.cloud import automl_v1beta1 as automl + + +def batch_predict( + project_id="YOUR_PROJECT_ID", + model_id="YOUR_MODEL_ID", + input_uri="gs://YOUR_BUCKET_ID/path/to/your/input/csv_or_jsonl", + output_uri="gs://YOUR_BUCKET_ID/path/to/save/results/", +): + """Batch predict""" + prediction_client = automl.PredictionServiceClient() + + # Get the full path of the model. + model_full_id = prediction_client.model_path( + project_id, "us-central1", model_id + ) + + gcs_source = automl.types.GcsSource(input_uris=[input_uri]) + + input_config = automl.types.BatchPredictInputConfig(gcs_source=gcs_source) + gcs_destination = automl.types.GcsDestination(output_uri_prefix=output_uri) + output_config = automl.types.BatchPredictOutputConfig( + gcs_destination=gcs_destination + ) + + response = prediction_client.batch_predict( + model_full_id, input_config, output_config + ) + + print("Waiting for operation to complete...") + print( + "Batch Prediction results saved to Cloud Storage bucket. {}".format( + response.result() + ) + ) +# [END automl_batch_predict_beta] diff --git a/automl/beta/batch_predict_test.py b/automl/beta/batch_predict_test.py new file mode 100644 index 00000000000..0f9417f6410 --- /dev/null +++ b/automl/beta/batch_predict_test.py @@ -0,0 +1,47 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific ladnguage governing permissions and +# limitations under the License. + +import datetime +import os + +import batch_predict + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) +MODEL_ID = "TEN0000000000000000000" +PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime( + "%Y%m%d%H%M%S" +) + + +def test_batch_predict(capsys): + # As batch prediction can take a long time. Try to batch predict on a model + # and confirm that the model was not found, but other elements of the + # request were valid. + try: + input_uri = "gs://{}/entity-extraction/input.jsonl".format(BUCKET_ID) + output_uri = "gs://{}/{}/".format(BUCKET_ID, PREFIX) + batch_predict.batch_predict( + PROJECT_ID, MODEL_ID, input_uri, output_uri + ) + out, _ = capsys.readouterr() + assert ( + "The model is either not found or not supported for prediction yet" + in out + ) + except Exception as e: + assert ( + "The model is either not found or not supported for prediction yet" + in e.message + ) diff --git a/automl/beta/delete_dataset.py b/automl/beta/delete_dataset.py new file mode 100644 index 00000000000..f851e66646d --- /dev/null +++ b/automl/beta/delete_dataset.py @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START automl_delete_dataset_beta] +from google.cloud import automl_v1beta1 as automl + + +def delete_dataset(project_id="YOUR_PROJECT_ID", dataset_id="YOUR_DATASET_ID"): + """Delete a dataset.""" + client = automl.AutoMlClient() + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + response = client.delete_dataset(dataset_full_id) + + print("Dataset deleted. {}".format(response.result())) +# [END automl_delete_dataset_beta] diff --git a/automl/beta/delete_dataset_test.py b/automl/beta/delete_dataset_test.py new file mode 100644 index 00000000000..897861de5e7 --- /dev/null +++ b/automl/beta/delete_dataset_test.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from google.cloud import automl_v1beta1 as automl +import pytest + +import delete_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) + + +@pytest.fixture(scope="function") +def dataset_id(): + client = automl.AutoMlClient() + project_location = client.location_path(PROJECT_ID, "us-central1") + display_name = "test_" + uuid.uuid4() + metadata = automl.types.TextExtractionDatasetMetadata() + dataset = automl.types.Dataset( + display_name=display_name, text_extraction_dataset_metadata=metadata + ) + response = client.create_dataset(project_location, dataset) + dataset_id = response.result().name.split("/")[-1] + + yield dataset_id + + +def test_delete_dataset(capsys, dataset_id): + # delete dataset + delete_dataset.delete_dataset(PROJECT_ID, dataset_id) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out From fe7bf71f1e996a696bdd9b3fac4509792895436c Mon Sep 17 00:00:00 2001 From: nnegrey Date: Thu, 5 Mar 2020 16:13:17 -0700 Subject: [PATCH 2/3] fix uuid --- automl/beta/delete_dataset_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automl/beta/delete_dataset_test.py b/automl/beta/delete_dataset_test.py index 897861de5e7..9781ad26066 100644 --- a/automl/beta/delete_dataset_test.py +++ b/automl/beta/delete_dataset_test.py @@ -28,13 +28,13 @@ def dataset_id(): client = automl.AutoMlClient() project_location = client.location_path(PROJECT_ID, "us-central1") - display_name = "test_" + uuid.uuid4() + display_name = "test_{}".format(uuid.uuid4()).replace("-", "")[:32] metadata = automl.types.TextExtractionDatasetMetadata() dataset = automl.types.Dataset( display_name=display_name, text_extraction_dataset_metadata=metadata ) response = client.create_dataset(project_location, dataset) - dataset_id = response.result().name.split("/")[-1] + dataset_id = response.name.split("/")[-1] yield dataset_id From 8f594258e9cc447dad39cf3b7ee5398c6d1cf35a Mon Sep 17 00:00:00 2001 From: Noah Negrey Date: Tue, 10 Mar 2020 12:08:33 -0600 Subject: [PATCH 3/3] Update batch_predict_test.py --- automl/beta/batch_predict_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automl/beta/batch_predict_test.py b/automl/beta/batch_predict_test.py index 0f9417f6410..2869873a919 100644 --- a/automl/beta/batch_predict_test.py +++ b/automl/beta/batch_predict_test.py @@ -37,11 +37,11 @@ def test_batch_predict(capsys): ) out, _ = capsys.readouterr() assert ( - "The model is either not found or not supported for prediction yet" + "does not exist" in out ) except Exception as e: assert ( - "The model is either not found or not supported for prediction yet" + "does not exist" in e.message )