chore: samples overhaul (#3109)

* Centralize environment variables resolution in new Env utility class
* Leverage StorageITRunner and associated fixtures
* Condense PubSubNotification test classes into one class that can all share a topic
* Replace usages of the word file with blob (GCS doesn't have files, and our library always presents Blob as its name for a gcs object)
* Use TemporaryBucket for tests which depend on specific bucket state.
  1. Avoids contention between tests trying to change bucket state
  2. Avoids the "too many bucket modifications" error from gcs due to bucket mutations
  3. Makes it possible, for running tests in parallel in the future
* Update samples to apply try-with-resource to TransferManager and Storage instances
* Avoid using syntax classes in tests (use storage.x(blob) instead of blob.x())
* Add ITCleanupOldBucketsTest

Author: BenWhitehead

samples/install-without-bom/pom.xml

@@ -69,13 +69,32 @@
       <version>1.139.4</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-kms</artifactId>
+      <version>2.66.0</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-storage</artifactId>
       <version>2.52.3</version>
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <!-- tests jars don't have transitive dependencies automatically resolved -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <version>2.0.17</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.3.15</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud.opentelemetry</groupId>
       <artifactId>exporter-trace</artifactId>

samples/snapshot/pom.xml

@@ -61,13 +61,32 @@
       <version>1.139.4</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-kms</artifactId>
+      <version>2.66.0</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-storage</artifactId>
       <version>2.52.4-SNAPSHOT</version><!-- {x-version-update:google-cloud-storage:current} -->
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <!-- tests jars don't have transitive dependencies automatically resolved -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <version>2.0.17</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.3.15</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud.opentelemetry</groupId>
       <artifactId>exporter-trace</artifactId>

samples/snippets/pom.xml

@@ -79,6 +79,12 @@
       <version>1.139.4</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-kms</artifactId>
+      <version>2.66.0</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud.opentelemetry</groupId>
       <artifactId>exporter-trace</artifactId>
@@ -97,6 +103,19 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <!-- tests jars don't have transitive dependencies automatically resolved -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <version>2.0.17</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.3.15</version>
+      <scope>test</scope>
+    </dependency>
     <!-- [START storage_install_with_bom] -->
   </dependencies>
   <!-- [END storage_install_with_bom] -->

samples/snippets/src/main/java/com/example/storage/QuickstartStorageControlSample.java

@@ -34,7 +34,7 @@ public static void main(String... args) throws Exception {
               .setName(StorageLayoutName.format("_", bucketName))
               .build();
       StorageLayout response = storageControlClient.getStorageLayout(request);
-      System.out.printf("Performed getStorageLayout request for %s", response.getName());
+      System.out.printf("Performed getStorageLayout request for %s%n", response.getName());
     }
   }
 }

samples/snippets/src/main/java/com/example/storage/object/AddFileOwner.java renamed to samples/snippets/src/main/java/com/example/storage/object/AddBlobOwner.java

@@ -26,9 +26,9 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 
-public class AddFileOwner {
+public class AddBlobOwner {
 
-  public static void addFileOwner(
+  public static void addBlobOwner(
       String projectId, String bucketName, String userEmail, String blobName) {
     // The ID of your GCP project
     // String projectId = "your-project-id";
@@ -50,7 +50,7 @@ public static void addFileOwner(
     System.out.println(
         "Added user "
             + userEmail
-            + " as an owner on file "
+            + " as an owner on blob "
             + blobName
             + " in bucket "
             + bucketName);

samples/snippets/src/main/java/com/example/storage/object/BatchSetObjectMetadata.java

@@ -19,15 +19,21 @@
 // [START storage_batch_request]
 import com.google.api.gax.paging.Page;
 import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobInfo;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageBatch;
+import com.google.cloud.storage.StorageBatchResult;
+import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class BatchSetObjectMetadata {
   public static void batchSetObjectMetadata(
-      String projectId, String bucketName, String directoryPrefix) {
+      String projectId, String bucketName, String pathPrefix) {
     // The ID of your GCP project
     // String projectId = "your-project-id";
 
@@ -36,32 +42,56 @@ public static void batchSetObjectMetadata(
 
     // The directory prefix. All objects in the bucket with this prefix will have their metadata
     // updated
-    // String directoryPrefix = "yourDirectory/";
+    // String pathPrefix = "yourPath/";
 
     Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
     Map<String, String> newMetadata = new HashMap<>();
     newMetadata.put("keyToAddOrUpdate", "value");
     Page<Blob> blobs =
         storage.list(
             bucketName,
-            Storage.BlobListOption.prefix(directoryPrefix),
-            Storage.BlobListOption.currentDirectory());
+            Storage.BlobListOption.prefix(pathPrefix),
+            Storage.BlobListOption.delimiter("/"));
     StorageBatch batchRequest = storage.batch();
 
     // Add all blobs with the given prefix to the batch request
-    for (Blob blob : blobs.iterateAll()) {
-      batchRequest.update(blob.toBuilder().setMetadata(newMetadata).build());
-    }
+    List<StorageBatchResult<Blob>> batchResults =
+        blobs
+            .streamAll()
+            .map(blob -> batchRequest.update(blob.toBuilder().setMetadata(newMetadata).build()))
+            .collect(Collectors.toList());
 
     // Execute the batch request
     batchRequest.submit();
+    List<StorageException> failures =
+        batchResults.stream()
+            .map(
+                r -> {
+                  try {
+                    BlobInfo blob = r.get();
+                    return null;
+                  } catch (StorageException e) {
+                    return e;
+                  }
+                })
+            .filter(Objects::nonNull)
+            .collect(Collectors.toList());
 
     System.out.println(
-        "All blobs in bucket "
+        (batchResults.size() - failures.size())
+            + " blobs in bucket "
             + bucketName
             + " with prefix '"
-            + directoryPrefix
-            + "' had their metadata updated.");
+            + pathPrefix
+            + "' had their metadata updated successfully.");
+
+    if (!failures.isEmpty()) {
+      System.out.println("While processing, there were " + failures.size() + " failures");
+
+      for (StorageException failure : failures) {
+        failure.printStackTrace(System.out);
+      }
+    }
   }
 }
 // [END storage_batch_request]

samples/snippets/src/main/java/com/example/storage/object/DownloadByteRange.java

@@ -66,7 +66,7 @@ public static void downloadByteRange(
       ByteStreams.copy(from, to);
 
       System.out.printf(
-          "%s downloaded to %s from byte %d to byte %d",
+          "%s downloaded to %s from byte %d to byte %d%n",
           blobId.toGsUtilUri(), destFileName, startByte, endBytes);
     }
   }

samples/snippets/src/main/java/com/example/storage/object/DownloadObject.java

@@ -18,15 +18,15 @@
 
 // [START storage_download_file]
 
-import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import java.nio.file.Paths;
 
 public class DownloadObject {
   public static void downloadObject(
-      String projectId, String bucketName, String objectName, String destFilePath) {
+      String projectId, String bucketName, String objectName, String destFilePath)
+      throws Exception {
     // The ID of your GCP project
     // String projectId = "your-project-id";
 
@@ -39,18 +39,19 @@ public static void downloadObject(
     // The path to which the file should be downloaded
     // String destFilePath = "/local/path/to/file.txt";
 
-    Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
+    StorageOptions storageOptions = StorageOptions.newBuilder().setProjectId(projectId).build();
+    try (Storage storage = storageOptions.getService()) {
 
-    Blob blob = storage.get(BlobId.of(bucketName, objectName));
-    blob.downloadTo(Paths.get(destFilePath));
+      storage.downloadTo(BlobId.of(bucketName, objectName), Paths.get(destFilePath));
 
-    System.out.println(
-        "Downloaded object "
-            + objectName
-            + " from bucket name "
-            + bucketName
-            + " to "
-            + destFilePath);
+      System.out.println(
+          "Downloaded object "
+              + objectName
+              + " from bucket name "
+              + bucketName
+              + " to "
+              + destFilePath);
+    }
   }
 }
 // [END storage_download_file]

samples/snippets/src/main/java/com/example/storage/object/DownloadPublicObject.java

@@ -18,7 +18,6 @@
 
 // [START storage_download_public_file]
 
-import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
@@ -39,8 +38,7 @@ public static void downloadPublicObject(
     // Instantiate an anonymous Google Cloud Storage client, which can only access public files
     Storage storage = StorageOptions.getUnauthenticatedInstance().getService();
 
-    Blob blob = storage.get(BlobId.of(bucketName, publicObjectName));
-    blob.downloadTo(destFilePath);
+    storage.downloadTo(BlobId.of(bucketName, publicObjectName), destFilePath);
 
     System.out.println(
         "Downloaded public object "

samples/snippets/src/main/java/com/example/storage/object/PrintFileAcl.java renamed to samples/snippets/src/main/java/com/example/storage/object/PrintBlobAcl.java

@@ -25,9 +25,9 @@
 import com.google.cloud.storage.StorageOptions;
 import java.util.List;
 
-public class PrintFileAcl {
+public class PrintBlobAcl {
 
-  public static void printFileAcl(String bucketName, String blobName) {
+  public static void printBlobAcl(String bucketName, String blobName) {
 
     // The ID to give your GCS bucket
     // String bucketName = "your-unique-bucket-name";

samples/snippets/src/main/java/com/example/storage/object/PrintFileAclForUser.java renamed to samples/snippets/src/main/java/com/example/storage/object/PrintBlobAclForUser.java

@@ -25,9 +25,10 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 
-public class PrintFileAclForUser {
+public class PrintBlobAclForUser {
 
-  public static void printFileAclForUser(String bucketName, String blobName, String userEmail) {
+  public static void printBlobAclForUser(String bucketName, String blobName, String userEmail)
+      throws Exception {
 
     // The ID to give your GCS bucket
     // String bucketName = "your-unique-bucket-name";
@@ -38,14 +39,15 @@ public static void printFileAclForUser(String bucketName, String blobName, Strin
     // The email of the user whose acl is being retrieved.
     // String userEmail = "[email protected]"
 
-    Storage storage = StorageOptions.newBuilder().build().getService();
-    Blob blob = storage.get(BlobId.of(bucketName, blobName));
-    Acl blobAcl = blob.getAcl(new User(userEmail));
-    if (blobAcl != null) {
-      String userRole = blobAcl.getRole().name();
-      System.out.println("User " + userEmail + " has role " + userRole);
-    } else {
-      System.out.println("User " + userEmail + " not found");
+    try (Storage storage = StorageOptions.newBuilder().build().getService()) {
+      Blob blob = storage.get(BlobId.of(bucketName, blobName));
+      Acl blobAcl = blob.getAcl(new User(userEmail));
+      if (blobAcl != null) {
+        String userRole = blobAcl.getRole().name();
+        System.out.println("User " + userEmail + " has role " + userRole);
+      } else {
+        System.out.println("User " + userEmail + " not found");
+      }
     }
   }
 }

samples/snippets/src/main/java/com/example/storage/object/RemoveFileOwner.java renamed to samples/snippets/src/main/java/com/example/storage/object/RemoveBlobOwner.java

@@ -24,9 +24,9 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 
-public class RemoveFileOwner {
+public class RemoveBlobOwner {
 
-  public static void removeFileOwner(
+  public static void removeBlobOwner(
       String projectId, String bucketName, String userEmail, String blobName) {
     // The ID of your GCP project
     // String projectId = "your-project-id";

samples/snippets/src/main/java/com/example/storage/object/SetObjectMetadata.java

@@ -20,6 +20,7 @@
 
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import java.util.HashMap;
@@ -53,7 +54,8 @@ public static void setObjectMetadata(String projectId, String bucketName, String
 
     // Does an upsert operation, if the key already exists it's replaced by the new value, otherwise
     // it's added.
-    blob.toBuilder().setMetadata(newMetadata).build().update(precondition);
+    BlobInfo pendingUpdate = blob.toBuilder().setMetadata(newMetadata).build();
+    storage.update(pendingUpdate, precondition);
 
     System.out.println(
         "Updated custom metadata for object " + objectName + " in bucket " + bucketName);