feat: add ability to upload entire directory, add samples for upload … (#2118)

ddelgrosso1 · gcf-owl-bot[bot] · web-flow · commit b0f32ced04ea · 2023-01-04T09:59:48.000-05:00
* feat: add ability to upload entire directory, add samples for upload directory / download folder * linter fix * update phrasing of comments * update comment in downloadFolderWithTransferManager sample * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/README.md b/README.md
@@ -133,6 +133,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage/tre
 | Download File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFile.js,samples/README.md) |
 | Download a File in Chunks Utilzing Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileInChunksWithTransferManager.js,samples/README.md) |
 | Download File Using Requester Pays | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileUsingRequesterPays.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileUsingRequesterPays.js,samples/README.md) |
+| Download Folder With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFolderWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFolderWithTransferManager.js,samples/README.md) |
 | Download Into Memory | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadIntoMemory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadIntoMemory.js,samples/README.md) |
 | Download Many Files With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadManyFilesWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadManyFilesWithTransferManager.js,samples/README.md) |
 | Storage Download Public File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadPublicFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadPublicFile.js,samples/README.md) |
@@ -205,6 +206,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage/tre
 | Stream File Download | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/streamFileDownload.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/streamFileDownload.js,samples/README.md) |
 | Stream File Upload | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/streamFileUpload.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/streamFileUpload.js,samples/README.md) |
 | Upload a directory to a bucket. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectory.js,samples/README.md) |
+| Upload Directory With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectoryWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectoryWithTransferManager.js,samples/README.md) |
 | Upload Encrypted File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadEncryptedFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadEncryptedFile.js,samples/README.md) |
 | Upload File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFile.js,samples/README.md) |
 | Upload File With Kms Key | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileWithKmsKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileWithKmsKey.js,samples/README.md) |
diff --git a/samples/README.md b/samples/README.md
@@ -52,6 +52,7 @@ objects to users via direct download.
   * [Download File](#download-file)
   * [Download a File in Chunks Utilzing Transfer Manager](#download-a-file-in-chunks-utilzing-transfer-manager)
   * [Download File Using Requester Pays](#download-file-using-requester-pays)
+  * [Download Folder With Transfer Manager](#download-folder-with-transfer-manager)
   * [Download Into Memory](#download-into-memory)
   * [Download Many Files With Transfer Manager](#download-many-files-with-transfer-manager)
   * [Storage Download Public File.](#storage-download-public-file.)
@@ -124,6 +125,7 @@ objects to users via direct download.
   * [Stream File Download](#stream-file-download)
   * [Stream File Upload](#stream-file-upload)
   * [Upload a directory to a bucket.](#upload-a-directory-to-a-bucket.)
+  * [Upload Directory With Transfer Manager](#upload-directory-with-transfer-manager)
   * [Upload Encrypted File](#upload-encrypted-file)
   * [Upload File](#upload-file)
   * [Upload File With Kms Key](#upload-file-with-kms-key)
@@ -752,6 +754,25 @@ __Usage:__
 
 
 
+### Download Folder With Transfer Manager
+
+Downloads a folder in parallel utilizing transfer manager.
+
+View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFolderWithTransferManager.js).
+
+[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFolderWithTransferManager.js,samples/README.md)
+
+__Usage:__
+
+
+`node downloadFolderWithTransferManager.js <BUCKET_NAME> <FOLDER_NAME>`
+
+
+-----
+
+
+
+
 ### Download Into Memory
 
 View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadIntoMemory.js).
@@ -2012,6 +2033,25 @@ __Usage:__
 
 
 
+### Upload Directory With Transfer Manager
+
+Uploads a directory in parallel utilizing transfer manager.
+
+View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectoryWithTransferManager.js).
+
+[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectoryWithTransferManager.js,samples/README.md)
+
+__Usage:__
+
+
+`node uploadFolderWithTransferManager.js <BUCKET_NAME> <DIRECTORY_NAME>`
+
+
+-----
+
+
+
+
 ### Upload Encrypted File
 
 View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadEncryptedFile.js).
diff --git a/samples/downloadFileInChunksWithTransferManager.js b/samples/downloadFileInChunksWithTransferManager.js
@@ -51,7 +51,7 @@ function main(
   // Creates a client
   const storage = new Storage();
 
-  // Creates a transfer manager instance
+  // Creates a transfer manager client
   const transferManager = new TransferManager(storage.bucket(bucketName));
 
   async function downloadFileInChunksWithTransferManager() {
diff --git a/samples/downloadFolderWithTransferManager.js b/samples/downloadFolderWithTransferManager.js
@@ -0,0 +1,60 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * @experimental
+ */
+
+// sample-metadata:
+//   title: Download Folder With Transfer Manager
+//   description: Downloads a folder in parallel utilizing transfer manager.
+//   usage: node downloadFolderWithTransferManager.js <BUCKET_NAME> <FOLDER_NAME>
+
+function main(bucketName = 'my-bucket', folderName = 'my-folder') {
+  // [START storage_download_folder_transfer_manager]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the GCS folder to download. The folder will be downloaded to the local path of the executing code.
+  // const folderName = 'your-folder-name';
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager client
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadFolderWithTransferManager() {
+    // Downloads the folder
+    await transferManager.downloadManyFiles(folderName);
+
+    console.log(
+      `gs://${bucketName}/${folderName} downloaded to ${folderName}.`
+    );
+  }
+
+  downloadFolderWithTransferManager().catch(console.error);
+  // [END storage_download_folder_transfer_manager]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+main(...process.argv.slice(2));
diff --git a/samples/downloadManyFilesWithTransferManager.js b/samples/downloadManyFilesWithTransferManager.js
@@ -44,7 +44,7 @@ function main(
   // Creates a client
   const storage = new Storage();
 
-  // Creates a transfer manager instance
+  // Creates a transfer manager client
   const transferManager = new TransferManager(storage.bucket(bucketName));
 
   async function downloadManyFilesWithTransferManager() {
diff --git a/samples/system-test/transfer-manager.test.js b/samples/system-test/transfer-manager.test.js
@@ -28,8 +28,9 @@ const bucketName = generateName();
 const bucket = storage.bucket(bucketName);
 const firstFileName = 'test.txt';
 const secondFileName = 'test2.txt';
-const firstFilePath = path.join(cwd, 'resources', firstFileName);
-const secondFilePath = path.join(cwd, 'resources', secondFileName);
+const resourcesPath = path.join(cwd, 'resources');
+const firstFilePath = path.join(resourcesPath, firstFileName);
+const secondFilePath = path.join(resourcesPath, secondFileName);
 const downloadFilePath = path.join(cwd, 'downloaded.txt');
 const chunkSize = 1024;
 
@@ -78,6 +79,28 @@ describe('transfer manager', () => {
       )
     );
   });
+
+  it('should upload a directory', async () => {
+    const output = execSync(
+      `node uploadDirectoryWithTransferManager.js ${bucketName} ${resourcesPath}`
+    );
+    assert.match(
+      output,
+      new RegExp(`${resourcesPath} uploaded to ${bucketName}.`)
+    );
+  });
+
+  it('should download a directory', async () => {
+    const output = execSync(
+      `node downloadFolderWithTransferManager.js ${bucketName} ${resourcesPath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${resourcesPath} downloaded to ${resourcesPath}.`
+      )
+    );
+  });
 });
 
 function generateName() {
diff --git a/samples/uploadDirectoryWithTransferManager.js b/samples/uploadDirectoryWithTransferManager.js
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * @experimental
+ */
+
+// sample-metadata:
+//   title: Upload Directory With Transfer Manager
+//   description: Uploads a directory in parallel utilizing transfer manager.
+//   usage: node uploadFolderWithTransferManager.js <BUCKET_NAME> <DIRECTORY_NAME>
+
+function main(bucketName = 'my-bucket', directoryName = 'my-directory') {
+  // [START storage_upload_directory_transfer_manager]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The local directory to upload
+  // const directoryName = 'your-directory';
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager client
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function uploadDirectoryWithTransferManager() {
+    // Uploads the directory
+    await transferManager.uploadManyFiles(directoryName);
+
+    console.log(`${directoryName} uploaded to ${bucketName}.`);
+  }
+
+  uploadDirectoryWithTransferManager().catch(console.error);
+  // [END storage_upload_directory_transfer_manager]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+main(...process.argv.slice(2));
diff --git a/samples/uploadManyFilesWithTransferManager.js b/samples/uploadManyFilesWithTransferManager.js
@@ -44,7 +44,7 @@ function main(
   // Creates a client
   const storage = new Storage();
 
-  // Creates a transfer manager instance
+  // Creates a transfer manager client
   const transferManager = new TransferManager(storage.bucket(bucketName));
 
   async function uploadManyFilesWithTransferManager() {
diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts
@@ -99,7 +99,8 @@ export class TransferManager {
    * Upload multiple files in parallel to the bucket. This is a convenience method
    * that utilizes {@link Bucket#upload} to perform the upload.
    *
-   * @param {array} [filePaths] An array of fully qualified paths to the files.
+   * @param {array | string} [filePathsOrDirectory] An array of fully qualified paths to the files or a directory name.
+   * If a directory name is provided, the directory will be recursively walked and all files will be added to the upload list.
    * to be uploaded to the bucket
    * @param {UploadManyFilesOptions} [options] Configuration options.
    * @returns {Promise<UploadResponse[]>}
@@ -118,11 +119,13 @@ export class TransferManager {
    * // Your bucket now contains:
    * // - "local/path/file1.txt" (with the contents of '/local/path/file1.txt')
    * // - "local/path/file2.txt" (with the contents of '/local/path/file2.txt')
+   * const response = await transferManager.uploadManyFiles('/local/directory');
+   * // Your bucket will now contain all files contained in '/local/directory' maintaining the subdirectory structure.
    * ```
    * @experimental
    */
   async uploadManyFiles(
-    filePaths: string[],
+    filePathsOrDirectory: string[] | string,
     options: UploadManyFilesOptions = {}
   ): Promise<UploadResponse[]> {
     if (options.skipIfExists && options.passthroughOptions?.preconditionOpts) {
@@ -142,8 +145,18 @@ export class TransferManager {
       options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT
     );
     const promises = [];
+    let allPaths: string[] = [];
+    if (!Array.isArray(filePathsOrDirectory)) {
+      for await (const curPath of this.getPathsFromDirectory(
+        filePathsOrDirectory
+      )) {
+        allPaths.push(curPath);
+      }
+    } else {
+      allPaths = filePathsOrDirectory;
+    }
 
-    for (const filePath of filePaths) {
+    for (const filePath of allPaths) {
       const stat = await fsp.lstat(filePath);
       if (stat.isDirectory()) {
         continue;
@@ -355,4 +368,18 @@ export class TransferManager {
         });
     });
   }
+
+  private async *getPathsFromDirectory(
+    directory: string
+  ): AsyncGenerator<string> {
+    const filesAndSubdirectories = await fsp.readdir(directory, {
+      withFileTypes: true,
+    });
+    for (const curFileOrDirectory of filesAndSubdirectories) {
+      const fullPath = path.join(directory, curFileOrDirectory.name);
+      curFileOrDirectory.isDirectory()
+        ? yield* this.getPathsFromDirectory(fullPath)
+        : yield fullPath;
+    }
+  }
 }
