Merge branch 'main' into async-configstores

* main:
  Bump typescript from 4.9.3 to 5.0.4 (tus#425)
  Bump sinon from 15.0.0 to 15.0.4 (tus#426)
  @tus/gcs-store: allow user to pass bucket instance to GCSStore (tus#388)
  Fix e2e test typecheking (tus#429)
  Update incorrect comment (tus#430)
  @tus/s3-store: add missing error code in termination extension (tus#413)
  @tus/[email protected]
  @tus/s3-store: fix uncaught exception when cancelling an upload (tus#412)
  @tus/[email protected]
  @tus/s3-store: fix uncaught exception when removing files (tus#410)
  @tus/s3-store: add termination extension (tus#401)
  @tus/[email protected]
  Bump @google-cloud/storage from 6.8.0 to 6.9.3 (tus#404)
  Bump http-cache-semantics from 4.1.0 to 4.1.1 (tus#393)
  Bump eslint-config-prettier from 8.5.0 to 8.6.0 (tus#392)
  Bump cookiejar from 2.1.3 to 2.1.4 (tus#383)
  Bump aws-sdk from 2.1269.0 to 2.1325.0 (tus#403)
  @tus/server: refactor and improve request validator (tus#402)
  Fix demo (tus#386)

Author: Murderlon

README.md

@@ -110,28 +110,28 @@ The tus protocol supports optional [extensions][]. Below is a table of the suppo
 | [Creation With Upload][] | ✅                                | ✅                            | ✅                              |
 | [Expiration][]           | ✅                                | ❌                            | ❌                              |
 | [Checksum][]             | ❌                                | ❌                            | ❌                              |
-| [Termination][]          | ✅                                | ❌                            | ❌                              |
+| [Termination][]          | ✅                                | ✅                            | ❌                              |
 | [Concatenation][]        | ❌                                | ❌                            | ❌                              |
 
 ## Demos
 
 Start the demo server using Local File Storage
 
 ```bash
-yarn workspace demo start
+yarn build && yarn demo
 ```
 
 Start up the demo server using AWS S3. The environment variables `AWS_BUCKET`,
 `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` need to be present.
 
 ```bash
-yarn workspace demo start:s3
+yarn build && yarn demo:s3
 ```
 
 Start up the demo server using Google Cloud Storage. A `keyfile.json` needs to be present in the root of the repository.
 
 ```bash
-yarn workspace demo start:gcs
+yarn build && yarn demo:gcs
 ```
 
 Then navigate to the demo ([localhost:1080](http://localhost:1080)) which uses [`tus-js-client`](https://github.com/tus/tus-js-client).

demo/server.js

@@ -59,7 +59,7 @@ const writeFile = (req, res) => {
   if (!filename.startsWith('/dist/')) {
     filename = '/demos/browser' + filename
   }
-  filename = path.join(process.cwd(), '/node_modules/tus-js-client', filename)
+  filename = path.join(process.cwd(), '../node_modules/tus-js-client', filename)
   fs.readFile(filename, 'binary', (err, file) => {
     if (err) {
       res.writeHead(500, {'Content-Type': 'text/plain'})

packages/eslint-config-custom/package.json

@@ -12,7 +12,7 @@
     "@typescript-eslint/eslint-plugin": "^5.45.1",
     "@typescript-eslint/parser": "^5.47.1",
     "eslint": "^8.29.0",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^8.6.0",
     "eslint-config-turbo": "latest",
     "eslint-plugin-prettier": "^4.2.1",
     "prettier": "^2.8.1"

packages/gcs-store/README.md

@@ -29,12 +29,14 @@ npm install @tus/gcs-store
 const {Server} = require('@tus/server')
 const {GCSStore} = require('@tus/gcs-store')
 
+const {Storage} = require('@google-cloud/storage');
+
+const storage = new Storage({keyFilename: 'key.json'})
+
 const server = new Server({
   path: '/files',
   datastore: new GCSStore({
-    projectId: 'id',
-    keyFilename: path.resolve('./some-path', 'keyfile.json'),
-    bucket: 'tus-node-server-ci',
+    bucket: storage.bucket('tus-node-server-ci'),
   }),
 })
 // ...
@@ -46,19 +48,11 @@ This package exports `GCSStore`. There is no default export.
 
 ### `new GCSStore(options)`
 
-Creates a new Google Cloud Storage store with options.
-
-#### `options.projectId`
-
-The GCS project ID (`string`).
-
-#### `options.keyFilename`
-
-Path to the keyfile with credentials (`string`).
+Creates a new Google Cloud Storage store by passing a GCS bucket instance.
 
 #### `options.bucket`
 
-The bucket name.
+The bucket instance
 
 ## Extensions
 

packages/gcs-store/index.ts

@@ -1,23 +1,17 @@
-import {Storage, Bucket} from '@google-cloud/storage'
+import {Bucket} from '@google-cloud/storage'
 import stream from 'node:stream'
 import http from 'node:http'
 import debug from 'debug'
 
 import {ERRORS, TUS_RESUMABLE} from '@tus/server'
 import {Upload, DataStore} from '@tus/server'
 
-type Options = {
-  bucket: string
-  projectId: string
-  keyFilename: string
-}
-
 const log = debug('tus-node-server:stores:gcsstore')
 
+type Options = {bucket: Bucket}
+
 export class GCSStore extends DataStore {
   bucket: Bucket
-  bucket_name: string
-  gcs: Storage
 
   constructor(options: Options) {
     super()
@@ -26,13 +20,9 @@ export class GCSStore extends DataStore {
       throw new Error('GCSDataStore must have a bucket')
     }
 
+    this.bucket = options.bucket
+
     this.extensions = ['creation', 'creation-with-upload', 'creation-defer-length']
-    this.bucket_name = options.bucket
-    this.gcs = new Storage({
-      projectId: options.projectId,
-      keyFilename: options.keyFilename,
-    })
-    this.bucket = this.gcs.bucket(this.bucket_name)
   }
 
   create(file: Upload): Promise<Upload> {

packages/gcs-store/package.json

@@ -21,10 +21,10 @@
     "test": "mocha test.ts --timeout 30000 --exit --extension ts --require ts-node/register"
   },
   "dependencies": {
-    "@google-cloud/storage": "^6.2.2",
     "debug": "^4.3.3"
   },
   "devDependencies": {
+    "@google-cloud/storage": "^6.9.3",
     "@tus/server": "workspace:^",
     "@types/debug": "^4.1.7",
     "@types/mocha": "^10.0.1",
@@ -35,7 +35,8 @@
     "should": "^13.2.3",
     "typescript": "latest"
   },
-  "peerdependencies": {
+  "peerDependencies": {
+    "@google-cloud/storage": "*",
     "@tus/server": "workspace:^"
   },
   "engines": {

packages/gcs-store/test.ts

@@ -4,6 +4,8 @@ import {GCSStore} from './'
 
 import * as shared from '../../test/stores.test'
 
+import {Storage} from '@google-cloud/storage'
+
 const fixturesPath = path.resolve('../', '../', 'test', 'fixtures')
 const storePath = path.resolve('../', '../', 'test', 'output')
 
@@ -16,10 +18,13 @@ describe('GCSStore', () => {
   })
 
   beforeEach(function () {
-    this.datastore = new GCSStore({
+    const storage = new Storage({
       projectId: 'tus-node-server',
       keyFilename: path.resolve('../', '../', 'keyfile.json'),
-      bucket: 'tus-node-server-ci',
+    })
+
+    this.datastore = new GCSStore({
+      bucket: storage.bucket('tus-node-server-ci'),
     })
   })
 

packages/s3-store/README.md

@@ -2,7 +2,7 @@
 
 > 👉 **Note**: since 1.0.0 packages are split and published under the `@tus` scope.
 > The old package, `tus-node-server`, is considered unstable and will only receive security fixes.
-> Make sure to use the new packages, currently in beta at `1.0.0-beta.1`.
+> Make sure to use the new package, currently in beta at `1.0.0-beta.5`.
 
 ## Contents
 
@@ -67,7 +67,7 @@ but may increase it to not exceed the S3 10K parts limit.
 
 Options to pass to the AWS S3 SDK.
 Checkout the [`S3ClientConfig`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html)
-docs for the supported options. You need to at least set the `region`, `bucket` name, and your preferred method of authentication. 
+docs for the supported options. You need to at least set the `region`, `bucket` name, and your preferred method of authentication.
 
 ## Extensions
 
@@ -79,9 +79,13 @@ The tus protocol supports optional [extensions][]. Below is a table of the suppo
 | [Creation With Upload][] | ✅              |
 | [Expiration][]           | ❌              |
 | [Checksum][]             | ❌              |
-| [Termination][]          | ❌              |
+| [Termination][]          | ✅              |
 | [Concatenation][]        | ❌              |
 
+### Termination
+
+After a multipart upload is aborted, no additional parts can be uploaded using that upload ID. The storage consumed by any previously uploaded parts will be freed. However, if any part uploads are currently in progress, those part uploads might or might not succeed. As a result, it might be necessary to set an [S3 Lifecycle configuration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html) to abort incomplete multipart uploads.
+
 ## Examples
 
 ### Example: using `credentials` to fetch credentials inside a AWS container

packages/s3-store/index.ts

@@ -72,7 +72,12 @@ export class S3Store extends DataStore {
     super()
     const {partSize, s3ClientConfig} = options
     const {bucket, ...restS3ClientConfig} = s3ClientConfig
-    this.extensions = ['creation', 'creation-with-upload', 'creation-defer-length']
+    this.extensions = [
+      'creation',
+      'creation-with-upload',
+      'creation-defer-length',
+      'termination',
+    ]
     this.bucket = bucket
     this.preferredPartSize = partSize || 8 * 1024 * 1024
     this.client = new aws.S3(restS3ClientConfig)
@@ -257,19 +262,23 @@ export class S3Store extends DataStore {
               ])
             }
 
+            const readable = fs.createReadStream(path)
+            readable.on('error', reject)
             if (partSize > this.minPartSize || isFinalChunk) {
-              await this.uploadPart(metadata, fs.createReadStream(path), partNumber)
+              await this.uploadPart(metadata, readable, partNumber)
               offset += partSize
             } else {
-              await this.uploadIncompletePart(incompletePartId, fs.createReadStream(path))
+              await this.uploadIncompletePart(incompletePartId, readable)
             }
 
             bytesUploaded += partSize
             resolve()
           } catch (error) {
             reject(error)
           } finally {
-            fsProm.rm(path).catch(/* ignore */)
+            fsProm.rm(path).catch(() => {
+              /* ignore */
+            })
           }
         })
 
@@ -517,4 +526,36 @@ export class S3Store extends DataStore {
 
     this.saveMetadata(file, upload_id)
   }
+
+  public async remove(id: string): Promise<void> {
+    try {
+      const {upload_id} = await this.getMetadata(id)
+      if (upload_id) {
+        await this.client
+          .abortMultipartUpload({
+            Bucket: this.bucket,
+            Key: id,
+            UploadId: upload_id,
+          })
+          .promise()
+      }
+    } catch (error) {
+      if (error?.code && ['NotFound', 'NoSuchKey', 'NoSuchUpload'].includes(error.code)) {
+        log('remove: No file found.', error)
+        throw ERRORS.FILE_NOT_FOUND
+      }
+      throw error
+    }
+
+    await this.client
+      .deleteObjects({
+        Bucket: this.bucket,
+        Delete: {
+          Objects: [{Key: id}, {Key: `${id}.info`}],
+        },
+      })
+      .promise()
+
+    this.clearCache(id)
+  }
 }

packages/s3-store/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@tus/s3-store",
-  "version": "1.0.0-beta.3",
+  "version": "1.0.0-beta.5",
   "description": "AWS S3 store for @tus/server",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,7 +21,7 @@
     "test": "mocha test.ts --timeout 40000 --exit --extension ts --require ts-node/register"
   },
   "dependencies": {
-    "aws-sdk": "^2.1064.0",
+    "aws-sdk": "^2.1325.0",
     "debug": "^4.3.3"
   },
   "devDependencies": {