diff --git a/CHANGELOG.md b/CHANGELOG.md index 448cfe2..694fd53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] -### Fixed - -- Use pathToFileURL so that import() works under ESM across platforms [#38](https://github.com/fastly/compute-js-static-publish/issues/38) - -## [7.0.2] - 2025-09-16 - -### Fixed - -- Fix script listed in readme instructions -- Handle symlinks properly in file enumeration [#41](https://github.com/fastly/compute-js-static-publish/issues/41) - ### Breaking - Rename symbols @@ -26,12 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- S3-compatible storage +- S3-compatible storage (BETA) - Add support for S3-compatible storage, such as Fastly Object Storage - Store items using same keys as KV Store - Use S3 object metadata for storing asset metadata - Storage factored out to StorageProvider, and S3 is implemented using this architecture - - Configure using + - Add AWS-related configurations to fastly.toml in local_server and setup sections - `static-publish.rc.js` - Add `s3` mode configuration @@ -41,6 +30,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `publish-content.config.js` - `kvStoreAssetInclusionTest` renamed to `assetInclusionTest`. Previous name deprecated. +## [7.0.3] - 2025-09-29 + +### Fixed + +- Use pathToFileURL so that import() works under ESM across platforms [#38](https://github.com/fastly/compute-js-static-publish/issues/38) + +## [7.0.2] - 2025-09-16 + +### Fixed + +- Fix script listed in readme instructions +- Handle symlinks properly in file enumeration [#41](https://github.com/fastly/compute-js-static-publish/issues/41) + ## [7.0.1] - 2025-04-24 ### Fixed diff --git a/README.md b/README.md index 767f6f2..b9ebd49 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,6 @@ Create a directory for your project, place your static files in `./public`, then ```sh npx @fastly/compute-js-static-publish@latest \ --root-dir=./public \ - --storage-mode=kv-store \ --kv-store-name=site-content ``` @@ -236,7 +235,7 @@ const rc = { export default rc; ``` -### Using S3-compatible storage +### Using S3-compatible storage (BETA) #### Fields: @@ -525,7 +524,7 @@ If you do need to rebuild and redeploy the Compute app, simply run: npm run fastly:deploy ``` -### Using S3-compatible storage +### Using S3-compatible storage (BETA) #### Local development @@ -731,10 +730,9 @@ Run outside an existing Compute app directory: # Using KV store storage npx @fastly/compute-js-static-publish@latest \ --root-dir=./public \ - --storage-mode=kv-store \ --kv-store-name= -# Using S3 storage +# Using S3 storage (BETA) npx @fastly/compute-js-static-publish@latest \ --root-dir=./public \ --storage-mode=s3 \ @@ -748,7 +746,7 @@ npx @fastly/compute-js-static-publish@latest \ ```sh npx @fastly/compute-js-static-publish@latest \ --root-dir=./public \ - { --storage-mode=kv-store --kv-store-name= | \ + { [--storage-mode=kv-store] --kv-store-name= | \ --storage-mode=s3 --s3-region= --s3-bucket= [--s3-endpoint=] } \ [--output=./compute-js] \ [--static-publisher-working-dir=/static-publisher] \ @@ -768,7 +766,7 @@ npx @fastly/compute-js-static-publish@latest \ #### Options: **Used to generate the Compute app:** -- `--storage-mode`: Required. Specifies the storage mode. Must be either `kv-store` or `s3`. +- `--storage-mode`: Specifies the storage mode. Must be either `kv-store` or `s3` (default: `kv-store`). If `--storage-mode=kv-store`: - `--kv-store-name`: Required. Name of KV Store to use. diff --git a/README.short.md b/README.short.md index 882e933..5730fb6 100644 --- a/README.short.md +++ b/README.short.md @@ -18,7 +18,7 @@ This CLI tool helps you: Create a directory for your project, place your static files in `./public`, then type: ```sh -npx @fastly/compute-js-static-publish@latest --root-dir=./public --storage-mode=kv-store --kv-store-name=site-content +npx @fastly/compute-js-static-publish@latest --root-dir=./public --kv-store-name=site-content ``` **New in v8:** S3-compatible storage (such as Fastly Object Storage) is also supported (Beta). To use this mode, type: diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts index 64632b2..8e3ce1b 100644 --- a/src/cli/commands/manage/clean.ts +++ b/src/cli/commands/manage/clean.ts @@ -39,7 +39,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts index fe2ad5b..d2ef956 100644 --- a/src/cli/commands/manage/collections/delete.ts +++ b/src/cli/commands/manage/collections/delete.ts @@ -37,7 +37,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/collections/index.ts b/src/cli/commands/manage/collections/index.ts index cd6499f..92f4b03 100644 --- a/src/cli/commands/manage/collections/index.ts +++ b/src/cli/commands/manage/collections/index.ts @@ -35,7 +35,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/collections/list.ts b/src/cli/commands/manage/collections/list.ts index 92b259a..b5675b3 100644 --- a/src/cli/commands/manage/collections/list.ts +++ b/src/cli/commands/manage/collections/list.ts @@ -32,7 +32,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts index 4a441ff..dae2ec8 100644 --- a/src/cli/commands/manage/collections/promote.ts +++ b/src/cli/commands/manage/collections/promote.ts @@ -54,7 +54,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts index eb8b87b..1594a86 100644 --- a/src/cli/commands/manage/collections/update-expiration.ts +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -50,7 +50,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/index.ts b/src/cli/commands/manage/index.ts index dc0aec9..20a74f4 100644 --- a/src/cli/commands/manage/index.ts +++ b/src/cli/commands/manage/index.ts @@ -37,7 +37,7 @@ KV Store Options: 1. FASTLY_API_TOKEN environment variable 2. Logged-in Fastly CLI profile -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: diff --git a/src/cli/commands/manage/publish-content.ts b/src/cli/commands/manage/publish-content.ts index a1da11a..7dbd1ca 100644 --- a/src/cli/commands/manage/publish-content.ts +++ b/src/cli/commands/manage/publish-content.ts @@ -22,6 +22,7 @@ import { calculateFileSizeAndHash, enumerateFiles, rootRelative } from '../../ut import { ensureVariantFileExists, type Variants } from '../../util/variants.js'; import { loadStorageProviderFromStaticPublishRc, + StorageProvider, StorageProviderBatch, type StorageProviderBatchEntry, } from '../../storage/storage-provider.js'; @@ -56,6 +57,11 @@ Optional: --root-dir= Directory to publish from. Overrides the config file setting. Default: rootDir from publish-content.config.js + --fastly-api-token= Fastly API token for KV Store or cache access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + --overwrite-existing Always overwrite existing entries in storage, even if unchanged. Expiration: @@ -75,14 +81,9 @@ KV Store Options: local files that will be used to simulate the KV Store with the local development environment. - --fastly-api-token= Fastly API token for KV Store access. - If not set, the tool will check: - 1. FASTLY_API_TOKEN environment variable - 2. Logged-in Fastly CLI profile - --kv-overwrite Alias for --overwrite-existing. -S3 Storage Options: +S3 Storage Options (BETA): --aws-access-key-id= AWS Access Key ID and Secret Access Key used to --aws-secret-access-key= interface with S3. If not set, the tool will check: @@ -229,7 +230,7 @@ export async function action(actionArgs: string[]) { console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); // Storage Provider - let storageProvider: any; + let storageProvider: StorageProvider; try { storageProvider = await loadStorageProviderFromStaticPublishRc(staticPublisherRc, { computeAppDir, @@ -594,6 +595,8 @@ export async function action(actionArgs: string[]) { console.log(`✅ Settings have been saved.`); + await storageProvider.purgeSurrogateKey(`${publishId}-${collectionName}`); + console.log(`🎉 Completed.`); } diff --git a/src/cli/commands/scaffold/index.ts b/src/cli/commands/scaffold/index.ts index 9e17a4f..ac23c5d 100644 --- a/src/cli/commands/scaffold/index.ts +++ b/src/cli/commands/scaffold/index.ts @@ -32,8 +32,9 @@ Description: management mode. Options: - --storage-mode (required) Storage mode for content storage. - Can be "kv-store" or "s3". + --storage-mode Storage mode for content storage. + Can be "kv-store" or "s3" (BETA). + (default: kv-store) If --storage-mode=kv-store, then: --kv-store-name (required) Name of the KV Store. @@ -45,8 +46,8 @@ Options: --root-dir (required) Path to static content (e.g., ./public) -o, --output Output directory for Compute app (default: ./compute-js) --static-publisher-working-dir Working directory for build artifacts (default: /static-publisher) - --publish-id Advanced. Prefix for KV keys (default: "default") (default: ./compute-js/static-publisher) + --publish-id Advanced. Prefix for KV keys (default: "default") Compute Service Metadata: --name App name (for fastly.toml) @@ -407,8 +408,8 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, - // Required. Storage mode for content storage. Can be "kv-store" or "s3". - { name: 'storage-mode', type: String, }, + // Storage mode for content storage. Can be "kv-store" or "s3". + { name: 'storage-mode', type: String, defaultValue: 'kv-store', }, // If storage-mode=kv-store, then: @@ -651,7 +652,7 @@ export async function action(actionArgs: string[]) { const fastlyServiceId = options.serviceId; const storageMode = options.storageMode; if (!(storageMode === 'kv-store' || storageMode === 's3')) { - console.error(`❌ required parameter --storage-mode must be set to 'kv-store' or 's3'.`); + console.error(`❌ parameter --storage-mode must be set to 'kv-store' or 's3'.`); process.exitCode = 1; return; } @@ -725,7 +726,7 @@ export async function action(actionArgs: string[]) { console.log('Storage Mode : Fastly KV Store'); console.log('KV Store Name :', kvStoreName); } else if (storageMode === 's3') { - console.log('Storage Name : S3-compatible storage'); + console.log('Storage Name : S3-compatible storage (BETA)'); console.log('S3 Region :', s3Region); console.log('S3 Bucket :', s3Bucket); console.log('S3 Endpoint :', String(s3EndpointUrl)); @@ -808,10 +809,15 @@ export async function action(actionArgs: string[]) { if (storageMode === 'kv-store') { const localServerKvStorePath = dotRelative(computeJsDir, path.resolve(staticPublisherWorkingDir, 'kvstore.json')); fastlyTomlLocalServer = /* language=text */ `\ +[local_server] + [local_server.kv_stores] ${kvStoreName} = { file = "${localServerKvStorePath}", format = "json" } `; fastlyTomlSetup = /* language=text */ `\ +[setup] + +[setup.kv_stores] [setup.kv_stores.${kvStoreName}] `; @@ -819,6 +825,8 @@ ${kvStoreName} = { file = "${localServerKvStorePath}", format = "json" } resourceFiles[localServerKvStorePath] = '{}'; } else if (storageMode === 's3') { fastlyTomlLocalServer = /* language=text */ `\ +[local_server] + [local_server.secret_stores] [[local_server.secret_stores.AWS_CREDENTIALS]] key = "AWS_ACCESS_KEY_ID" @@ -833,10 +841,21 @@ url = "${String(s3EndpointUrl)}" override_host = "${s3EndpointUrl!.hostname}" `; fastlyTomlSetup = /* language=text */ `\ +[setup] + +[setup.backends] [setup.backends.aws] address = "${s3EndpointUrl!.hostname}" description = "S3 API endpoint" port = 443 +[setup.secret_stores] +[setup.secret_stores.AWS_CREDENTIALS] +description = "Credentials for S3 storage" +[setup.secret_stores.AWS_CREDENTIALS.items] +[setup.secret_stores.AWS_CREDENTIALS.items.AWS_ACCESS_KEY_ID] +description = "AWS Access Key ID" +[setup.secret_stores.AWS_CREDENTIALS.items.AWS_SECRET_ACCESS_KEY] +description = "AWS Secret Access Key" `; } diff --git a/src/cli/storage/kv-store-local-provider.ts b/src/cli/storage/kv-store-local-provider.ts index cad6e33..5ef4cf7 100644 --- a/src/cli/storage/kv-store-local-provider.ts +++ b/src/cli/storage/kv-store-local-provider.ts @@ -186,4 +186,7 @@ export class KvStoreLocalProvider implements StorageProvider { // to save time by checking for an existing item. This is not applicable for local. return null; } + + async purgeSurrogateKey(_surrogateKey: string): Promise { + } } diff --git a/src/cli/storage/kv-store-provider.ts b/src/cli/storage/kv-store-provider.ts index 4c19545..f63479b 100644 --- a/src/cli/storage/kv-store-provider.ts +++ b/src/cli/storage/kv-store-provider.ts @@ -267,6 +267,9 @@ export class KvStoreProvider implements StorageProvider { return kvStoreItemMetadata; } + + async purgeSurrogateKey(_surrogateKey: string): Promise { + } } export function kvStoreEntryToStorageEntry( diff --git a/src/cli/storage/s3-storage-provider.ts b/src/cli/storage/s3-storage-provider.ts index 6a9fc8b..6c5c857 100644 --- a/src/cli/storage/s3-storage-provider.ts +++ b/src/cli/storage/s3-storage-provider.ts @@ -4,6 +4,7 @@ */ import fs from 'node:fs'; +import path from 'node:path'; import { DeleteObjectCommand, DeleteObjectCommandInput, @@ -51,6 +52,16 @@ import { import { rootRelative, } from '../util/files.js'; +import { + type FastlyApiContext, + loadApiToken, +} from '../util/api-token.js'; +import { + readServiceId, +} from '../util/fastly-toml.js'; +import { + purgeSurrogateKey, +} from '../util/purge.js'; type CommandOutput = C extends Command ? O : never; @@ -59,7 +70,7 @@ export const buildStoreProvider: StorageProviderBuilder = async ( context: StorageProviderBuilderContext, ) => { if (isS3StorageConfigRc(config)) { - console.log(` Working on S3 (or compatible) storage...`); + console.log(` Working on S3 (or compatible) storage (BETA)...`); } else { return null; } @@ -69,7 +80,7 @@ export const buildStoreProvider: StorageProviderBuilder = async ( bucket, endpoint, } = getS3StorageConfigFromRc(config); - console.log(` | Using S3 storage`); + console.log(` | Using S3 storage (BETA)`); console.log(` Region : ${region}`); console.log(` Bucket : ${bucket}`); console.log(` Endpoint: ${endpoint ?? 'default'}`); @@ -83,7 +94,27 @@ export const buildStoreProvider: StorageProviderBuilder = async ( throw new Error("❌ S3 Credentials not provided.\nProvide an AWS access key ID and secret access key that has write access to the S3 Storage.\nRefer to the README file and --help for additional information."); } console.log(`✔️ S3 Credentials: ${awsCredentialsResult.awsAccessKeyId.slice(0, 4)}${'*'.repeat(awsCredentialsResult.awsAccessKeyId.length-4)} from '${awsCredentialsResult.source}'`); + + const fastlyTomlPath = path.resolve(context.computeAppDir, 'fastly.toml'); + const serviceId = readServiceId(fastlyTomlPath); + + let apiToken = undefined; + if (serviceId == null) { + console.log(`- Service ID not found in fastly.toml, application may not have been deployed yet. Will skip purge step after publish.`); + } else { + console.log(`✔️ Service ID from fastly.toml: ${serviceId}`); + + const apiTokenResult = loadApiToken({commandLine: context.fastlyApiToken}); + if (apiTokenResult == null) { + throw new Error("❌ Fastly API Token not provided.\nSet the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + } + console.log(`✔️ Fastly API Token: ${apiTokenResult.apiToken.slice(0, 4)}${'*'.repeat(apiTokenResult.apiToken.length - 4)} from '${apiTokenResult.source}'`); + apiToken = apiTokenResult.apiToken; + } + return new S3StorageProvider( + serviceId, + apiToken, region, awsCredentialsResult.awsAccessKeyId, awsCredentialsResult.awsSecretAccessKey, @@ -95,12 +126,16 @@ export const buildStoreProvider: StorageProviderBuilder = async ( export class S3StorageProvider implements StorageProvider { constructor( + fastlyServiceId: string | undefined, + fastlyApiToken: string | undefined, s3Region: string, accessKeyId: string, secretAccessKey: string, s3Bucket: string, s3Endpoint?: string, ) { + this.fastlyServiceId = fastlyServiceId; + this.fastlyApiContext = fastlyApiToken != null ? { apiToken: fastlyApiToken } : undefined; this.s3Region = s3Region; this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; @@ -108,6 +143,8 @@ export class S3StorageProvider implements StorageProvider { this.s3Endpoint = s3Endpoint; } + private readonly fastlyServiceId?: string; + private readonly fastlyApiContext?: FastlyApiContext; private readonly s3Region: string; private readonly accessKeyId: string; private readonly secretAccessKey: string; @@ -326,4 +363,32 @@ export class S3StorageProvider implements StorageProvider { return assetVariantMetadata; } + async purgeSurrogateKey(surrogateKey: string): Promise { + + if (this.fastlyServiceId == null) { + console.log('Fastly Service ID not set, skipping purge...'); + return; + } + + if (this.fastlyApiContext?.apiToken == null) { + console.log('Fastly API token not set, skipping purge...'); + return; + } + + console.log(`Purging surrogate key [${surrogateKey}] on service [${this.fastlyServiceId}]...`); + + const result = await purgeSurrogateKey( + this.fastlyApiContext, + this.fastlyServiceId, + surrogateKey, + true, + ); + + if (result) { + console.log('Purged'); + } else { + console.log('Failed purging'); + } + + } } diff --git a/src/cli/storage/storage-provider.ts b/src/cli/storage/storage-provider.ts index 498ca8d..01b5adc 100644 --- a/src/cli/storage/storage-provider.ts +++ b/src/cli/storage/storage-provider.ts @@ -36,6 +36,8 @@ export interface StorageProvider { calculateNumChunks(size: number): number; getExistingAssetVariant(variantKey: string): Promise; + + purgeSurrogateKey(surrogateKey: string): Promise; } export type StorageProviderBatchEntry = { diff --git a/src/cli/util/purge.ts b/src/cli/util/purge.ts new file mode 100644 index 0000000..26cd27b --- /dev/null +++ b/src/cli/util/purge.ts @@ -0,0 +1,35 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { callFastlyApi, type FastlyApiContext, FetchError } from './api-token.js'; + +export async function purgeSurrogateKey( + fastlyApiContext: FastlyApiContext, + fastlyServiceId: string, + surrogateKey: string, + softPurge: boolean = false, +) { + + const endpoint = `/service/${encodeURIComponent(fastlyServiceId)}/purge`; + + try { + + const headers = new Headers(); + headers.set('surrogate-key', surrogateKey); + if (softPurge) { + headers.set('fastly-soft-purge', '1'); + } + await callFastlyApi(fastlyApiContext, endpoint, `Purging surrogate key [${surrogateKey}] on service [${fastlyServiceId}]`, null, { method: 'POST', headers }); + + } catch(err) { + if (err instanceof FetchError) { + return false; + } + throw err; + } + + return true; + +} \ No newline at end of file diff --git a/src/models/config/static-publish-rc.ts b/src/models/config/static-publish-rc.ts index 27968ff..d11a777 100644 --- a/src/models/config/static-publish-rc.ts +++ b/src/models/config/static-publish-rc.ts @@ -64,7 +64,7 @@ export function isKvStoreConfigRc(rc: unknown): rc is StaticPublishKvStore { return false; } -// S3 Storage config +// S3 Storage config (BETA) export type StaticPublishPartialS3Storage = { storageMode: 's3', s3: { diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts index c94ac94..e4ce207 100644 --- a/src/server/publisher-server/index.ts +++ b/src/server/publisher-server/index.ts @@ -120,7 +120,7 @@ export class PublisherServer { return this.settingsCached; } const settingsFileKey = `${this.publishId}_settings_${this.activeCollectionName}`; - const settingsFile = await this.storageProvider.getEntry(settingsFileKey); + const settingsFile = await this.storageProvider.getEntry(settingsFileKey, [`${this.publishId}-${this.activeCollectionName}`, 'settings']); if (settingsFile == null) { console.error(`Settings File not found at ${settingsFileKey}.`); console.error(`You may need to publish your application.`); @@ -156,7 +156,7 @@ export class PublisherServer { return this.assetEntryMapCache; } const indexFileKey = `${this.publishId}_index_${this.activeCollectionName}`; - const indexFile = await this.storageProvider.getEntry(indexFileKey); + const indexFile = await this.storageProvider.getEntry(indexFileKey, [`${this.publishId}-${this.activeCollectionName}`, 'index']); if (indexFile == null) { console.error(`Index File not found at ${indexFileKey}.`); console.error(`You may need to publish your application.`); diff --git a/src/server/storage/s3-storage-provider.ts b/src/server/storage/s3-storage-provider.ts index 967464c..1a54071 100644 --- a/src/server/storage/s3-storage-provider.ts +++ b/src/server/storage/s3-storage-provider.ts @@ -3,14 +3,19 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import { CacheOverride } from 'fastly:cache-override'; import { SecretStore } from 'fastly:secret-store'; +import { Command } from '@smithy/types'; import { FetchHttpHandler } from '@smithy/fetch-http-handler'; import { GetObjectCommand, GetObjectCommandInput, GetObjectCommandOutput, S3Client, - S3ServiceException + S3ClientResolvedConfig, + S3ServiceException, + ServiceInputTypes, + ServiceOutputTypes, } from '@aws-sdk/client-s3'; import { @@ -66,7 +71,11 @@ export function setSecretStoreKeyForAwsSecretAccessKey(secretStoreKey: string) { _secretStoreKeyForAwsSecretAccessKey = secretStoreKey; } +let _awsCredentialsFromSecretStore: AwsCredentials | undefined = undefined; export async function buildAwsCredentialsFromSecretStore() { + if (_awsCredentialsFromSecretStore != null) { + return _awsCredentialsFromSecretStore; + } let secretStore; try { secretStore = new SecretStore(_secretStoreForAwsCredentials); @@ -85,10 +94,11 @@ export async function buildAwsCredentialsFromSecretStore() { } const secretAccessKey = secretAccessKeyEntry.plaintext(); - return { + _awsCredentialsFromSecretStore = { accessKeyId, secretAccessKey, }; + return _awsCredentialsFromSecretStore; } let _awsCredentialsBuilder: AwsCredentialsBuilder = buildAwsCredentialsFromSecretStore; @@ -101,6 +111,9 @@ export type S3StorageProviderParams = { s3FastlyBackendName?: string, }; +type S3ClientCommand = + Command; + export class S3StorageProvider implements StorageProvider { constructor( s3Region: string, @@ -118,14 +131,12 @@ export class S3StorageProvider implements StorageProvider { private readonly s3Endpoint?: string; private readonly s3FastlyBackendName?: string; - private s3Client?: S3Client; - async getS3Client() { - if (this.s3Client != null) { - return this.s3Client; - } + async sendS3Command( + command: S3ClientCommand, + requestInit?: RequestInit, + ): Promise { const awsCredentials = await _awsCredentialsBuilder(); - const s3FastlyBackendName = this.s3FastlyBackendName ?? "aws"; - this.s3Client = new S3Client({ + const s3Client = new S3Client({ region: this.s3Region, endpoint: this.s3Endpoint, forcePathStyle: this.s3Endpoint != null, @@ -133,16 +144,17 @@ export class S3StorageProvider implements StorageProvider { accessKeyId: awsCredentials.accessKeyId, secretAccessKey: awsCredentials.secretAccessKey, }, - maxAttempts: 1, + maxAttempts: 5, requestHandler: new FetchHttpHandler({ - requestInit() { return { backend: s3FastlyBackendName } } + requestInit() { + return requestInit ?? {}; + }, }), }); - return this.s3Client; + return s3Client.send(command); } - async getEntry(key: string): Promise { - + async getEntry(key: string, tags?: string[]): Promise { const input = { Bucket: this.s3Bucket, // required Key: key, // required @@ -150,8 +162,13 @@ export class S3StorageProvider implements StorageProvider { const command = new GetObjectCommand(input); let response: GetObjectCommandOutput; try { - const s3Client = await this.getS3Client(); - response = await s3Client.send(command); + response = await this.sendS3Command(command, { + backend: this.s3FastlyBackendName ?? "aws", + cacheOverride: new CacheOverride({ + ttl: 3600, + surrogateKey: (tags ?? []).join(' ') || undefined, + }), + }); } catch(err) { if (err instanceof S3ServiceException && (err.name === "NotFound" || err.name === "NoSuchKey")) { console.log("Object does not exist"); diff --git a/src/server/storage/storage-provider.ts b/src/server/storage/storage-provider.ts index b1fa63f..b35ee9a 100644 --- a/src/server/storage/storage-provider.ts +++ b/src/server/storage/storage-provider.ts @@ -8,7 +8,7 @@ import { } from '../../models/config/static-publish-rc.js'; export interface StorageProvider { - getEntry(key: string): Promise; + getEntry(key: string, tags?: string[]): Promise; } export type StorageProviderBuilder = (config: StaticPublishRc) => (StorageProvider | null);