feat(queue): Make batch size configurable for artifact removal

adds `EXPIRE_ARTIFACTS_BATCH_SIZE` env var support

Author: lotas

changelog/issue-6405.md

@@ -4,3 +4,4 @@ reference: issue 6405
 ---
 
 Expire artifacts supports both bulk deletion and single deletion. This can be configured for the deployment using `AWS_USE_BULK_DELETE` environment variable (`false` by default). This is needed because not all S3 compatible storages support bulk delete, specifically [GCS](https://cloud.google.com/storage/docs/migrating#methods-comparison).
+`EXPIRE_ARTIFACTS_BATCH_SIZE` can be used to control how many records to process at once, i.e. how many parallel delete requests would be sent to storage service (`100` by default).

dev-docs/dev-config-example.yml

@@ -7,6 +7,7 @@ SIGN_PUBLIC_ARTIFACT_URLS=
 WORKER_INFO_UPDATE_FREQUENCY=
 PUBLIC_ARTIFACT_BUCKET_CDN=
 TASK_CACHE_MAX_SIZE=
+EXPIRE_ARTIFACTS_BATCH_SIZE=
 TASKCLUSTER_ROOT_URL=http://taskcluster
 TASKCLUSTER_CLIENT_ID=static/taskcluster/queue
 TASKCLUSTER_ACCESS_TOKEN=j2Z6zW2QSLehailBXlosdw9e2Ti8R_Qh2M4buAEQfsMA

docker/env/.queue

@@ -1057,6 +1057,9 @@
           "title": "node debug env var",
           "type": "string"
         },
+        "expire_artifacts_batch_size": {
+          "type": "number"
+        },
         "level": {
           "type": "string"
         },

generated/references.json

@@ -71,7 +71,10 @@ defaults:
     # CDN for public artifact bucket
     publicArtifactBucketCDN:      !env:optional PUBLIC_ARTIFACT_BUCKET_CDN
 
-    taskCacheMaxSize: !env:number:optional TASK_CACHE_MAX_SIZE
+    taskCacheMaxSize:             !env:number:optional TASK_CACHE_MAX_SIZE
+
+    # How many artifacts to remove at once when bulk deletion is disabled
+    expireArtifactsBatchSize:     !env:number:optional EXPIRE_ARTIFACTS_BATCH_SIZE
 
 
   taskcluster:

infrastructure/k8s/templates/taskcluster-queue-secret.yaml

@@ -29,6 +29,9 @@ const DEFAULT_CLAIM_TIMEOUT = 1200;
 // default `expires`, `last_date_active` update frequency
 const DEFAULT_UPDATE_FREQUENCY = '30 minutes';
 
+// for supported bulk deletion S3 operations
+const MAX_BULK_DELETE_SIZE = 1000;
+
 require('./monitor');
 
 // Create component loader
@@ -297,15 +300,24 @@ let load = loader({
     setup: ({ cfg, db, publicArtifactBucket, privateArtifactBucket, monitor }, ownName) => {
       return monitor.oneShot(ownName, async () => {
         const now = taskcluster.fromNow(cfg.app.artifactExpirationDelay);
-        debug('Expiring artifacts at: %s, from before %s', new Date(), now);
+        const useBulkDelete = !!cfg.aws.useBulkDelete;
+        // when using bulk delete, maximum number of objects in bulk request is 1000
+        // if using single delete, we have to be cautious not to overload the API with too many parallel requests
+        // so we limit the batch size to 100 by default, which proved to be a good value in production
+        const expireArtifactsBatchSize = useBulkDelete ? MAX_BULK_DELETE_SIZE : (cfg.expireArtifactsBatchSize || 100);
+
+        debug('Expiring artifacts at: %s, from before %s, useBulkDelete: %s, batchSize: %d',
+          new Date(), now, useBulkDelete, expireArtifactsBatchSize);
+
         const count = await artifactUtils.expire({
           db,
           publicBucket: publicArtifactBucket,
           privateBucket: privateArtifactBucket,
           monitor,
           ignoreError: false,
           expires: now,
-          useBulkDelete: !!cfg.aws.useBulkDelete,
+          expireArtifactsBatchSize,
+          useBulkDelete,
         });
         debug('Expired %s artifacts', count);
         monitor.log.expiredArtifactsRemoved({ count, expires: now });

infrastructure/k8s/values.schema.json

@@ -42,17 +42,18 @@ const artifactUtils = {
    * need to handle that case.
    */
   async expire({ db, publicBucket, privateBucket, ignoreError, monitor,
-    expires, useBulkDelete = false }) {
+    expires, useBulkDelete, expireArtifactsBatchSize }) {
     let count = 0;
-    const maxPageSize = useBulkDelete ? 1000 : 100;
+
+    assert(!useBulkDelete || expireArtifactsBatchSize <= 1000, 'expireArtifactsBatchSize must be <= 1000 when useBulkDelete is true');
 
     // Fetch all expired artifacts and batch delete the S3 ones
     // then remove the entity from the database
     // repeat until there are no more expired artifacts
     while (true) {
       const rows = await db.fns.get_expired_artifacts_for_deletion({
         expires_in: expires,
-        page_size_in: maxPageSize,
+        page_size_in: expireArtifactsBatchSize,
       });
       if (!rows.length) {
         break;

services/queue/config.yml

@@ -17,9 +17,10 @@ helper.secrets.mockSuite(testing.suiteName(), ['aws'], function (mock, skipping)
   const MAX_ARTIFACTS = 5;
 
   [
-    ['expire s3 artifacts using bulk delete', true],
-    ['expire s3 artifacts using single delete', false],
-  ].forEach(([name, useBulkDelete]) =>
+    ['expire s3 artifacts using bulk delete', true, undefined],
+    ['expire s3 artifacts using single delete', false, undefined],
+    ['expire s3 artifacts using single delete and batch size 1', false, 1],
+  ].forEach(([name, useBulkDelete, batchSize]) =>
     test(name, async () => {
       const yesterday = taskcluster.fromNow('-1 day');
       const today = new Date();
@@ -28,6 +29,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['aws'], function (mock, skipping)
 
       await helper.load('cfg');
       helper.load.cfg('aws.useBulkDelete', useBulkDelete);
+      if (batchSize) {
+        helper.load.cfg('expireArtifactsBatchSize', batchSize);
+      }
 
       for (let i = 0; i < MAX_ARTIFACTS; i++) {
         await helper.db.fns.create_queue_artifact(