Issue #28: Increase sleep timeout to prevent api rate limiting

code-build.js

@@ -35,13 +35,16 @@ async function build(sdk, params) {
 }
 
 async function waitForBuildEndTime(sdk, { id, logs }, nextToken) {
-  const { codeBuild, cloudWatchLogs, wait = 1000 * 5 } = sdk;
+  const { codeBuild, cloudWatchLogs, wait = 1000 * 30 } = sdk;
 
   // Get the CloudWatchLog info
   const startFromHead = true;
   const { cloudWatchLogsArn } = logs;
   const { logGroupName, logStreamName } = logName(cloudWatchLogsArn);
 
+  let errorDetected = false;
+  let errMessage = "";
+
   // Check the state
   const [batch, cloudWatch = {}] = await Promise.all([
     codeBuild.batchGetBuilds({ ids: [id] }).promise(),
@@ -50,7 +53,37 @@ async function waitForBuildEndTime(sdk, { id, logs }, nextToken) {
       cloudWatchLogs
         .getLogEvents({ logGroupName, logStreamName, startFromHead, nextToken })
         .promise()
-  ]);
+  ]).catch(err => {
+    errorDetected = true;
+    errMessage = err;
+
+    return [];
+  });
+
+  if (errorDetected) {
+    //We caught an error in trying to make the AWS api call, and are now checking to see if it was just a rate limiting error
+    if (
+      errMessage.message &&
+      errMessage.message.search("Rate exceeded") !== -1
+    ) {
+      //We were rate-limited, so add 15 seconds to the wait time
+      let newWait = wait + 15000;
+
+      //Sleep before trying again
+      await new Promise(resolve => setTimeout(resolve, newWait));
+
+      // Try again from the same token position
+      return waitForBuildEndTime(
+        { ...sdk, wait: newWait },
+        { id, logs },
+        nextToken
+      );
+    } else {
+      //The error returned from the API wasn't about rate limiting, so throw it as an actual error and fail the job
+      throw errMessage;
+    }
+  }
+
   // Pluck off the relevant state
   const [current] = batch.builds;
   const { nextForwardToken, events = [] } = cloudWatch;
@@ -64,7 +97,7 @@ async function waitForBuildEndTime(sdk, { id, logs }, nextToken) {
   // We did it! We can stop looking!
   if (current.endTime && !events.length) return current;
 
-  // More to do: Sleep for 5 seconds :)
+  // More to do: Sleep for a few seconds to avoid rate limiting
   await new Promise(resolve => setTimeout(resolve, wait));
 
   // Try again

test/code-build-test.js

@@ -318,6 +318,104 @@ describe("waitForBuildEndTime", () => {
     });
     expect(test).to.equal(buildReplies.pop().builds[0]);
   });
+
+  it("waits after being rate limited and tries again", async function() {
+    this.timeout(45000);
+    const buildID = "buildID";
+    const nullArn =
+      "arn:aws:logs:us-west-2:111122223333:log-group:null:log-stream:null";
+    const cloudWatchLogsArn =
+      "arn:aws:logs:us-west-2:111122223333:log-group:/aws/codebuild/CloudWatchLogGroup:log-stream:1234abcd-12ab-34cd-56ef-1234567890ab";
+
+    const buildReplies = [
+      () => {
+        throw { message: "Rate exceeded" };
+      },
+      { builds: [{ id: buildID, logs: { cloudWatchLogsArn } }] },
+      {
+        builds: [
+          { id: buildID, logs: { cloudWatchLogsArn }, endTime: "endTime" }
+        ]
+      }
+    ];
+
+    const sdk = help(
+      () => {
+        //similar to the ret function in the helper, allows me to throw an error in a function or return a more standard reply
+        let reply = buildReplies.shift();
+
+        if (typeof reply === "function") return reply();
+        return reply;
+      },
+      () => {
+        if (!buildReplies.length) {
+          return { events: [] };
+        }
+
+        return { events: [{ message: "got one" }] };
+      }
+    );
+
+    const test = await waitForBuildEndTime(sdk, {
+      id: buildID,
+      logs: { cloudWatchLogsArn: nullArn }
+    });
+
+    expect(test.id).to.equal(buildID);
+  });
+
+  it("dies after getting an error from the aws sdk that isn't rate limiting", async function() {
+    this.timeout(45000);
+    const buildID = "buildID";
+    const nullArn =
+      "arn:aws:logs:us-west-2:111122223333:log-group:null:log-stream:null";
+    const cloudWatchLogsArn =
+      "arn:aws:logs:us-west-2:111122223333:log-group:/aws/codebuild/CloudWatchLogGroup:log-stream:1234abcd-12ab-34cd-56ef-1234567890ab";
+
+    const buildReplies = [
+      () => {
+        throw { message: "Some AWS error" };
+      },
+      { builds: [{ id: buildID, logs: { cloudWatchLogsArn } }] },
+      {
+        builds: [
+          { id: buildID, logs: { cloudWatchLogsArn }, endTime: "endTime" }
+        ]
+      }
+    ];
+
+    const sdk = help(
+      () => {
+        //similar to the ret function in the helper, allows me to throw an error in a function or return a more standard reply
+        let reply = buildReplies.shift();
+
+        if (typeof reply === "function") return reply();
+        return reply;
+      },
+      () => {
+        if (!buildReplies.length) {
+          return { events: [] };
+        }
+
+        return { events: [{ message: "got one" }] };
+      }
+    );
+
+    //run the thing and it should fail
+    let didFail = false;
+
+    try {
+      await waitForBuildEndTime(sdk, {
+        id: buildID,
+        logs: { cloudWatchLogsArn: nullArn }
+      });
+    } catch (err) {
+      didFail = true;
+      expect(err.message).to.equal("Some AWS error");
+    }
+
+    expect(didFail).to.equal(true);
+  });
 });
 
 function help(builds, logs) {