Merge pull request #210 from circleci/ONPREM-2431/improve-readiness-check

[ONPREM-2431] Improve service container readiness check resiliency

Author: christian-stephen

task/orchestrator.go

@@ -72,7 +72,7 @@ func (o *Orchestrator) Run(parentCtx context.Context) (err error) {
 	if len(o.config.ReadinessFilePath) > 0 {
 		// Wait for readiness from the other containers before starting the task agent process
 		if err := o.waitForReadiness(ctx); err != nil {
-			return err
+			return taskerrors.RetryableErrorf("error waiting for service containers to become ready: %w", err)
 		}
 	}
 
@@ -120,7 +120,7 @@ func (o *Orchestrator) waitForReadiness(ctx context.Context) (err error) {
 	ctx, cancel := context.WithTimeout(ctx, waitForReadinessTimeout) // so we don't wait indefinitely if there's a problem
 	defer cancel()
 
-	watcher, err := fsnotify.NewWatcher()
+	watcher, err := fsnotify.NewBufferedWatcher(100) // use a buffered chan to prevent a possible race condition
 	if err != nil {
 		return err
 	}

task/orchestrator_test.go

@@ -59,6 +59,7 @@ func TestOrchestrator(t *testing.T) {
 		gracePeriod     time.Duration
 		timeout         time.Duration
 		additionalTasks []fakerunnerapi.Task
+		cleanup         func()
 
 		wantError        string
 		wantTimeout      bool
@@ -196,6 +197,32 @@ func TestOrchestrator(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "retryable error: service containers didn't become ready in time",
+			config: func() Config {
+				waitForReadinessTimeout = 1 * time.Millisecond
+				c := defaultConfig
+				c.ReadinessFilePath = "does-not-exist"
+				return c
+			}(),
+			cleanup: func() {
+				waitForReadinessTimeout = 10 * time.Minute
+			},
+			wantError: "error waiting for service containers to become ready: context deadline exceeded",
+			wantTaskUnclaims: []fakerunnerapi.TaskUnclaim{
+				{
+					Token: "testtoken",
+				},
+			},
+			wantTaskEvents: []fakerunnerapi.TaskEvent{
+				{
+					Allocation:     defaultConfig.Allocation,
+					TimestampMilli: time.Now().UnixMilli(),
+					Message: []byte("error waiting for service containers to become ready: context deadline exceeded: " +
+						"Check container logs for more details"),
+				},
+			},
+		},
 		{
 			name: "retryable error: an unsafe retry",
 			config: Config{
@@ -252,6 +279,10 @@ func TestOrchestrator(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Setenv("BE_TASK_AGENT", "true")
 
+			if tt.cleanup != nil {
+				t.Cleanup(tt.cleanup)
+			}
+
 			for k, v := range tt.env {
 				t.Setenv(k, v)
 			}