codeintel: first implementation of auto-indexing secrets (#45580)

Author: Strum355

client/web/src/enterprise/executors/secrets/ExecutorSecretsListPage.tsx

@@ -154,16 +154,18 @@ const ExecutorSecretsListPage: React.FunctionComponent<React.PropsWithChildren<E
             )}
 
             <div className="d-flex mb-3">
-                {Object.values(ExecutorSecretScope).map(scope => (
-                    <ExecutorSecretScopeSelector
-                        key={scope}
-                        scope={scope}
-                        label={executorSecretScopeContext(scope).label}
-                        onSelect={() => setSelectedScope(scope)}
-                        selected={scope === selectedScope}
-                        description={executorSecretScopeContext(scope).description}
-                    />
-                ))}
+                {(namespaceID === null ? Object.values(ExecutorSecretScope) : [ExecutorSecretScope.BATCHES]).map(
+                    scope => (
+                        <ExecutorSecretScopeSelector
+                            key={scope}
+                            scope={scope}
+                            label={executorSecretScopeContext(scope).label}
+                            onSelect={() => setSelectedScope(scope)}
+                            selected={scope === selectedScope}
+                            description={executorSecretScopeContext(scope).description}
+                        />
+                    )
+                )}
             </div>
 
             <Container>
@@ -207,5 +209,7 @@ function executorSecretScopeContext(scope: ExecutorSecretScope): { label: string
     switch (scope) {
         case ExecutorSecretScope.BATCHES:
             return { label: 'Batch changes', description: 'Batch change execution secrets' }
+        case ExecutorSecretScope.CODEINTEL:
+            return { label: 'Code graph', description: 'Code graph execution secrets' }
     }
 }

client/web/src/enterprise/executors/secrets/SecretAccessLogsModal.tsx

@@ -77,13 +77,10 @@ const ExecutorSecretAccessLogNode: React.FunctionComponent<React.PropsWithChildr
         <div className="d-flex justify-content-between align-items-center flex-wrap mb-0">
             <PersonLink
                 person={{
-                    displayName: node.user.displayName || node.user.username,
-                    email: node.user.email,
-                    user: {
-                        displayName: node.user.displayName,
-                        url: node.user.url,
-                        username: node.user.username,
-                    },
+                    // empty strings are fine here, as they are only used when `user` is not null
+                    displayName: (node.user?.displayName || node.user?.username) ?? '',
+                    email: node.user?.email ?? '',
+                    user: node.user,
                 }}
             />
             <Timestamp date={node.createdAt} />

cmd/frontend/graphqlbackend/executor_secret_access_log.go

@@ -77,7 +77,11 @@ func (r *executorSecretAccessLogResolver) User(ctx context.Context) (*UserResolv
 		return NewUserResolver(r.db, r.preloadedUser), nil
 	}
 
-	u, err := UserByIDInt32(ctx, r.db, r.log.UserID)
+	if r.log.UserID == nil {
+		return nil, nil
+	}
+
+	u, err := UserByIDInt32(ctx, r.db, *r.log.UserID)
 	if err != nil {
 		if errcode.IsNotFound(err) {
 			return nil, nil
@@ -87,6 +91,10 @@ func (r *executorSecretAccessLogResolver) User(ctx context.Context) (*UserResolv
 	return u, nil
 }
 
+func (r *executorSecretAccessLogResolver) MachineUser() string {
+	return r.log.MachineUser
+}
+
 func (r *executorSecretAccessLogResolver) CreatedAt() gqlutil.DateTime {
 	return gqlutil.DateTime{Time: r.log.CreatedAt}
 }

cmd/frontend/graphqlbackend/executor_secret_access_logs_connection.go

@@ -38,8 +38,10 @@ func (r *executorSecretAccessLogConnectionResolver) Nodes(ctx context.Context) (
 			log:                  log,
 			attemptPreloadedUser: true,
 		}
-		if user, ok := userMap[log.UserID]; ok {
-			r.preloadedUser = user
+		if log.UserID != nil {
+			if user, ok := userMap[*log.UserID]; ok {
+				r.preloadedUser = user
+			}
 		}
 		resolvers = append(resolvers, r)
 	}
@@ -75,9 +77,12 @@ func (r *executorSecretAccessLogConnectionResolver) compute(ctx context.Context)
 			userIDMap := make(map[int32]struct{})
 			userIDs := []int32{}
 			for _, log := range r.logs {
-				if _, ok := userIDMap[log.UserID]; !ok {
-					userIDMap[log.UserID] = struct{}{}
-					userIDs = append(userIDs, log.UserID)
+				if log.UserID == nil {
+					continue
+				}
+				if _, ok := userIDMap[*log.UserID]; !ok {
+					userIDMap[*log.UserID] = struct{}{}
+					userIDs = append(userIDs, *log.UserID)
 				}
 			}
 			r.users, r.err = r.db.Users().List(ctx, &database.UsersListOptions{UserIDs: userIDs})

cmd/frontend/graphqlbackend/schema.graphql

@@ -1723,6 +1723,11 @@ enum ExecutorSecretScope {
     The secret is meant to be used with Batch Changes execution.
     """
     BATCHES
+
+    """
+    The secret is meant to be used with Auto-indexing.
+    """
+    CODEINTEL
 }
 
 """
@@ -1837,8 +1842,14 @@ type ExecutorSecretAccessLog implements Node {
     executorSecret: ExecutorSecret!
     """
     The user in which name the secret has been used.
+    This is null when the access was not by a user account, or
+    when the user account was deleted.
     """
-    user: User!
+    user: User
+    """
+    True when the secret was accessed by an internal procedure.
+    """
+    machineUser: String!
     """
     The date and time when the secret has been used.
     """

enterprise/cmd/frontend/internal/executorqueue/queues/codeintel/queue.go

@@ -14,7 +14,7 @@ import (
 
 func QueueOptions(observationCtx *observation.Context, db database.DB, accessToken func() string) handler.QueueOptions[types.Index] {
 	recordTransformer := func(ctx context.Context, _ string, record types.Index, resourceMetadata handler.ResourceMetadata) (apiclient.Job, error) {
-		return transformRecord(record, resourceMetadata, accessToken())
+		return transformRecord(ctx, record, db, resourceMetadata, accessToken())
 	}
 
 	store := store.New(observationCtx, db.Handle(), autoindexing.IndexWorkerStoreOptions)

enterprise/cmd/frontend/internal/executorqueue/queues/codeintel/transform.go

@@ -1,34 +1,85 @@
 package codeintel
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"strings"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/c2h5oh/datasize"
 	"github.com/kballard/go-shellquote"
 
 	"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/executorqueue/handler"
 	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/shared/types"
 	apiclient "github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
 	"github.com/sourcegraph/sourcegraph/internal/conf"
+	"github.com/sourcegraph/sourcegraph/internal/database"
+	"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
+)
+
+const (
+	defaultOutfile      = "dump.lsif"
+	uploadRoute         = "/.executors/lsif/upload"
+	schemeExecutorToken = "token-executor"
 )
 
-const defaultOutfile = "dump.lsif"
-const uploadRoute = "/.executors/lsif/upload"
-const schemeExecutorToken = "token-executor"
+// accessLogTransformer sets the approriate fields on the executor secret access log entry
+// for auto-indexing access
+type accessLogTransformer struct {
+	database.ExecutorSecretAccessLogCreator
+}
+
+func (e *accessLogTransformer) Create(ctx context.Context, log *database.ExecutorSecretAccessLog) error {
+	log.MachineUser = "codeintel-autoindexing"
+	log.UserID = nil
+	return e.ExecutorSecretAccessLogCreator.Create(ctx, log)
+}
 
-func transformRecord(index types.Index, resourceMetadata handler.ResourceMetadata, accessToken string) (apiclient.Job, error) {
+func transformRecord(ctx context.Context, index types.Index, db database.DB, resourceMetadata handler.ResourceMetadata, accessToken string) (apiclient.Job, error) {
 	resourceEnvironment := makeResourceEnvironment(resourceMetadata)
 
+	var secrets []*database.ExecutorSecret
+	var err error
+	if len(index.RequestedEnvVars) > 0 {
+		secretsStore := db.ExecutorSecrets(keyring.Default().ExecutorSecretKey)
+		secrets, _, err = secretsStore.List(ctx, database.ExecutorSecretScopeCodeIntel, database.ExecutorSecretsListOpts{
+			// Note: No namespace set, codeintel secrets are only available in the global namespace for now.
+			Keys: index.RequestedEnvVars,
+		})
+		if err != nil {
+			return apiclient.Job{}, err
+		}
+	}
+
+	// And build the env vars from the secrets.
+	secretEnvVars := make([]string, len(secrets))
+	redactedEnvVars := make(map[string]string, len(secrets))
+	secretStore := &accessLogTransformer{db.ExecutorSecretAccessLogs()}
+	for i, secret := range secrets {
+		// Get the secret value. This also creates an access log entry in the
+		// name of the user.
+		val, err := secret.Value(ctx, secretStore)
+		if err != nil {
+			return apiclient.Job{}, err
+		}
+
+		secretEnvVars[i] = fmt.Sprintf("%s=%s", secret.Key, val)
+		// We redact secret values as ${{ secrets.NAME }}.
+		redactedEnvVars[val] = fmt.Sprintf("${{ secrets.%s }}", secret.Key)
+	}
+
+	envVars := append(resourceEnvironment, secretEnvVars...)
+
 	dockerSteps := make([]apiclient.DockerStep, 0, len(index.DockerSteps)+2)
 	for i, dockerStep := range index.DockerSteps {
 		dockerSteps = append(dockerSteps, apiclient.DockerStep{
 			Key:      fmt.Sprintf("pre-index.%d", i),
 			Image:    dockerStep.Image,
 			Commands: dockerStep.Commands,
 			Dir:      dockerStep.Root,
-			Env:      resourceEnvironment,
+			Env:      envVars,
 		})
 	}
 
@@ -38,7 +89,7 @@ func transformRecord(index types.Index, resourceMetadata handler.ResourceMetadat
 			Image:    index.Indexer,
 			Commands: append(index.LocalSteps, shellquote.Join(index.IndexerArgs...)),
 			Dir:      index.Root,
-			Env:      resourceEnvironment,
+			Env:      envVars,
 		})
 	}
 
@@ -57,11 +108,8 @@ func transformRecord(index types.Index, resourceMetadata handler.ResourceMetadat
 		outfile = defaultOutfile
 	}
 
-	fetchTags := false
 	// TODO: Temporary workaround. LSIF-go needs tags, but they make git fetching slower.
-	if strings.HasPrefix(index.Indexer, "sourcegraph/lsif-go") {
-		fetchTags = true
-	}
+	fetchTags := strings.HasPrefix(index.Indexer, "sourcegraph/lsif-go")
 
 	dockerSteps = append(dockerSteps, apiclient.DockerStep{
 		Key:   "upload",
@@ -87,29 +135,35 @@ func transformRecord(index types.Index, resourceMetadata handler.ResourceMetadat
 		},
 	})
 
+	allRedactedValues := map[string]string{
+		// 🚨 SECURITY: Catch leak of authorization header.
+		authorizationHeader: redactedAuthorizationHeader,
+
+		// 🚨 SECURITY: Catch uses of fragments pulled from auth header to
+		// construct another target (in src-cli). We only pass the
+		// Authorization header to src-cli, which we trust not to ship the
+		// values to a third party, but not to trust to ensure the values
+		// are absent from the command's stdout or stderr streams.
+		accessToken: "PASSWORD_REMOVED",
+	}
+	// 🚨 SECURITY: Catch uses of executor secrets from the executor secret store
+	maps.Copy(allRedactedValues, redactedEnvVars)
+
 	return apiclient.Job{
 		ID:             index.ID,
 		Commit:         index.Commit,
 		RepositoryName: index.RepositoryName,
 		ShallowClone:   true,
 		FetchTags:      fetchTags,
 		DockerSteps:    dockerSteps,
-		RedactedValues: map[string]string{
-			// 🚨 SECURITY: Catch leak of authorization header.
-			authorizationHeader: redactedAuthorizationHeader,
-
-			// 🚨 SECURITY: Catch uses of fragments pulled from auth header to
-			// construct another target (in src-cli). We only pass the
-			// Authorization header to src-cli, which we trust not to ship the
-			// values to a third party, but not to trust to ensure the values
-			// are absent from the command's stdout or stderr streams.
-			accessToken: "PASSWORD_REMOVED",
-		},
+		RedactedValues: allRedactedValues,
 	}, nil
 }
 
-const defaultMemory = "12G"
-const defaultDiskSpace = "20G"
+const (
+	defaultMemory    = "12G"
+	defaultDiskSpace = "20G"
+)
 
 func makeResourceEnvironment(resourceMetadata handler.ResourceMetadata) []string {
 	env := []string{}