[server] abort running prebuilds on same branch

svenefftinge · roboquat · commit ba78bd4d0852 · 2022-06-30T11:40:38.000+05:30
diff --git a/components/dashboard/src/projects/Prebuilds.tsx b/components/dashboard/src/projects/Prebuilds.tsx
@@ -221,7 +221,10 @@ export default function (props: { project?: Project; isAdminDashboard?: boolean
                                         }`}
                                     >
                                         <div>
-                                            <div className="text-base text-gray-900 dark:text-gray-50 font-medium uppercase mb-1">
+                                            <div
+                                                className="text-base text-gray-900 dark:text-gray-50 font-medium uppercase mb-1"
+                                                title={getPrebuildStatusDescription(p)}
+                                            >
                                                 <div className="inline-block align-text-bottom mr-2 w-4 h-4">
                                                     {prebuildStatusIcon(p)}
                                                 </div>
@@ -329,38 +332,25 @@ export function prebuildStatusIcon(prebuild?: PrebuildWithStatus) {
     }
 }
 
-function PrebuildStatusDescription(props: { prebuild: PrebuildWithStatus }) {
-    switch (props.prebuild.status) {
+function getPrebuildStatusDescription(prebuild: PrebuildWithStatus): string {
+    switch (prebuild.status) {
         case "queued":
-            return <span>Prebuild is queued and will be processed when there is execution capacity.</span>;
+            return `Prebuild is queued and will be processed when there is execution capacity.`;
         case "building":
-            return <span>Prebuild is currently in progress.</span>;
+            return `Prebuild is currently in progress.`;
         case "aborted":
-            return (
-                <span>
-                    Prebuild has been cancelled. Either a user cancelled it, or the prebuild rate limit has been
-                    exceeded. {props.prebuild.error}
-                </span>
-            );
+            return `Prebuild has been cancelled. Either a newer commit was pushed to the same branch, a user cancelled it manually, or the prebuild rate limit has been exceeded.`;
         case "failed":
-            return <span>Prebuild failed for system reasons. Please contact support. {props.prebuild.error}</span>;
+            return `Prebuild failed for system reasons. Please contact support. ${prebuild.error}`;
         case "timeout":
-            return (
-                <span>
-                    Prebuild timed out. Either the image, or the prebuild tasks took too long. {props.prebuild.error}
-                </span>
-            );
+            return `Prebuild timed out. Either the image, or the prebuild tasks took too long. ${prebuild.error}`;
         case "available":
-            if (props.prebuild?.error) {
-                return (
-                    <span>
-                        The tasks executed in the prebuild returned a non-zero exit code. {props.prebuild.error}
-                    </span>
-                );
+            if (prebuild.error) {
+                return `The tasks executed in the prebuild returned a non-zero exit code. ${prebuild.error}`;
             }
-            return <span>Prebuild completed successfully.</span>;
+            return `Prebuild completed successfully.`;
         default:
-            return <span>Unknown prebuild status.</span>;
+            return `Unknown prebuild status.`;
     }
 }
 
@@ -376,7 +366,7 @@ export function PrebuildStatus(props: { prebuild: PrebuildWithStatus }) {
                 </div>
             </div>
             <div className="flex space-x-1 items-center text-gray-400">
-                <PrebuildStatusDescription prebuild={prebuild} />
+                <span>{getPrebuildStatusDescription(prebuild)}</span>
             </div>
         </div>
     );
diff --git a/components/dashboard/src/projects/ProjectSettings.tsx b/components/dashboard/src/projects/ProjectSettings.tsx
@@ -14,7 +14,6 @@ import { PageWithSubMenu } from "../components/PageWithSubMenu";
 import PillLabel from "../components/PillLabel";
 import { ProjectContext } from "./project-context";
 import { getExperimentsClient } from "./../experiments/client";
-import { DeepPartial } from "@gitpod/gitpod-protocol/lib/util/deep-partial";
 
 export function getProjectSettingsMenu(project?: Project, team?: Team) {
     const teamOrUserSlug = !!team ? "t/" + team.slug : "projects";
@@ -56,19 +55,14 @@ export default function () {
     const { teams } = useContext(TeamsContext);
     const team = getCurrentTeam(location, teams);
 
-    const [isLoading, setIsLoading] = useState<boolean>(true);
-    const [isIncrementalPrebuildsEnabled, setIsIncrementalPrebuildsEnabled] = useState<boolean>(false);
-    const [isPersistentVolumeClaimEnabled, setIsPersistentVolumeClaimEnabled] = useState<boolean>(false);
     const [isShowPersistentVolumeClaim, setIsShowPersistentVolumeClaim] = useState<boolean>(false);
+    const [projectSettings, setProjectSettings] = useState<ProjectSettings>({});
 
     useEffect(() => {
         if (!project) {
             return;
         }
-        setIsLoading(false);
-        setIsIncrementalPrebuildsEnabled(!!project.settings?.useIncrementalPrebuilds);
-        setIsPersistentVolumeClaimEnabled(!!project.settings?.usePersistentVolumeClaim);
-
+        setProjectSettings({ ...project.settings });
         (async () => {
             const showPersistentVolumeClaim = await getExperimentsClient().getValueAsync(
                 "persistent_volume_claim",
@@ -84,49 +78,34 @@ export default function () {
         })();
     }, [project, team, teams]);
 
-    const updateProjectPartial = (settings: DeepPartial<ProjectSettings | undefined>) => {
+    const updateProjectSettings = () => {
         if (!project) {
             return;
         }
-
-        return getGitpodService().server.updateProjectPartial({ id: project.id, settings });
+        setProjectSettings({
+            ...projectSettings,
+        });
+        return getGitpodService().server.updateProjectPartial({ id: project.id, settings: projectSettings });
     };
 
     const toggleIncrementalPrebuilds = async () => {
-        if (!project) {
-            return;
-        }
-        setIsLoading(true);
-        try {
-            await updateProjectPartial({
-                useIncrementalPrebuilds: !isIncrementalPrebuildsEnabled,
-                usePersistentVolumeClaim: isPersistentVolumeClaimEnabled,
-            });
-            setIsIncrementalPrebuildsEnabled(!isIncrementalPrebuildsEnabled);
-        } finally {
-            setIsLoading(false);
-        }
+        projectSettings.useIncrementalPrebuilds = !projectSettings.useIncrementalPrebuilds;
+        updateProjectSettings();
+    };
+
+    const toggleCancelOutdatedPrebuilds = async () => {
+        projectSettings.keepOutdatedPrebuildsRunning = !projectSettings.keepOutdatedPrebuildsRunning;
+        updateProjectSettings();
     };
 
     const togglePersistentVolumeClaim = async () => {
-        if (!project) {
-            return;
-        }
-        setIsLoading(true);
-        try {
-            await updateProjectPartial({
-                useIncrementalPrebuilds: isIncrementalPrebuildsEnabled,
-                usePersistentVolumeClaim: !isPersistentVolumeClaimEnabled,
-            });
-            setIsPersistentVolumeClaimEnabled(!isPersistentVolumeClaimEnabled);
-        } finally {
-            setIsLoading(false);
-        }
+        projectSettings.usePersistentVolumeClaim = !projectSettings.usePersistentVolumeClaim;
+        updateProjectSettings();
     };
 
     return (
         <ProjectSettingsPage project={project}>
-            <h3>Incremental Prebuilds</h3>
+            <h3>Prebuilds</h3>
             <CheckBox
                 title={
                     <span>
@@ -146,10 +125,15 @@ export default function () {
                         </a>
                     </span>
                 }
-                checked={isIncrementalPrebuildsEnabled}
-                disabled={isLoading}
+                checked={!!projectSettings.useIncrementalPrebuilds}
                 onChange={toggleIncrementalPrebuilds}
             />
+            <CheckBox
+                title={<span>Cancel Prebuilds on Outdated Commits </span>}
+                desc={<span>Cancels all pending and running prebuilds on the branch when a new commit is pushed.</span>}
+                checked={!projectSettings.keepOutdatedPrebuildsRunning}
+                onChange={toggleCancelOutdatedPrebuilds}
+            />
             <br></br>
             <h3>Persistent Volume Claim</h3>
             <CheckBox
@@ -162,8 +146,8 @@ export default function () {
                     </span>
                 }
                 desc={<span>Experimental feature that is still under development.</span>}
-                checked={isPersistentVolumeClaimEnabled}
-                disabled={isLoading || !isShowPersistentVolumeClaim}
+                checked={!!projectSettings.usePersistentVolumeClaim}
+                disabled={!isShowPersistentVolumeClaim}
                 onChange={togglePersistentVolumeClaim}
             />
         </ProjectSettingsPage>
diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts
@@ -18,6 +18,7 @@ import {
     WorkspaceAndOwner,
     WorkspacePortsAuthData,
     WorkspaceOwnerAndSoftDeleted,
+    PrebuildWithWorkspaceAndInstances,
 } from "../workspace-db";
 import {
     Workspace,
@@ -816,6 +817,38 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
             .getOne();
     }
 
+    public async findActivePrebuiltWorkspacesByBranch(
+        projectId: string,
+        branch: string,
+    ): Promise<PrebuildWithWorkspaceAndInstances[]> {
+        if (!branch) {
+            return [];
+        }
+        const repo = await this.getPrebuiltWorkspaceRepo();
+        const result = await repo
+            .createQueryBuilder("pws")
+            .where(
+                "(pws.state = 'queued' OR pws.state = 'building') AND pws.projectId = :projectId AND pws.branch = :branch",
+                { projectId, branch },
+            )
+            .orderBy("pws.creationTime", "DESC")
+            .innerJoinAndMapOne(
+                "pws.workspace",
+                DBWorkspace,
+                "ws",
+                "pws.buildWorkspaceId = ws.id and ws.contentDeletedTime = ''",
+            )
+            .innerJoinAndMapMany("pws.instances", DBWorkspaceInstance, "wsi", "pws.buildWorkspaceId = wsi.workspaceId")
+            .getMany();
+        return result.map((r) => {
+            return {
+                prebuild: r,
+                workspace: (<any>r).workspace,
+                instances: (<any>r).instances,
+            };
+        });
+    }
+
     public async findPrebuildByWorkspaceID(wsid: string): Promise<PrebuiltWorkspace | undefined> {
         const repo = await this.getPrebuiltWorkspaceRepo();
         return await repo.createQueryBuilder("pws").where("pws.buildWorkspaceId = :wsid", { wsid }).getOne();
diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts
@@ -63,6 +63,12 @@ export interface PrebuildWithWorkspace {
     workspace: Workspace;
 }
 
+export interface PrebuildWithWorkspaceAndInstances {
+    prebuild: PrebuiltWorkspace;
+    workspace: Workspace;
+    instances: WorkspaceInstance[];
+}
+
 export type WorkspaceAndOwner = Pick<Workspace, "id" | "ownerId">;
 export type WorkspaceOwnerAndSoftDeleted = Pick<Workspace, "id" | "ownerId" | "softDeleted">;
 
@@ -170,6 +176,10 @@ export interface WorkspaceDB {
 
     storePrebuiltWorkspace(pws: PrebuiltWorkspace): Promise<PrebuiltWorkspace>;
     findPrebuiltWorkspaceByCommit(cloneURL: string, commit: string): Promise<PrebuiltWorkspace | undefined>;
+    findActivePrebuiltWorkspacesByBranch(
+        projectId: string,
+        branch: string,
+    ): Promise<PrebuildWithWorkspaceAndInstances[]>;
     findPrebuildsWithWorkpace(cloneURL: string): Promise<PrebuildWithWorkspace[]>;
     findPrebuildByWorkspaceID(wsid: string): Promise<PrebuiltWorkspace | undefined>;
     findPrebuildByID(pwsid: string): Promise<PrebuiltWorkspace | undefined>;
diff --git a/components/gitpod-protocol/src/teams-projects-protocol.ts b/components/gitpod-protocol/src/teams-projects-protocol.ts
@@ -15,6 +15,7 @@ export interface ProjectConfig {
 export interface ProjectSettings {
     useIncrementalPrebuilds?: boolean;
     usePersistentVolumeClaim?: boolean;
+    keepOutdatedPrebuildsRunning?: boolean;
 }
 
 export interface Project {
diff --git a/components/gitpod-protocol/src/util/logging.ts b/components/gitpod-protocol/src/util/logging.ts
@@ -6,6 +6,7 @@
 
 const inspect: (object: any) => string = require("util").inspect; // undefined in frontend
 
+let plainLogging: boolean = false; // set to true during development to get non JSON output
 let jsonLogging: boolean = false;
 let component: string | undefined;
 let version: string | undefined;
@@ -309,7 +310,7 @@ function makeLogItem(
     }
 
     const payload: any = payloadArgs.length == 0 ? undefined : payloadArgs.length == 1 ? payloadArgs[0] : payloadArgs;
-    const logItem: any = {
+    const logItem = {
         // undefined fields get eliminated in JSON.stringify()
         ...reportedErrorEvent,
         component,
@@ -321,11 +322,19 @@ function makeLogItem(
         payload,
         loggedViaConsole: calledViaConsole ? true : undefined,
     };
+    if (plainLogging) {
+        return `[${logItem.severity}] [${logItem.component}] ${logItem.message}
+            ${JSON.stringify(payload || "", undefined, "              ")}
+            ${error || ""}
+        `.trim();
+    }
     let result: string = stringifyLogItem(logItem);
 
     if (result.length > maxAllowedLogItemLength && payload !== undefined) {
         delete logItem.payload;
-        logItem.payloadStub = `Payload stripped as log item was longer than ${maxAllowedLogItemLength} characters`;
+        (<any>(
+            logItem
+        )).payloadStub = `Payload stripped as log item was longer than ${maxAllowedLogItemLength} characters`;
 
         result = stringifyLogItem(logItem);
 
diff --git a/components/server/ee/src/prebuilds/gitlab-app.ts b/components/server/ee/src/prebuilds/gitlab-app.ts
@@ -155,6 +155,8 @@ export class GitLabApp {
             );
 
             return ws;
+        } catch (e) {
+            log.error("error processing GitLab webhook", e, body);
         } finally {
             span.finish();
         }
diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts
@@ -58,6 +58,22 @@ export class PrebuildManager {
     @inject(Config) protected readonly config: Config;
     @inject(ProjectsService) protected readonly projectService: ProjectsService;
 
+    async abortPrebuildsForBranch(ctx: TraceContext, project: Project, user: User, branch: string): Promise<void> {
+        const span = TraceContext.startSpan("abortPrebuildsForBranch", ctx);
+        const prebuilds = await this.workspaceDB
+            .trace({ span })
+            .findActivePrebuiltWorkspacesByBranch(project.id, branch);
+        const results: Promise<any>[] = [];
+        for (const prebuild of prebuilds) {
+            for (const instance of prebuild.instances) {
+                results.push(this.workspaceStarter.stopWorkspaceInstance({ span }, instance.id, instance.region));
+            }
+            prebuild.prebuild.state = "aborted";
+            results.push(this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild.prebuild));
+        }
+        await Promise.all(results);
+    }
+
     async startPrebuild(
         ctx: TraceContext,
         { context, project, user, commitInfo }: StartPrebuildParams,
@@ -108,6 +124,13 @@ export class PrebuildManager {
                     return { prebuildId: existingPB.id, wsid: existingPB.buildWorkspaceId, done: true };
                 }
             }
+            if (project && context.ref && !project.settings?.keepOutdatedPrebuildsRunning) {
+                try {
+                    await this.abortPrebuildsForBranch({ span }, project, user, context.ref);
+                } catch (e) {
+                    console.error("Error aborting prebuilds", e);
+                }
+            }
 
             const prebuildContext: StartPrebuildContext = {
                 title: `Prebuild of "${context.title}"`,
diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts
@@ -591,7 +591,12 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
         (await Promise.all(workspaces.map((workspace) => workspaceDb.findRunningInstance(workspace.id))))
             .filter(isDefined)
             .forEach((instance) =>
-                this.internalStopWorkspaceInstance(ctx, instance.id, instance.region, StopWorkspacePolicy.IMMEDIATELY),
+                this.workspaceStarter.stopWorkspaceInstance(
+                    ctx,
+                    instance.id,
+                    instance.region,
+                    StopWorkspacePolicy.IMMEDIATELY,
+                ),
             );
 
         // For some reason, returning the result of `this.userDB.storeUser(target)` does not work. The response never arrives the caller.
diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts
diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ export interface ProjectConfig {`
`15`	`15`	`export interface ProjectSettings {`
`16`	`16`	`useIncrementalPrebuilds?: boolean;`
`17`	`17`	`usePersistentVolumeClaim?: boolean;`
	`18`	`+ keepOutdatedPrebuildsRunning?: boolean;`
`18`	`19`	`}`
`19`	`20`
`20`	`21`	`export interface Project {`
Original file line number	Diff line number	Diff line change
`@@ -155,6 +155,8 @@ export class GitLabApp {`
`155`	`155`	`);`
`156`	`156`
`157`	`157`	`return ws;`
	`158`	`+ } catch (e) {`
	`159`	`+ log.error("error processing GitLab webhook", e, body);`
`158`	`160`	`} finally {`
`159`	`161`	`span.finish();`
`160`	`162`	`}`