From 264b32141a0b89eb106ac9b62997d55bb730e792 Mon Sep 17 00:00:00 2001 From: Sven Efftinge Date: Fri, 13 May 2022 05:03:36 +0000 Subject: [PATCH 1/3] [prebuilds] no prebuilds for inactive repos --- .../1652365883273-CloneUrlIndexed.ts | 33 +++++++++++++ .../src/typeorm/workspace-db-impl.ts | 16 +++++++ .../gitpod-db/src/workspace-db.spec.db.ts | 48 ++++++++++++++++++- components/gitpod-db/src/workspace-db.ts | 1 + .../ee/src/prebuilds/prebuild-manager.ts | 19 ++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts diff --git a/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts b/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts new file mode 100644 index 00000000000000..bde79d04477ae5 --- /dev/null +++ b/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the Gitpod Enterprise Source Code License, + * See License.enterprise.txt in the project root folder. + */ + +import { MigrationInterface, QueryRunner } from "typeorm"; +import { columnExists, indexExists } from "./helper/helper"; + +export class CloneUrlIndexed1652365883273 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const TABLE_NAME = "d_b_workspace"; + const COLUMN_NAME = "cloneURL"; + const TYPE_INDEX_NAME = "d_b_workspace_cloneURL_idx"; + + if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) { + await queryRunner.query( + `ALTER TABLE + ${TABLE_NAME} + ADD COLUMN + ${COLUMN_NAME} VARCHAR(256) + GENERATED ALWAYS AS ( + context ->> "$.repository.cloneUrl" + )`, + ); + } + if (!(await indexExists(queryRunner, TABLE_NAME, TYPE_INDEX_NAME))) { + await queryRunner.query(`CREATE INDEX ${TYPE_INDEX_NAME} ON ${TABLE_NAME} (${COLUMN_NAME})`); + } + } + + public async down(queryRunner: QueryRunner): Promise {} +} diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index e37a12849ecd15..661756c97726da 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -356,6 +356,22 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { return workspaceRepo.find({ ownerId: userId }); } + public async getWorkspaceCountByCloneURL( + cloneURL: string, + sinceLastDays: number = 7, + type: string = "regular", + ): Promise { + const workspaceRepo = await this.getWorkspaceRepo(); + const since = new Date(); + since.setDate(since.getDate() - sinceLastDays); + return workspaceRepo + .createQueryBuilder("ws") + .where("cloneURL = :cloneURL", { cloneURL }) + .andWhere("creationTime > :since", { since: since.toISOString() }) + .andWhere("type = :type", { type }) + .getCount(); + } + public async findCurrentInstance(workspaceId: string): Promise { const workspaceInstanceRepo = await this.getWorkspaceInstanceRepo(); const qb = workspaceInstanceRepo diff --git a/components/gitpod-db/src/workspace-db.spec.db.ts b/components/gitpod-db/src/workspace-db.spec.db.ts index 5cb2dec8d02e05..cf49bd6fc375e3 100644 --- a/components/gitpod-db/src/workspace-db.spec.db.ts +++ b/components/gitpod-db/src/workspace-db.spec.db.ts @@ -9,7 +9,7 @@ const expect = chai.expect; import { suite, test, timeout } from "mocha-typescript"; import { fail } from "assert"; -import { WorkspaceInstance, Workspace, PrebuiltWorkspace } from "@gitpod/gitpod-protocol"; +import { WorkspaceInstance, Workspace, PrebuiltWorkspace, CommitContext } from "@gitpod/gitpod-protocol"; import { testContainer } from "./test-container"; import { TypeORMWorkspaceDBImpl } from "./typeorm/workspace-db-impl"; import { TypeORM } from "./typeorm/typeorm"; @@ -539,6 +539,52 @@ class WorkspaceDBSpec { expect(unabortedCount).to.eq(1); } + @test(timeout(10000)) + public async testGetWorkspaceCountForCloneURL() { + const now = new Date(); + const eightDaysAgo = new Date(); + eightDaysAgo.setDate(eightDaysAgo.getDate() - 8); + const activeRepo = "http://github.com/myorg/active.git"; + const inactiveRepo = "http://github.com/myorg/inactive.git"; + await Promise.all([ + this.db.store({ + id: "12345", + creationTime: eightDaysAgo.toISOString(), + description: "something", + contextURL: "http://github.com/myorg/inactive", + ownerId: "1221423", + context: { + title: "my title", + repository: { + cloneUrl: inactiveRepo, + }, + }, + config: {}, + type: "regular", + }), + this.db.store({ + id: "12346", + creationTime: now.toISOString(), + description: "something", + contextURL: "http://github.com/myorg/active", + ownerId: "1221423", + context: { + title: "my title", + repository: { + cloneUrl: activeRepo, + }, + }, + config: {}, + type: "regular", + }), + ]); + + const inactiveCount = await this.db.getWorkspaceCountByCloneURL(inactiveRepo, 7, "regular"); + expect(inactiveCount).to.eq(0, "there should be no regular workspaces in the past 7 days"); + const activeCount = await this.db.getWorkspaceCountByCloneURL(activeRepo, 7, "regular"); + expect(activeCount).to.eq(1, "there should be exactly one regular workspace"); + } + private async storePrebuiltWorkspace(pws: PrebuiltWorkspace) { // store the creationTime directly, before it is modified by the store function in the ORM layer const creationTime = pws.creationTime; diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts index 7eae4b91935d29..d7838973b61158 100644 --- a/components/gitpod-db/src/workspace-db.ts +++ b/components/gitpod-db/src/workspace-db.ts @@ -125,6 +125,7 @@ export interface WorkspaceDB { findInstancesByPhaseAndRegion(phase: string, region: string): Promise; getWorkspaceCount(type?: String): Promise; + getWorkspaceCountByCloneURL(cloneURL: string, sinceLastDays?: number, type?: string): Promise; getInstanceCount(type?: string): Promise; findAllWorkspaceInstances( diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index 3575a1b348f6dc..c52b5b135dfbfe 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -195,6 +195,11 @@ export class PrebuildManager { prebuild.error = "Project is inactive. Please start a new workspace for this project to re-enable prebuilds."; await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); + } else if (!project && (await this.shouldSkipInactiveRepository({ span }, cloneURL))) { + prebuild.state = "aborted"; + prebuild.error = + "Repository is inactive. Please create a project for this repository to re-enable prebuilds."; + await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); } else { span.setTag("starting", true); const projectEnvVars = await projectEnvVarsPromise; @@ -356,4 +361,18 @@ export class PrebuildManager { const inactiveProjectTime = 1000 * 60 * 60 * 24 * 7 * 1; // 1 week return now - lastUse > inactiveProjectTime; } + + private async shouldSkipInactiveRepository(ctx: TraceContext, cloneURL: string): Promise { + const span = TraceContext.startSpan("shouldSkipInactiveRepository", ctx); + try { + return ( + (await this.workspaceDB + .trace({ span }) + .getWorkspaceCountByCloneURL(cloneURL, 7 /* last week */, "regular")) === 0 + ); + } catch (error) { + log.error("cannot compute activity for repository", { cloneURL }, error); + return false; + } + } } From b828ed37937c6936fa2dfd99342d76693d7204de Mon Sep 17 00:00:00 2001 From: Alex Tugarev Date: Wed, 18 May 2022 13:20:05 +0000 Subject: [PATCH 2/3] Managed cloneURL column. --- .../src/typeorm/entity/db-workspace.ts | 6 ++++++ .../migration/1652365883273-CloneUrlIndexed.ts | 9 ++------- .../gitpod-db/src/typeorm/workspace-db-impl.ts | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/components/gitpod-db/src/typeorm/entity/db-workspace.ts b/components/gitpod-db/src/typeorm/entity/db-workspace.ts index f9bcd3fa0058d7..9b5f03d1fad291 100644 --- a/components/gitpod-db/src/typeorm/entity/db-workspace.ts +++ b/components/gitpod-db/src/typeorm/entity/db-workspace.ts @@ -50,6 +50,12 @@ export class DBWorkspace implements Workspace { @Column("simple-json") context: WorkspaceContext; + @Column({ + default: "", + transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED, + }) + cloneUrl?: string; + @Column("simple-json") config: WorkspaceConfig; diff --git a/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts b/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts index bde79d04477ae5..d51edf76c34bbd 100644 --- a/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts +++ b/components/gitpod-db/src/typeorm/migration/1652365883273-CloneUrlIndexed.ts @@ -15,13 +15,8 @@ export class CloneUrlIndexed1652365883273 implements MigrationInterface { if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) { await queryRunner.query( - `ALTER TABLE - ${TABLE_NAME} - ADD COLUMN - ${COLUMN_NAME} VARCHAR(256) - GENERATED ALWAYS AS ( - context ->> "$.repository.cloneUrl" - )`, + `ALTER TABLE ${TABLE_NAME} + ADD COLUMN ${COLUMN_NAME} VARCHAR(255) NOT NULL DEFAULT ''`, ); } if (!(await indexExists(queryRunner, TABLE_NAME, TYPE_INDEX_NAME))) { diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index 661756c97726da..d4d03096e9aa9b 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -4,6 +4,7 @@ * See License-AGPL.txt in the project root for license information. */ +import * as crypto from "crypto"; import { injectable, inject } from "inversify"; import { Repository, EntityManager, DeepPartial, UpdateQueryBuilder, Brackets } from "typeorm"; import { @@ -138,8 +139,22 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { public async store(workspace: Workspace) { const workspaceRepo = await this.getWorkspaceRepo(); const dbWorkspace = workspace as DBWorkspace; + + // `cloneUrl` is stored redundandly to optimize for `getWorkspaceCountByCloneURL`. + // As clone URLs are lesser constrained we want to shorten the value to work well with the indexed column. + let cloneUrl: string = this.toCloneUrl255((workspace as any).context?.repository?.cloneUrl || ""); + + dbWorkspace.cloneUrl = cloneUrl; return await workspaceRepo.save(dbWorkspace); } + + protected toCloneUrl255(cloneUrl: string) { + if (cloneUrl.length > 255) { + return `cloneUrl-sha:${crypto.createHash("sha256").update(cloneUrl, "utf8").digest("hex")}`; + } + return cloneUrl; + } + public async updatePartial(workspaceId: string, partial: DeepPartial) { const workspaceRepo = await this.getWorkspaceRepo(); await workspaceRepo.update(workspaceId, partial); @@ -366,7 +381,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { since.setDate(since.getDate() - sinceLastDays); return workspaceRepo .createQueryBuilder("ws") - .where("cloneURL = :cloneURL", { cloneURL }) + .where("cloneURL = :cloneURL", { cloneURL: this.toCloneUrl255(cloneURL) }) .andWhere("creationTime > :since", { since: since.toISOString() }) .andWhere("type = :type", { type }) .getCount(); From ce6b91822b0864e67d3c9de71aae903d6161253c Mon Sep 17 00:00:00 2001 From: Alex Tugarev Date: Wed, 18 May 2022 14:18:56 +0000 Subject: [PATCH 3/3] Adding Config.inactivityPeriodForRepos --- .../server/ee/src/prebuilds/prebuild-manager.ts | 7 ++++++- components/server/src/config.ts | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index c52b5b135dfbfe..38307e10bab482 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -364,11 +364,16 @@ export class PrebuildManager { private async shouldSkipInactiveRepository(ctx: TraceContext, cloneURL: string): Promise { const span = TraceContext.startSpan("shouldSkipInactiveRepository", ctx); + const { inactivityPeriodForRepos } = this.config; + if (!inactivityPeriodForRepos) { + // skipping is disabled if `inactivityPeriodForRepos` is not set + return false; + } try { return ( (await this.workspaceDB .trace({ span }) - .getWorkspaceCountByCloneURL(cloneURL, 7 /* last week */, "regular")) === 0 + .getWorkspaceCountByCloneURL(cloneURL, inactivityPeriodForRepos /* in days */, "regular")) === 0 ); } catch (error) { log.error("cannot compute activity for repository", { cloneURL }, error); diff --git a/components/server/src/config.ts b/components/server/src/config.ts index 4c9c1d3543b710..41fa4bce220a06 100644 --- a/components/server/src/config.ts +++ b/components/server/src/config.ts @@ -27,6 +27,7 @@ export type Config = Omit< chargebeeProviderOptions?: ChargebeeProviderOptions; builtinAuthProvidersConfigured: boolean; blockedRepositories: { urlRegExp: RegExp; blockUser: boolean }[]; + inactivityPeriodForRepos?: number; }; export interface WorkspaceDefaults { @@ -162,6 +163,12 @@ export interface ConfigSerialized { * `blockUser` attribute to control handling of the user's account. */ blockedRepositories?: { urlRegExp: string; blockUser: boolean }[]; + + /** + * If a numeric value interpreted as days is set, repositories not beeing opened with Gitpod are + * considered inactive. + */ + inactivityPeriodForRepos?: number; } export namespace ConfigFile { @@ -220,6 +227,12 @@ export namespace ConfigFile { }); } } + let inactivityPeriodForRepos: number | undefined; + if (typeof config.inactivityPeriodForRepos === "number") { + if (config.inactivityPeriodForRepos >= 1) { + inactivityPeriodForRepos = config.inactivityPeriodForRepos; + } + } return { ...config, hostUrl, @@ -234,6 +247,7 @@ export namespace ConfigFile { : Date.now(), }, blockedRepositories, + inactivityPeriodForRepos, }; } }