diff --git a/components/content-service-api/go/blobs_grpc.pb.go b/components/content-service-api/go/blobs_grpc.pb.go index e56da060d6fc28..e9fec073a75a26 100644 --- a/components/content-service-api/go/blobs_grpc.pb.go +++ b/components/content-service-api/go/blobs_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. diff --git a/components/content-service-api/go/content_grpc.pb.go b/components/content-service-api/go/content_grpc.pb.go index 739d561ced67d0..9b377b16fe061b 100644 --- a/components/content-service-api/go/content_grpc.pb.go +++ b/components/content-service-api/go/content_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. diff --git a/components/content-service-api/go/headless-log_grpc.pb.go b/components/content-service-api/go/headless-log_grpc.pb.go index 3c889eb771fc82..70e0bd63989267 100644 --- a/components/content-service-api/go/headless-log_grpc.pb.go +++ b/components/content-service-api/go/headless-log_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. diff --git a/components/content-service-api/go/ideplugin_grpc.pb.go b/components/content-service-api/go/ideplugin_grpc.pb.go index 762a361b55f551..6076be955215fe 100644 --- a/components/content-service-api/go/ideplugin_grpc.pb.go +++ b/components/content-service-api/go/ideplugin_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. diff --git a/components/content-service-api/go/initializer.pb.go b/components/content-service-api/go/initializer.pb.go index f2af243286177a..61154d087c6d96 100644 --- a/components/content-service-api/go/initializer.pb.go +++ b/components/content-service-api/go/initializer.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. @@ -656,14 +656,14 @@ func (x *SnapshotInitializer) GetSnapshot() string { } // A prebuild initializer combines snapshots with Git: first we try the snapshot, then apply the Git clone target. -// If restoring the snapshot fails, we fall back to a regular Git initializer. +// If restoring the snapshot fails, we fall back to a regular Git initializer, which might be composite git initializer for multi-repo projects. type PrebuildInitializer struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Prebuild *SnapshotInitializer `protobuf:"bytes,1,opt,name=prebuild,proto3" json:"prebuild,omitempty"` - Git *GitInitializer `protobuf:"bytes,2,opt,name=git,proto3" json:"git,omitempty"` + Git []*GitInitializer `protobuf:"bytes,2,rep,name=git,proto3" json:"git,omitempty"` } func (x *PrebuildInitializer) Reset() { @@ -705,7 +705,7 @@ func (x *PrebuildInitializer) GetPrebuild() *SnapshotInitializer { return nil } -func (x *PrebuildInitializer) GetGit() *GitInitializer { +func (x *PrebuildInitializer) GetGit() []*GitInitializer { if x != nil { return x.Git } @@ -1037,7 +1037,7 @@ var file_initializer_proto_rawDesc = []byte{ 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x12, 0x30, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x69, 0x74, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x52, 0x03, 0x67, 0x69, 0x74, 0x22, 0x17, 0x0a, 0x15, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, diff --git a/components/content-service-api/go/workspace_grpc.pb.go b/components/content-service-api/go/workspace_grpc.pb.go index e7180769a6dcfe..932ebebbdf8c68 100644 --- a/components/content-service-api/go/workspace_grpc.pb.go +++ b/components/content-service-api/go/workspace_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). // See License-AGPL.txt in the project root for license information. diff --git a/components/content-service-api/initializer.proto b/components/content-service-api/initializer.proto index 2ada5a7c18b622..1376deffde9564 100644 --- a/components/content-service-api/initializer.proto +++ b/components/content-service-api/initializer.proto @@ -115,10 +115,10 @@ message SnapshotInitializer { } // A prebuild initializer combines snapshots with Git: first we try the snapshot, then apply the Git clone target. -// If restoring the snapshot fails, we fall back to a regular Git initializer. +// If restoring the snapshot fails, we fall back to a regular Git initializer, which might be composite git initializer for multi-repo projects. message PrebuildInitializer { SnapshotInitializer prebuild = 1; - GitInitializer git = 2; + repeated GitInitializer git = 2; } // FromBackupInitializer initializes content from a previously made backup diff --git a/components/content-service-api/typescript/src/blobs_grpc_pb.d.ts b/components/content-service-api/typescript/src/blobs_grpc_pb.d.ts index 6b711a86506dd0..0da328a1bdabf9 100644 --- a/components/content-service-api/typescript/src/blobs_grpc_pb.d.ts +++ b/components/content-service-api/typescript/src/blobs_grpc_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/blobs_pb.d.ts b/components/content-service-api/typescript/src/blobs_pb.d.ts index 787734b927cc2f..a6aea0e847ac4e 100644 --- a/components/content-service-api/typescript/src/blobs_pb.d.ts +++ b/components/content-service-api/typescript/src/blobs_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/blobs_pb.js b/components/content-service-api/typescript/src/blobs_pb.js index 720f76c514c5c2..d048c19732521d 100644 --- a/components/content-service-api/typescript/src/blobs_pb.js +++ b/components/content-service-api/typescript/src/blobs_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/content_grpc_pb.d.ts b/components/content-service-api/typescript/src/content_grpc_pb.d.ts index 1969316f61ea00..695c36e8eca872 100644 --- a/components/content-service-api/typescript/src/content_grpc_pb.d.ts +++ b/components/content-service-api/typescript/src/content_grpc_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/content_pb.d.ts b/components/content-service-api/typescript/src/content_pb.d.ts index 4fb0c28486af03..3d5967a8cc1037 100644 --- a/components/content-service-api/typescript/src/content_pb.d.ts +++ b/components/content-service-api/typescript/src/content_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/content_pb.js b/components/content-service-api/typescript/src/content_pb.js index 7cb048c5342202..8c78bdb88690b5 100644 --- a/components/content-service-api/typescript/src/content_pb.js +++ b/components/content-service-api/typescript/src/content_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts b/components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts index ff7be715953433..d9a11d7f692274 100644 --- a/components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts +++ b/components/content-service-api/typescript/src/headless-log_grpc_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/headless-log_pb.d.ts b/components/content-service-api/typescript/src/headless-log_pb.d.ts index d74fe3dc71fe48..2bf2ee172c14fb 100644 --- a/components/content-service-api/typescript/src/headless-log_pb.d.ts +++ b/components/content-service-api/typescript/src/headless-log_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/headless-log_pb.js b/components/content-service-api/typescript/src/headless-log_pb.js index d4d53ba298b1c3..ceb2c35b55675f 100644 --- a/components/content-service-api/typescript/src/headless-log_pb.js +++ b/components/content-service-api/typescript/src/headless-log_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts b/components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts index 085b50ca0dc472..0bcdea4b762601 100644 --- a/components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts +++ b/components/content-service-api/typescript/src/ideplugin_grpc_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/ideplugin_pb.d.ts b/components/content-service-api/typescript/src/ideplugin_pb.d.ts index 69c389d868ca99..2a173551d74e18 100644 --- a/components/content-service-api/typescript/src/ideplugin_pb.d.ts +++ b/components/content-service-api/typescript/src/ideplugin_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/ideplugin_pb.js b/components/content-service-api/typescript/src/ideplugin_pb.js index c8a322bba6382c..a83597f702919d 100644 --- a/components/content-service-api/typescript/src/ideplugin_pb.js +++ b/components/content-service-api/typescript/src/ideplugin_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/initializer_grpc_pb.js b/components/content-service-api/typescript/src/initializer_grpc_pb.js index e15fa3a43f1773..c72bac715b8ab1 100644 --- a/components/content-service-api/typescript/src/initializer_grpc_pb.js +++ b/components/content-service-api/typescript/src/initializer_grpc_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/initializer_pb.d.ts b/components/content-service-api/typescript/src/initializer_pb.d.ts index 9d8c6df4fa075f..ba604c52eb352d 100644 --- a/components/content-service-api/typescript/src/initializer_pb.d.ts +++ b/components/content-service-api/typescript/src/initializer_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ @@ -275,11 +275,10 @@ export class PrebuildInitializer extends jspb.Message { clearPrebuild(): void; getPrebuild(): SnapshotInitializer | undefined; setPrebuild(value?: SnapshotInitializer): PrebuildInitializer; - - hasGit(): boolean; - clearGit(): void; - getGit(): GitInitializer | undefined; - setGit(value?: GitInitializer): PrebuildInitializer; + clearGitList(): void; + getGitList(): Array; + setGitList(value: Array): PrebuildInitializer; + addGit(value?: GitInitializer, index?: number): GitInitializer; serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PrebuildInitializer.AsObject; @@ -294,7 +293,7 @@ export class PrebuildInitializer extends jspb.Message { export namespace PrebuildInitializer { export type AsObject = { prebuild?: SnapshotInitializer.AsObject, - git?: GitInitializer.AsObject, + gitList: Array, } } diff --git a/components/content-service-api/typescript/src/initializer_pb.js b/components/content-service-api/typescript/src/initializer_pb.js index 8a00062cdd444d..ff85920692f330 100644 --- a/components/content-service-api/typescript/src/initializer_pb.js +++ b/components/content-service-api/typescript/src/initializer_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ @@ -220,7 +220,7 @@ if (goog.DEBUG && !COMPILED) { * @constructor */ proto.contentservice.PrebuildInitializer = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); + jspb.Message.initialize(this, opt_data, 0, -1, proto.contentservice.PrebuildInitializer.repeatedFields_, null); }; goog.inherits(proto.contentservice.PrebuildInitializer, jspb.Message); if (goog.DEBUG && !COMPILED) { @@ -2086,6 +2086,13 @@ proto.contentservice.SnapshotInitializer.prototype.setSnapshot = function(value) +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.contentservice.PrebuildInitializer.repeatedFields_ = [2]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -2118,7 +2125,8 @@ proto.contentservice.PrebuildInitializer.prototype.toObject = function(opt_inclu proto.contentservice.PrebuildInitializer.toObject = function(includeInstance, msg) { var f, obj = { prebuild: (f = msg.getPrebuild()) && proto.contentservice.SnapshotInitializer.toObject(includeInstance, f), - git: (f = msg.getGit()) && proto.contentservice.GitInitializer.toObject(includeInstance, f) + gitList: jspb.Message.toObjectList(msg.getGitList(), + proto.contentservice.GitInitializer.toObject, includeInstance) }; if (includeInstance) { @@ -2163,7 +2171,7 @@ proto.contentservice.PrebuildInitializer.deserializeBinaryFromReader = function( case 2: var value = new proto.contentservice.GitInitializer; reader.readMessage(value,proto.contentservice.GitInitializer.deserializeBinaryFromReader); - msg.setGit(value); + msg.addGit(value); break; default: reader.skipField(); @@ -2202,9 +2210,9 @@ proto.contentservice.PrebuildInitializer.serializeBinaryToWriter = function(mess proto.contentservice.SnapshotInitializer.serializeBinaryToWriter ); } - f = message.getGit(); - if (f != null) { - writer.writeMessage( + f = message.getGitList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 2, f, proto.contentservice.GitInitializer.serializeBinaryToWriter @@ -2251,39 +2259,40 @@ proto.contentservice.PrebuildInitializer.prototype.hasPrebuild = function() { /** - * optional GitInitializer git = 2; - * @return {?proto.contentservice.GitInitializer} + * repeated GitInitializer git = 2; + * @return {!Array} */ -proto.contentservice.PrebuildInitializer.prototype.getGit = function() { - return /** @type{?proto.contentservice.GitInitializer} */ ( - jspb.Message.getWrapperField(this, proto.contentservice.GitInitializer, 2)); +proto.contentservice.PrebuildInitializer.prototype.getGitList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.contentservice.GitInitializer, 2)); }; /** - * @param {?proto.contentservice.GitInitializer|undefined} value + * @param {!Array} value * @return {!proto.contentservice.PrebuildInitializer} returns this */ -proto.contentservice.PrebuildInitializer.prototype.setGit = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.contentservice.PrebuildInitializer.prototype.setGitList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.contentservice.PrebuildInitializer} returns this + * @param {!proto.contentservice.GitInitializer=} opt_value + * @param {number=} opt_index + * @return {!proto.contentservice.GitInitializer} */ -proto.contentservice.PrebuildInitializer.prototype.clearGit = function() { - return this.setGit(undefined); +proto.contentservice.PrebuildInitializer.prototype.addGit = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.contentservice.GitInitializer, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.contentservice.PrebuildInitializer} returns this */ -proto.contentservice.PrebuildInitializer.prototype.hasGit = function() { - return jspb.Message.getField(this, 2) != null; +proto.contentservice.PrebuildInitializer.prototype.clearGitList = function() { + return this.setGitList([]); }; diff --git a/components/content-service-api/typescript/src/workspace_grpc_pb.d.ts b/components/content-service-api/typescript/src/workspace_grpc_pb.d.ts index e7809c822b94df..9dc526646a2fc9 100644 --- a/components/content-service-api/typescript/src/workspace_grpc_pb.d.ts +++ b/components/content-service-api/typescript/src/workspace_grpc_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/workspace_pb.d.ts b/components/content-service-api/typescript/src/workspace_pb.d.ts index 3bf3816e1ffcc2..1deb2c507faa40 100644 --- a/components/content-service-api/typescript/src/workspace_pb.d.ts +++ b/components/content-service-api/typescript/src/workspace_pb.d.ts @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service-api/typescript/src/workspace_pb.js b/components/content-service-api/typescript/src/workspace_pb.js index cc954dfd5b9df8..ffd071c31f754e 100644 --- a/components/content-service-api/typescript/src/workspace_pb.js +++ b/components/content-service-api/typescript/src/workspace_pb.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ diff --git a/components/content-service/pkg/initializer/git.go b/components/content-service/pkg/initializer/git.go index d4954c45a886e6..34b5a6fe6bebf7 100644 --- a/components/content-service/pkg/initializer/git.go +++ b/components/content-service/pkg/initializer/git.go @@ -141,7 +141,8 @@ func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) { if ws.TargetMode == RemoteBranch { // create local branch based on specific remote branch if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/"+ws.CloneTarget); err != nil { - return err + log.WithField("remoteURI", ws.RemoteURI).WithField("branch", ws.CloneTarget).Debug("Remote branch doesn't exist.") + return nil } } else if ws.TargetMode == LocalBranch { // checkout local branch based on remote HEAD diff --git a/components/content-service/pkg/initializer/initializer.go b/components/content-service/pkg/initializer/initializer.go index 5091e4a2da84e6..85e4ce1c1e15e0 100644 --- a/components/content-service/pkg/initializer/initializer.go +++ b/components/content-service/pkg/initializer/initializer.go @@ -111,14 +111,6 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader if ir.Prebuild == nil { return nil, status.Error(codes.InvalidArgument, "missing prebuild initializer spec") } - if ir.Prebuild.Git == nil { - return nil, status.Error(codes.InvalidArgument, "missing prebuild Git initializer spec") - } - - gitinit, err := newGitInitializer(ctx, loc, ir.Prebuild.Git, opts.ForceGitpodUserForGit) - if err != nil { - return nil, err - } var snapshot *SnapshotInitializer if ir.Prebuild.Prebuild != nil { snapshot, err = newSnapshotInitializer(loc, rs, ir.Prebuild.Prebuild) @@ -126,10 +118,17 @@ func NewFromRequest(ctx context.Context, loc string, rs storage.DirectDownloader return nil, status.Error(codes.Internal, fmt.Sprintf("cannot setup prebuild init: %v", err)) } } - + var gits []*GitInitializer + for _, gi := range ir.Prebuild.Git { + gitinit, err := newGitInitializer(ctx, loc, gi, opts.ForceGitpodUserForGit) + if err != nil { + return nil, err + } + gits = append(gits, gitinit) + } initializer = &PrebuildInitializer{ - Git: gitinit, Prebuild: snapshot, + Git: gits, } } else if ir, ok := spec.(*csapi.WorkspaceInitializer_Snapshot); ok { initializer, err = newSnapshotInitializer(loc, rs, ir.Snapshot) diff --git a/components/content-service/pkg/initializer/prebuild.go b/components/content-service/pkg/initializer/prebuild.go index da5015ce1f84a6..6585aca80b50df 100644 --- a/components/content-service/pkg/initializer/prebuild.go +++ b/components/content-service/pkg/initializer/prebuild.go @@ -26,7 +26,7 @@ import ( // PrebuildInitializer first tries to restore the snapshot/prebuild and if that succeeds performs Git operations. // If restoring the prebuild does not succeed we fall back to Git entriely. type PrebuildInitializer struct { - Git *GitInitializer + Git []*GitInitializer Prebuild *SnapshotInitializer } @@ -45,14 +45,11 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp tracelog.String("snapshot", p.Prebuild.Snapshot), ) } - if p.Git == nil { + if len(p.Git) == 0 { spandata = append(spandata, tracelog.Bool("hasGit", false)) } else { spandata = append(spandata, tracelog.Bool("hasGit", true), - tracelog.String("git.targetMode", string(p.Git.TargetMode)), - tracelog.String("git.cloneTarget", string(p.Git.CloneTarget)), - tracelog.String("git.location", string(p.Git.Location)), ) } span.LogFields(spandata...) @@ -71,7 +68,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: %w", err) } - return p.Git.Run(ctx, mappings) + for _, gi := range p.Git { + _, err = gi.Run(ctx, mappings) + if err != nil { + return csapi.WorkspaceInitFromOther, xerrors.Errorf("prebuild initializer: Git fallback: %w", err) + } + } } } @@ -80,12 +82,38 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp src = csapi.WorkspaceInitFromPrebuild // make sure we're on the correct branch + for _, gi := range p.Git { + err = runGitInit(ctx, gi) + if err != nil { + return src, err + } + } + log.Debug("Initialized workspace with prebuilt snapshot") + return +} + +func clearWorkspace(location string) error { + files, err := filepath.Glob(filepath.Join(location, "*")) + if err != nil { + return err + } + for _, file := range files { + err = os.RemoveAll(file) + if err != nil { + return xerrors.Errorf("prebuild initializer: %w", err) + } + } + return nil +} + +func runGitInit(ctx context.Context, gInit *GitInitializer) (err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "runGitInit") span.LogFields( - tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(p.Git.Location))), - tracelog.String("location", fmt.Sprintf("%v", p.Git.Location)), + tracelog.String("IsWorkingCopy", fmt.Sprintf("%v", git.IsWorkingCopy(gInit.Location))), + tracelog.String("location", fmt.Sprintf("%v", gInit.Location)), ) - if git.IsWorkingCopy(p.Git.Location) { - out, err := p.Git.GitWithOutput(ctx, "stash", "push", "-u") + if git.IsWorkingCopy(gInit.Location) { + out, err := gInit.GitWithOutput(ctx, "stash", "push", "-u") if err != nil { var giterr git.OpFailedError if errors.As(err, &giterr) && strings.Contains(giterr.Output, "You do not have the initial commit yet") { @@ -93,48 +121,32 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp // In this case that's not an error though, hence we don't want to fail here. } else { // git returned a non-zero exit code because of some reason we did not anticipate or an actual failure. - return src, xerrors.Errorf("prebuild initializer: %w", err) + return xerrors.Errorf("prebuild initializer: %w", err) } } didStash := !strings.Contains(string(out), "No local changes to save") - err = p.Git.Fetch(ctx) + err = gInit.Fetch(ctx) if err != nil { - return src, xerrors.Errorf("prebuild initializer: %w", err) + return xerrors.Errorf("prebuild initializer: %w", err) } - err = p.Git.realizeCloneTarget(ctx) + err = gInit.realizeCloneTarget(ctx) if err != nil { - return src, xerrors.Errorf("prebuild initializer: %w", err) + return xerrors.Errorf("prebuild initializer: %w", err) } // If any of these cleanup operations fail that's no reason to fail ws initialization. // It just results in a slightly degraded state. if didStash { - err = p.Git.Git(ctx, "stash", "pop") + err = gInit.Git(ctx, "stash", "pop") if err != nil { // If restoring the stashed changes produces merge conflicts on the new Git ref, simply // throw them away (they'll remain in the stash, but are likely outdated anyway). - _ = p.Git.Git(ctx, "reset", "--hard") + _ = gInit.Git(ctx, "reset", "--hard") } } log.Debug("prebuild initializer Git operations complete") } - - log.Debug("Initialized workspace with prebuilt snapshot") - return -} - -func clearWorkspace(location string) error { - files, err := filepath.Glob(filepath.Join(location, "*")) - if err != nil { - return err - } - for _, file := range files { - err = os.RemoveAll(file) - if err != nil { - return xerrors.Errorf("prebuild initializer: %w", err) - } - } return nil } diff --git a/components/content-service/pkg/layer/provider.go b/components/content-service/pkg/layer/provider.go index 4e146dc8c1a24d..851492c3280e85 100644 --- a/components/content-service/pkg/layer/provider.go +++ b/components/content-service/pkg/layer/provider.go @@ -184,7 +184,21 @@ func (s *Provider) GetContentLayer(ctx context.Context, owner, workspaceID strin span.LogKV("fallback-to-git", err.Error()) // we failed creating a prebuild initializer, so let's try falling back to the Git part. - initializer = &csapi.WorkspaceInitializer{Spec: &csapi.WorkspaceInitializer_Git{Git: pis.Git}} + var init []*csapi.WorkspaceInitializer + for _, gi := range pis.Git { + init = append(init, &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Git{ + Git: gi, + }, + }) + } + initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Composite{ + Composite: &csapi.CompositeInitializer{ + Initializer: init, + }, + }, + } } else { // creating the initializer worked - we're done here return diff --git a/components/dashboard/src/projects/Prebuilds.tsx b/components/dashboard/src/projects/Prebuilds.tsx index c1f763b0417b41..e72e5fda9b9668 100644 --- a/components/dashboard/src/projects/Prebuilds.tsx +++ b/components/dashboard/src/projects/Prebuilds.tsx @@ -219,14 +219,18 @@ export default function (props: { project?: Project, isAdminDashboard?: boolean
-
{shortCommitMessage(p.info.changeTitle)}
+ +
{shortCommitMessage(p.info.changeTitle)}
+

{p.info.changeAuthorAvatar && {p.info.changeAuthor}}Authored {formatDate(p.info.changeDate)} · {p.info.changeHash?.substring(0, 8)}

 -
- {p.info.branch} -
+ +
+ {p.info.branch} +
+ {!props.isAdminDashboard && } diff --git a/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts b/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts index b3360dc525138a..0c503d417eb811 100644 --- a/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts +++ b/components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace-updatable.ts @@ -31,6 +31,12 @@ export class DBPrebuiltWorkspaceUpdatable implements PrebuiltWorkspaceUpdatable @Column() repo: string; + @Column({ + default: '', + transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED + }) + commitSHA?: string; + @Column() isResolved: boolean; diff --git a/components/gitpod-db/src/typeorm/migration/1646803519382-PrebuildUpdatableSHA.ts b/components/gitpod-db/src/typeorm/migration/1646803519382-PrebuildUpdatableSHA.ts new file mode 100644 index 00000000000000..0739c264b72610 --- /dev/null +++ b/components/gitpod-db/src/typeorm/migration/1646803519382-PrebuildUpdatableSHA.ts @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import {MigrationInterface, QueryRunner} from "typeorm"; +import { columnExists } from "./helper/helper"; + +export class PrebuildUpdatableSHA1646803519382 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + if (!(await columnExists(queryRunner, "d_b_prebuilt_workspace_updatable", "commitSHA"))) { + await queryRunner.query("ALTER TABLE d_b_prebuilt_workspace_updatable ADD COLUMN commitSHA varchar(255) NOT NULL"); + } + } + + public async down(queryRunner: QueryRunner): Promise { + } + +} diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index 779f887d73d539..50658604cbc5c1 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -633,8 +633,9 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { let query = repo.createQueryBuilder('pws'); query = query.where('pws.cloneURL = :cloneURL', { cloneURL }) - query = query.orderBy('pws.creationTime', 'ASC'); + query = query.orderBy('pws.creationTime', 'DESC'); query = query.innerJoinAndMapOne('pws.workspace', DBWorkspace, 'ws', 'pws.buildWorkspaceId = ws.id'); + query = query.where('ws.deleted = false'); const res = await query.getMany(); return res.map(r => { diff --git a/components/gitpod-protocol/data/gitpod-schema.json b/components/gitpod-protocol/data/gitpod-schema.json index b81233a1397ed4..e6a733fc2b6d80 100644 --- a/components/gitpod-protocol/data/gitpod-schema.json +++ b/components/gitpod-protocol/data/gitpod-schema.json @@ -136,13 +136,38 @@ }, "additionalProperties": false }, + "additionalRepositories": { + "type": "array", + "description": "List of additional repositories that are part of this project.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "url": { + "type": ["string"], + "description": "The url of the git repository to clone. Supports any context URLs." + }, + "checkoutLocation": { + "type": "string", + "description": "Path to where the repository should be checked out relative to `/workspace`. Defaults to the simple repository name." + } + }, + "additionalProperties": false + } + }, + "mainConfiguration": { + "type": "string", + "description": "The main repository, containing the dev environment configuration." + }, "checkoutLocation": { "type": "string", - "description": "Path to where the repository should be checked out." + "description": "Path to where the repository should be checked out relative to `/workspace`. Defaults to the simple repository name." }, "workspaceLocation": { "type": "string", - "description": "Path to where the IDE's workspace should be opened." + "description": "Path to where the IDE's workspace should be opened. Supports vscode's `*.code-workspace` files." }, "gitConfig": { "type": [ @@ -224,6 +249,7 @@ }, "experimentalNetwork": { "type": "boolean", + "deprecationMessage": "The 'experimentalNetwork' property is deprecated.", "description": "Experimental network configuration in workspaces (deprecated). Enabled by default" } }, diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 8c42f139186c08..4d69521178d416 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -7,6 +7,7 @@ import { WorkspaceInstance, PortVisibility } from "./workspace-instance"; import { RoleOrPermission } from "./permission"; import { Project } from "./teams-projects-protocol"; +import { createHash } from "crypto"; export interface UserInfo { name?: string @@ -567,7 +568,14 @@ export interface VSCodeConfig { extensions?: string[]; } +export interface RepositoryCloneInformation { + url: string; + checkoutLocation?: string; +} + export interface WorkspaceConfig { + mainConfiguration?: string; + additionalRepositories?: RepositoryCloneInformation[]; image?: ImageConfig; ports?: PortConfig[]; tasks?: TaskConfig[]; @@ -688,6 +696,10 @@ export interface PrebuiltWorkspaceUpdatable { repo: string; isResolved: boolean; installationId: string; + /** + * the commitSHA of the commit that triggered the prebuild + */ + commitSHA?: string; issue?: string; contextUrl?: string; } @@ -868,6 +880,10 @@ export namespace SnapshotContext { export interface StartPrebuildContext extends WorkspaceContext { actual: WorkspaceContext; commitHistory?: string[]; + additionalRepositoryCommitHistories?: { + cloneUrl: string; + commitHistory: string[]; + }[]; project?: Project; branch?: string; } @@ -970,9 +986,43 @@ export namespace AdditionalContentContext { } } -export interface CommitContext extends WorkspaceContext, Commit { +export interface CommitContext extends WorkspaceContext, GitCheckoutInfo { /** @deprecated Moved to .repository.cloneUrl, left here for backwards-compatibility for old workspace contextes in the DB */ cloneUrl?: string + + /** + * The clone and checkout information for additional repositories in case of multi-repo projects. + */ + additionalRepositoryCheckoutInfo?: GitCheckoutInfo[]; +} + +export namespace CommitContext { + + /** + * Creates a hash for all the commits of the CommitContext and all sub-repo commit infos. + * The hash is max 255 chars long. + * @param commitContext + * @returns hash for commitcontext + */ + export function computeHash(commitContext: CommitContext): string { + // for single commits we use the revision to be backward compatible. + if (!commitContext.additionalRepositoryCheckoutInfo || commitContext.additionalRepositoryCheckoutInfo.length === 0) { + return commitContext.revision; + } + const hasher = createHash('sha256'); + hasher.update(commitContext.revision); + for (const info of commitContext.additionalRepositoryCheckoutInfo) { + hasher.update(info.revision); + } + return hasher.digest('hex'); + } + +} + +export interface GitCheckoutInfo extends Commit { + checkoutLocation?: string; + upstreamRemoteURI?: string; + localBranch?: string; } export namespace CommitContext { diff --git a/components/gitpod-protocol/src/wsready.ts b/components/gitpod-protocol/src/wsready.ts index 5cf18d354dc25a..1329047705ed1d 100644 --- a/components/gitpod-protocol/src/wsready.ts +++ b/components/gitpod-protocol/src/wsready.ts @@ -1,10 +1,10 @@ /** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. * Licensed under the GNU Affero General Public License (AGPL). * See License-AGPL.txt in the project root for license information. */ -// generated using github.com/32leaves/bel on 2021-11-04 12:16:53.917570766 +0000 UTC m=+0.006002884 +// generated using github.com/32leaves/bel on 2022-02-15 11:53:18.380158212 +0000 UTC m=+0.011913675 // DO NOT MODIFY export enum WorkspaceInitSource { diff --git a/components/server/ee/src/container-module.ts b/components/server/ee/src/container-module.ts index c232119d063e16..8be44c6b019fca 100644 --- a/components/server/ee/src/container-module.ts +++ b/components/server/ee/src/container-module.ts @@ -25,7 +25,6 @@ import { GitLabApp } from "./prebuilds/gitlab-app"; import { BitbucketApp } from "./prebuilds/bitbucket-app"; import { IPrefixContextParser } from "../../src/workspace/context-parser"; import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser"; -import { StartIncrementalPrebuildContextParser } from "./prebuilds/start-incremental-prebuild-context-parser"; import { WorkspaceFactory } from "../../src/workspace/workspace-factory"; import { WorkspaceFactoryEE } from "./workspace/workspace-factory"; import { MonitoringEndpointsAppEE } from "./monitoring-endpoint-ee"; @@ -59,7 +58,6 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is bind(WorkspaceHealthMonitoring).toSelf().inSingletonScope(); bind(PrebuildManager).toSelf().inSingletonScope(); bind(IPrefixContextParser).to(StartPrebuildContextParser).inSingletonScope(); - bind(IPrefixContextParser).to(StartIncrementalPrebuildContextParser).inSingletonScope(); bind(GithubApp).toSelf().inSingletonScope(); bind(GitHubAppSupport).toSelf().inSingletonScope(); bind(GithubAppRules).toSelf().inSingletonScope(); diff --git a/components/server/ee/src/prebuilds/bitbucket-app.ts b/components/server/ee/src/prebuilds/bitbucket-app.ts index 8107a7370b9d2b..d83dc118fc1d82 100644 --- a/components/server/ee/src/prebuilds/bitbucket-app.ts +++ b/components/server/ee/src/prebuilds/bitbucket-app.ts @@ -7,10 +7,13 @@ import * as express from 'express'; import { postConstruct, injectable, inject } from 'inversify'; import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib'; -import { User, StartPrebuildResult, Project } from '@gitpod/gitpod-protocol'; +import { User, StartPrebuildResult, CommitContext, CommitInfo, Project } from '@gitpod/gitpod-protocol'; import { PrebuildManager } from '../prebuilds/prebuild-manager'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TokenService } from '../../../src/user/token-service'; +import { ContextParser } from '../../../src/workspace/context-parser-service'; +import { HostContextProvider } from '../../../src/auth/host-context-provider'; +import { RepoURL } from '../../../src/repohost'; @injectable() export class BitbucketApp { @@ -20,6 +23,8 @@ export class BitbucketApp { @inject(TokenService) protected readonly tokenService: TokenService; @inject(ProjectDB) protected readonly projectDB: ProjectDB; @inject(TeamDB) protected readonly teamDB: TeamDB; + @inject(ContextParser) protected readonly contextParser: ContextParser; + @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; protected _router = express.Router(); public static path = '/apps/bitbucket/'; @@ -88,25 +93,24 @@ export class BitbucketApp { const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx); try { const contextURL = this.createContextUrl(data); + const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext; span.setTag('contextURL', contextURL); - const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); + const config = await this.prebuildManager.fetchConfig({ span }, user, context); if (!this.prebuildManager.shouldPrebuild(config)) { console.log('Bitbucket push event: No config. No prebuild.'); return undefined; } - console.debug('Bitbucket push event: Starting prebuild.', { contextURL }); - + console.log('Starting prebuild.', { contextURL }) + const {host, owner, repo} = RepoURL.parseRepoUrl(data.repoUrl)!; + const hostCtx = this.hostCtxProvider.get(host); + let commitInfo: CommitInfo | undefined; + if (hostCtx?.services?.repositoryProvider) { + commitInfo = await hostCtx.services.repositoryProvider.getCommitInfo(user, owner, repo, data.commitHash); + } const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user); - - const ws = await this.prebuildManager.startPrebuild({ span }, { - user: projectAndOwner.user, - project: projectAndOwner?.project, - branch: data.branchName, - contextURL, - cloneURL: data.gitCloneUrl, - commit: data.commitHash - }); + // todo@alex: add branch and project args + const ws = await this.prebuildManager.startPrebuild({ span }, { user, project: projectAndOwner?.project, context, commitInfo }); return ws; } finally { span.finish(); diff --git a/components/server/ee/src/prebuilds/github-app.ts b/components/server/ee/src/prebuilds/github-app.ts index 35f44710d6a4cd..8b3977d0415f69 100644 --- a/components/server/ee/src/prebuilds/github-app.ts +++ b/components/server/ee/src/prebuilds/github-app.ts @@ -12,13 +12,17 @@ import { Config } from '../../../src/config'; import { AppInstallationDB, TracedWorkspaceDB, DBWithTracing, UserDB, WorkspaceDB, ProjectDB, TeamDB } from '@gitpod/gitpod-db/lib'; import * as express from 'express'; import { log, LogContext, LogrusLogLevel } from '@gitpod/gitpod-protocol/lib/util/logging'; -import { WorkspaceConfig, User, Project, StartPrebuildResult } from '@gitpod/gitpod-protocol'; +import { WorkspaceConfig, User, Project, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol'; import { GithubAppRules } from './github-app-rules'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { PrebuildManager } from './prebuild-manager'; import { PrebuildStatusMaintainer } from './prebuilt-status-maintainer'; import { Options, ApplicationFunctionOptions } from 'probot/lib/types'; import { asyncHandler } from '../../../src/express-util'; +import { ContextParser } from '../../../src/workspace/context-parser-service'; +import { HostContextProvider } from '../../../src/auth/host-context-provider'; +import { RepoURL } from '../../../src/repohost'; + /** * GitHub app urls: @@ -39,6 +43,8 @@ export class GithubApp { @inject(TracedWorkspaceDB) protected readonly workspaceDB: DBWithTracing; @inject(GithubAppRules) protected readonly appRules: GithubAppRules; @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; + @inject(ContextParser) protected readonly contextParser: ContextParser; + @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; readonly server: Server | undefined; @@ -159,6 +165,19 @@ export class GithubApp { }); } + private async findOwnerAndProject(installationID: number | undefined, cloneURL: string): Promise<{ user: User, project?: Project }> { + const installationOwner = installationID ? await this.findInstallationOwner(installationID) : undefined; + const project = await this.projectDB.findProjectByCloneUrl(cloneURL); + const user = await this.selectUserForPrebuild(installationOwner, project); + if (!user) { + log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: cloneURL, installationID, project }); + throw new Error(`No installation found for ${installationID}`); + } + return { + user, project + } + } + protected async handlePushEvent(ctx: Context<'push'>): Promise { const span = TraceContext.startSpan("GithubApp.handlePushEvent", {}); span.setTag("request", ctx.id); @@ -166,13 +185,7 @@ export class GithubApp { try { const installationId = ctx.payload.installation?.id; const cloneURL = ctx.payload.repository.clone_url; - const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined; - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); - const user = await this.selectUserForPrebuild(installationOwner, project); - if (!user) { - log.info(`Did not find user for installation. Probably an incomplete app installation.`, { repo: ctx.payload.repository, installationId, project }); - return; - } + let { user, project } = await this.findOwnerAndProject(installationId, cloneURL); const logCtx: LogContext = { userId: user.id }; if (!!user.blocked) { @@ -192,7 +205,13 @@ export class GithubApp { const contextURL = `${repo.html_url}/tree/${branch}`; span.setTag('contextURL', contextURL); - let config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); + const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext; + const config = await this.prebuildManager.fetchConfig({ span }, user, context); + + const r = await this.ensureMainProjectAndUser(user, project, context, installationId); + user = r.user; + project = r.project; + const runPrebuild = this.appRules.shouldRunPrebuild(config, branch == repo.default_branch, false, false); if (!runPrebuild) { const reason = `Not running prebuild, the user did not enable it for this context`; @@ -201,7 +220,8 @@ export class GithubApp { return; } - this.prebuildManager.startPrebuild({ span }, { user, contextURL, cloneURL: repo.clone_url, commit: pl.after, branch, project}) + const commitInfo = await this.getCommitInfo(user, repo.html_url, ctx.payload.after); + this.prebuildManager.startPrebuild({ span }, { user, context, project, commitInfo}) .catch(err => log.error(logCtx, "Error while starting prebuild", err, { contextURL })); } catch (e) { TraceContext.setError({ span }, e); @@ -211,6 +231,33 @@ export class GithubApp { } } + private async ensureMainProjectAndUser(user: User, project: Project | undefined, context: CommitContext, installationId?: number): Promise<{user: User, project?: Project}> { + // if it's a sub-repo of a multi-repo project, we look up the owner of the main repo + if (!!context.additionalRepositoryCheckoutInfo && (!project || context.repository.cloneUrl !== project.cloneUrl)) { + const owner = await this.findOwnerAndProject(installationId, context.repository.cloneUrl); + if (owner) { + return { + user: owner.user, + project: owner.project || project + }; + } + } + return { + user, + project + }; + } + + private async getCommitInfo(user: User, repoURL: string, commitSHA: string) { + const parsedRepo = RepoURL.parseRepoUrl(repoURL)!; + const hostCtx = this.hostCtxProvider.get(parsedRepo.host); + let commitInfo: CommitInfo | undefined; + if (hostCtx?.services?.repositoryProvider) { + commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA); + } + return commitInfo; + } + protected getBranchFromRef(ref: string): string | undefined { const headsPrefix = "refs/heads/"; if (ref.startsWith(headsPrefix)) { @@ -227,22 +274,23 @@ export class GithubApp { try { const installationId = ctx.payload.installation?.id; const cloneURL = ctx.payload.repository.clone_url; - const installationOwner = installationId ? await this.findInstallationOwner(installationId) : undefined; - const project = await this.projectDB.findProjectByCloneUrl(cloneURL); - const user = await this.selectUserForPrebuild(installationOwner, project); - if (!user) { - log.info("Did not find user for installation. Probably an incomplete app installation.", { repo: ctx.payload.repository, installationId, project }); - return; - } - const pr = ctx.payload.pull_request; const contextURL = pr.html_url; - const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); + let { user, project} = await this.findOwnerAndProject(installationId, cloneURL); + + const context = await this.contextParser.handle( { span }, user, contextURL) as CommitContext; + const config = await this.prebuildManager.fetchConfig({ span }, user, context); - const prebuildStartPromise = this.onPrStartPrebuild({ span }, ctx, config, user, project); - this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise).catch(() => {/** ignore */}); - this.onPrAddBadge(config, ctx); - this.onPrAddComment(config, ctx).catch(() => {/** ignore */}); + const r = await this.ensureMainProjectAndUser(user, project, context, installationId); + user = r.user; + project = r.project; + + const prebuildStartPromise = await this.onPrStartPrebuild({ span }, ctx, config, context, user, project); + if (prebuildStartPromise) { + await this.onPrAddCheck({ span }, config, ctx, prebuildStartPromise); + this.onPrAddBadge(config, ctx); + await this.onPrAddComment(config, ctx); + } } catch (e) { TraceContext.setError({ span }, e); throw e; @@ -251,7 +299,7 @@ export class GithubApp { } } - protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: Promise | undefined) { + protected async onPrAddCheck(tracecContext: TraceContext, config: WorkspaceConfig | undefined, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, start: StartPrebuildResult) { if (!start) { return; } @@ -262,8 +310,7 @@ export class GithubApp { const span = TraceContext.startSpan("onPrAddCheck", tracecContext); try { - const spr = await start; - const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(spr.wsid); + const pws = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(start.wsid); if (!pws) { return; } @@ -286,18 +333,16 @@ export class GithubApp { } } - protected onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig | undefined, user: User, project?: Project): Promise | undefined { + protected async onPrStartPrebuild(tracecContext: TraceContext, ctx: Context<'pull_request.opened' | 'pull_request.synchronize' | 'pull_request.reopened'>, config: WorkspaceConfig, context: CommitContext, user: User, project?: Project): Promise { const pr = ctx.payload.pull_request; - const pr_head = pr.head; const contextURL = pr.html_url; - const branch = pr.head.ref; - const cloneURL = pr_head.repo.clone_url; const isFork = pr.head.repo.id !== pr.base.repo.id; const runPrebuild = this.appRules.shouldRunPrebuild(config, false, true, isFork); let prebuildStartPromise: Promise | undefined; if (runPrebuild) { - prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, contextURL, cloneURL, commit: pr_head.sha, branch, project}); + const commitInfo = await this.getCommitInfo(user, ctx.payload.repository.html_url, pr.head.sha); + prebuildStartPromise = this.prebuildManager.startPrebuild(tracecContext, {user, context, project, commitInfo}); prebuildStartPromise.catch(err => log.error(err, "Error while starting prebuild", { contextURL })); return prebuildStartPromise; } else { diff --git a/components/server/ee/src/prebuilds/gitlab-app.ts b/components/server/ee/src/prebuilds/gitlab-app.ts index c370db521070a4..20dbd732c0bde3 100644 --- a/components/server/ee/src/prebuilds/gitlab-app.ts +++ b/components/server/ee/src/prebuilds/gitlab-app.ts @@ -7,13 +7,15 @@ import * as express from 'express'; import { postConstruct, injectable, inject } from 'inversify'; import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib'; -import { Project, User, StartPrebuildResult } from '@gitpod/gitpod-protocol'; +import { Project, User, StartPrebuildResult, CommitContext, CommitInfo } from '@gitpod/gitpod-protocol'; import { PrebuildManager } from '../prebuilds/prebuild-manager'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; import { TokenService } from '../../../src/user/token-service'; import { HostContextProvider } from '../../../src/auth/host-context-provider'; import { GitlabService } from './gitlab-service'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; +import { ContextParser } from '../../../src/workspace/context-parser-service'; +import { RepoURL } from '../../../src/repohost'; @injectable() export class GitLabApp { @@ -24,6 +26,7 @@ export class GitLabApp { @inject(HostContextProvider) protected readonly hostCtxProvider: HostContextProvider; @inject(ProjectDB) protected readonly projectDB: ProjectDB; @inject(TeamDB) protected readonly teamDB: TeamDB; + @inject(ContextParser) protected readonly contextParser: ContextParser; protected _router = express.Router(); public static path = '/apps/gitlab/'; @@ -96,7 +99,9 @@ export class GitLabApp { const contextURL = this.createContextUrl(body); log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL }); span.setTag('contextURL', contextURL); - const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL); + const context = await this.contextParser.handle({ span }, user, contextURL) as CommitContext; + const projectAndOwner = await this.findProjectAndOwner(context.repository.cloneUrl, user); + const config = await this.prebuildManager.fetchConfig({ span }, user, context); if (!this.prebuildManager.shouldPrebuild(config)) { log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { context: body, contextURL }); return undefined; @@ -104,18 +109,12 @@ export class GitLabApp { log.debug({ userId: user.id }, "GitLab push hook: Starting prebuild", { body, contextURL }); - const cloneURL = body.repository.git_http_url; - const branch = this.getBranchFromRef(body.ref); - - const projectAndOwner = await this.findProjectAndOwner(cloneURL, user); - + const commitInfo = await this.getCommitInfo(user, body.repository.git_http_url, body.after); const ws = await this.prebuildManager.startPrebuild({ span }, { - user: projectAndOwner.user, + user: projectAndOwner?.user || user, project: projectAndOwner?.project, - contextURL, - cloneURL, - commit: body.after, - branch, + context, + commitInfo }); return ws; @@ -124,6 +123,16 @@ export class GitLabApp { } } + private async getCommitInfo(user: User, repoURL: string, commitSHA: string) { + const parsedRepo = RepoURL.parseRepoUrl(repoURL)!; + const hostCtx = this.hostCtxProvider.get(parsedRepo.host); + let commitInfo: CommitInfo | undefined; + if (hostCtx?.services?.repositoryProvider) { + commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA); + } + return commitInfo; + } + /** * Finds the relevant user account and project to the provided webhook event information. * @@ -135,7 +144,7 @@ export class GitLabApp { * @param webhookInstaller the user account known from the webhook installation * @returns a promise which resolves to a user account and an optional project. */ - protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> { + protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> { const project = await this.projectDB.findProjectByCloneUrl(cloneURL); if (project) { if (project.userId) { diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index 1504ab5ea0615d..1c2a74de508b1f 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -5,10 +5,10 @@ */ import { DBWithTracing, TracedWorkspaceDB, WorkspaceDB } from '@gitpod/gitpod-db/lib'; -import { CommitContext, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol'; +import { CommitContext, CommitInfo, PrebuiltWorkspace, Project, ProjectEnvVar, StartPrebuildContext, StartPrebuildResult, TaskConfig, User, Workspace, WorkspaceConfig, WorkspaceInstance } from '@gitpod/gitpod-protocol'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing'; -import { HostContextProvider } from '../../../src/auth/host-context-provider'; +import { getCommitInfo, HostContextProvider } from '../../../src/auth/host-context-provider'; import { WorkspaceFactory } from '../../../src/workspace/workspace-factory'; import { ConfigProvider } from '../../../src/workspace/config-provider'; import { WorkspaceStarter } from '../../../src/workspace/workspace-starter'; @@ -18,7 +18,6 @@ import { secondsBefore } from '@gitpod/gitpod-protocol/lib/util/timeutil'; import { inject, injectable } from 'inversify'; import * as opentracing from 'opentracing'; -import { URL } from 'url'; export class WorkspaceRunningError extends Error { constructor(msg: string, public instance: WorkspaceInstance) { @@ -28,11 +27,9 @@ export class WorkspaceRunningError extends Error { export interface StartPrebuildParams { user: User; - contextURL: string; - cloneURL: string; - branch?: string; - commit: string; + context: CommitContext; project?: Project; + commitInfo?: CommitInfo; } const PREBUILD_LIMITER_WINDOW_SECONDS = 60; @@ -48,43 +45,25 @@ export class PrebuildManager { @inject(Config) protected readonly config: Config; @inject(ProjectsService) protected readonly projectService: ProjectsService; - async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise { - const span = TraceContext.startSpan("hasPrebuilds", ctx); - span.setTag(cloneURL, cloneURL); - try { - const existingPBs = await this.workspaceDB.trace({ span }).findPrebuildsWithWorkpace(cloneURL); - for (const pb of existingPBs) { - if (!pb.workspace.contextURL.startsWith('prebuild')) { - return true; - } - } - return false; - } catch (err) { - TraceContext.setError({ span }, err); - throw err; - } finally { - span.finish(); - } - } - - async startPrebuild(ctx: TraceContext, { contextURL, cloneURL, commit, branch, project, user }: StartPrebuildParams): Promise { + async startPrebuild(ctx: TraceContext, { context, project, user, commitInfo }: StartPrebuildParams): Promise { const span = TraceContext.startSpan("startPrebuild", ctx); - span.setTag("contextURL", contextURL); + const cloneURL = context.repository.cloneUrl; + const commitSHAIdentifier = CommitContext.computeHash(context); span.setTag("cloneURL", cloneURL); - span.setTag("commit", commit); + span.setTag("commit", commitInfo?.sha); try { if (user.blocked) { throw new Error("Blocked users cannot start prebuilds."); } - const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commit); + const existingPB = await this.workspaceDB.trace({ span }).findPrebuiltWorkspaceByCommit(cloneURL, commitSHAIdentifier); // If the existing prebuild is failed, we want to retrigger it. if (!!existingPB && existingPB.state !== 'aborted' && existingPB.state !== 'failed' && existingPB.state !== 'timeout') { // If the existing prebuild is based on an outdated project config, we also want to retrigger it. const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId); const existingConfig = existingPBWS?.config; - const newConfig = await this.fetchConfig({ span }, user, contextURL); - log.debug(`startPrebuild | commit: ${commit}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`); + const newConfig = await this.fetchConfig({ span }, user, context); + log.debug(`startPrebuild | commits: ${commitSHAIdentifier}, existingPB: ${existingPB.id}, existingConfig: ${JSON.stringify(existingConfig)}, newConfig: ${JSON.stringify(newConfig)}}`); const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks .map(task => Object.keys(task) .filter(key => ['before', 'init', 'prebuild'].includes(key)) @@ -98,35 +77,46 @@ export class PrebuildManager { } } - const contextParser = this.getContextParserFor(contextURL); - if (!contextParser) { - throw new Error(`Cannot find context parser for URL: ${contextURL}`); - } - const actual = await contextParser.handle({ span }, user, contextURL) as CommitContext; - actual.revision = commit; // Make sure we target the correct commit here (might have changed between trigger and contextParser lookup) - actual.ref = undefined; - actual.forceCreateNewWorkspace = true; - const prebuildContext: StartPrebuildContext = { - title: `Prebuild of "${actual.title}"`, - actual, + title: `Prebuild of "${context.title}"`, + actual: context, project, - branch, - normalizedContextURL: actual.normalizedContextURL + branch: context.ref, + normalizedContextURL: context.normalizedContextURL }; - if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl, project)) { + if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) { const maxDepth = this.config.incrementalPrebuilds.commitHistory; - prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth); + const hostContext = this.hostContextProvider.get(context.repository.host); + const repoProvider = hostContext?.services?.repositoryProvider; + if (repoProvider) { + prebuildContext.commitHistory = await repoProvider.getCommitHistory(user, context.repository.owner, context.repository.name, context.revision, maxDepth); + if (context.additionalRepositoryCheckoutInfo && context.additionalRepositoryCheckoutInfo.length > 0) { + const histories = context.additionalRepositoryCheckoutInfo.map(async info => { + const commitHistory = await repoProvider.getCommitHistory(user, info.repository.owner, info.repository.name, info.revision, maxDepth); + return { + cloneUrl: info.repository.cloneUrl, + commitHistory + } + }); + prebuildContext.additionalRepositoryCommitHistories = await Promise.all(histories); + } + } } const projectEnvVarsPromise = project ? this.projectService.getProjectEnvironmentVariables(project.id) : []; - const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL); - const prebuild = await this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!; + + const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, context.normalizedContextURL!); + const prebuildPromise = this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(workspace.id)!; + + span.setTag("starting", true); + const projectEnvVars = await projectEnvVarsPromise; + await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']}); + const prebuild = await prebuildPromise; if (!prebuild) { - throw new Error(`Failed to create a prebuild for: ${contextURL}`); + throw new Error(`Failed to create a prebuild for: ${context.normalizedContextURL}`); } if (await this.shouldRateLimitPrebuild(span, cloneURL)) { @@ -143,9 +133,20 @@ export class PrebuildManager { }; } - span.setTag("starting", true); - const projectEnvVars = await projectEnvVarsPromise; - await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, {excludeFeatureFlags: ['full_workspace_backup']}); + if (project) { + let aCommitInfo = commitInfo; + if (!aCommitInfo) { + aCommitInfo = await getCommitInfo(this.hostContextProvider, user, context.repository.cloneUrl, context.revision); + if (!aCommitInfo) { + aCommitInfo = { + author: 'unknown', + commitMessage: 'unknown', + sha: context.revision + } + } + } + await this.storePrebuildInfo({ span }, project, prebuild, workspace, user, aCommitInfo); + } return { prebuildId: prebuild.id, wsid: workspace.id, done: false }; } catch (err) { TraceContext.setError({ span }, err); @@ -214,17 +215,10 @@ export class PrebuildManager { return this.config.incrementalPrebuilds.repositoryPasslist.some(url => trimRepoUrl(url) === repoUrl); } - async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise { + async fetchConfig(ctx: TraceContext, user: User, context: CommitContext): Promise { const span = TraceContext.startSpan("fetchConfig", ctx); - span.setTag("contextURL", contextURL); - try { - const contextParser = this.getContextParserFor(contextURL); - if (!contextParser) { - return undefined; - } - const context = await contextParser!.handle({ span }, user, contextURL); - return (await this.configProvider.fetchConfig({ span }, user, context as CommitContext)).config; + return (await this.configProvider.fetchConfig({ span }, user, context)).config; } catch (err) { TraceContext.setError({ span }, err); throw err; @@ -233,13 +227,31 @@ export class PrebuildManager { } } - protected getContextParserFor(contextURL: string) { - const host = new URL(contextURL).hostname; - const hostContext = this.hostContextProvider.get(host); - if (!hostContext) { - return undefined; - } - return hostContext.contextParser; + //TODO this doesn't belong so deep here. All this context should be stored on the surface not passed down. + protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User, commit: CommitInfo) { + const span = TraceContext.startSpan("storePrebuildInfo", ctx); + const { userId, teamId, name: projectName, id: projectId } = project; + await this.workspaceDB.trace({span}).storePrebuildInfo({ + id: pws.id, + buildWorkspaceId: pws.buildWorkspaceId, + basedOnPrebuildId: ws.basedOnPrebuildId, + teamId, + userId, + projectName, + projectId, + startedAt: pws.creationTime, + startedBy: "", // TODO + startedByAvatar: "", // TODO + cloneUrl: pws.cloneURL, + branch: pws.branch || "unknown", + changeAuthor: commit.author, + changeAuthorAvatar: commit.authorAvatarUrl, + changeDate: commit.authorDate || "", + changeHash: commit.sha, + changeTitle: commit.commitMessage, + // changePR + changeUrl: ws.contextURL, + }); } private async shouldRateLimitPrebuild(span: opentracing.Span, cloneURL: string): Promise { @@ -270,4 +282,4 @@ export class PrebuildManager { // Last resort default return PREBUILD_LIMITER_DEFAULT_LIMIT; } -} +} \ No newline at end of file diff --git a/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts b/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts index fe6c5026a60845..f263c9984decfc 100644 --- a/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts +++ b/components/server/ee/src/prebuilds/prebuilt-status-maintainer.ts @@ -46,7 +46,7 @@ export class PrebuildStatusMaintainer implements Disposable { this.disposables.push( repeat(this.periodicUpdatableCheck.bind(this), 60 * 1000) ); - log.debug("prebuild updatatable status maintainer started"); + log.debug("prebuild updatable status maintainer started"); } public async registerCheckRun(ctx: TraceContext, installationId: number, pws: PrebuiltWorkspace, cri: CheckRunInfo, config?: WorkspaceConfig) { @@ -64,6 +64,7 @@ export class PrebuildStatusMaintainer implements Disposable { id: uuidv4(), owner: cri.owner, repo: cri.repo, + commitSHA: cri.head_sha, isResolved: false, installationId: installationId.toString(), contextUrl: cri.details_url, @@ -131,8 +132,8 @@ export class PrebuildStatusMaintainer implements Disposable { return; } - const updatatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id); - await Promise.all(updatatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild))); + const updatables = await this.workspaceDB.trace({span}).findUpdatablesForPrebuild(prebuild.id); + await Promise.all(updatables.filter(u => !u.isResolved).map(u => this.doUpdate({span}, u, prebuild))); } catch (err) { TraceContext.setError({span}, err); throw err; @@ -141,38 +142,38 @@ export class PrebuildStatusMaintainer implements Disposable { } } - protected async doUpdate(ctx: TraceContext, updatatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise { + protected async doUpdate(ctx: TraceContext, updatable: PrebuiltWorkspaceUpdatable, pws: PrebuiltWorkspace): Promise { const span = TraceContext.startSpan("doUpdate", ctx); try { - const githubApi = await this.getGitHubApi(Number.parseInt(updatatable.installationId)); + const githubApi = await this.getGitHubApi(Number.parseInt(updatable.installationId)); if (!githubApi) { log.error("unable to authenticate GitHub app - this leaves user-facing checks dangling."); return; } const workspace = await this.workspaceDB.trace({span}).findById(pws.buildWorkspaceId); - if (!!updatatable.contextUrl && !!workspace) { + if (!!updatable.contextUrl && !!workspace) { const conclusion = this.getConclusionFromPrebuildState(pws); if (conclusion === 'pending') { - log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatatable.prebuiltWorkspaceId }); + log.info(`Prebuild is still running.`, { prebuiltWorkspaceId: updatable.prebuiltWorkspaceId }); return; } let found = true; try { await githubApi.repos.createCommitStatus({ - owner: updatatable.owner, - repo: updatatable.repo, + owner: updatable.owner, + repo: updatable.repo, context: "Gitpod", - sha: pws.commit, - target_url: updatatable.contextUrl, - description: conclusion === 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION, + sha: updatable.commitSHA || pws.commit, + target_url: updatable.contextUrl, + description: conclusion == 'success' ? DEFAULT_STATUS_DESCRIPTION : NON_PREBUILT_STATUS_DESCRIPTION, state: (workspace?.config?.github?.prebuilds?.addCheck === 'prevent-merge-on-error' ? conclusion : 'success') }); } catch (err) { if (err.message == "Not Found") { - log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatatable.owner, repo: updatatable.repo}); + log.info("Did not find repository while updating updatable. Probably we lost the GitHub permission for the repo.", {owner: updatable.owner, repo: updatable.repo}); found = true; } else { throw err; @@ -185,10 +186,10 @@ export class PrebuildStatusMaintainer implements Disposable { }, }); - await this.workspaceDB.trace({span}).markUpdatableResolved(updatatable.id); - log.info(`Resolved updatable. Marked check on ${updatatable.contextUrl} as ${conclusion}`); - } else if (!!updatatable.issue) { - // this updatatable updates a label + await this.workspaceDB.trace({span}).markUpdatableResolved(updatable.id); + log.info(`Resolved updatable. Marked check on ${updatable.contextUrl} as ${conclusion}`); + } else if (!!updatable.issue) { + // this updatable updates a label log.debug("Update label on a PR - we're not using this yet"); } } catch (err) { diff --git a/components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts b/components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts deleted file mode 100644 index 06b075bd521631..00000000000000 --- a/components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright (c) 2021 Gitpod GmbH. All rights reserved. - * Licensed under the Gitpod Enterprise Source Code License, - * See License.enterprise.txt in the project root folder. - */ - -import { User, WorkspaceContext, StartPrebuildContext, CommitContext, ContextURL } from "@gitpod/gitpod-protocol"; -import { inject, injectable } from "inversify"; -import { URL } from "url"; -import { Config } from '../../../src/config'; -import { HostContextProvider } from "../../../src/auth/host-context-provider"; -import { IPrefixContextParser } from "../../../src/workspace/context-parser"; - -@injectable() -export class StartIncrementalPrebuildContextParser implements IPrefixContextParser { - @inject(Config) protected readonly config: Config; - @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; - static PREFIX = ContextURL.INCREMENTAL_PREBUILD_PREFIX + '/'; - - findPrefix(user: User, context: string): string | undefined { - if (context.startsWith(StartIncrementalPrebuildContextParser.PREFIX)) { - return StartIncrementalPrebuildContextParser.PREFIX; - } - } - - public async handle(user: User, prefix: string, context: WorkspaceContext): Promise { - if (!CommitContext.is(context)) { - throw new Error("can only start incremental prebuilds on a commit context") - } - - const host = new URL(context.repository.cloneUrl).hostname; - const hostContext = this.hostContextProvider.get(host); - const maxDepth = this.config.incrementalPrebuilds.commitHistory; - const result: StartPrebuildContext = { - title: `Prebuild of "${context.title}"`, - actual: context, - commitHistory: await (hostContext?.contextParser?.fetchCommitHistory({}, user, context.repository.cloneUrl, context.revision, maxDepth) || []) - }; - return result; - } - -} \ No newline at end of file diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index defe22809fa153..0fa1797cf04a60 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -661,18 +661,19 @@ export class GitpodServerEEImpl extends GitpodServerImpl { return; } + const commitSHAs = CommitContext.computeHash(context); + const logCtx: LogContext = { userId: user.id }; const cloneUrl = context.repository.cloneUrl; - // Note: findPrebuiltWorkspaceByCommit always returns the last triggered prebuild (so, if you re-trigger a prebuild, the newer one will always be used here) - const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision); - const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace }; + const prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs); + const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace }; log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); if (!prebuiltWorkspace) { return; } if (prebuiltWorkspace.state === 'available') { - log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${context.revision}`, logPayload); + log.info(logCtx, `Found prebuilt workspace for ${cloneUrl}:${commitSHAs}`, logPayload); const result: PrebuiltWorkspaceContext = { title: context.title, originalContext: context, @@ -752,7 +753,7 @@ export class GitpodServerEEImpl extends GitpodServerImpl { } else { result.runningWorkspacePrebuild!.starting = 'running'; } - log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${context.revision}`, logPayload); + log.info(logCtx, `Found prebuilding (starting=${result.runningWorkspacePrebuild!.starting}) workspace for ${cloneUrl}:${commitSHAs}`, logPayload); return result; } } @@ -766,27 +767,6 @@ export class GitpodServerEEImpl extends GitpodServerImpl { await this.licenseEvaluator.reloadLicense(); } - // TODO(gpl) This is not part of our API interface, nor can I find any clients. Remove or re-surrect? - // async getLicenseInfo(ctx: TraceContext): Promise { - // const user = this.checkAndBlockUser("getLicenseInfo"); - - // const { key } = await this.licenseKeySource.getKey(); - // const { validUntil, seats } = this.licenseEvaluator.inspect(); - // const { valid } = this.licenseEvaluator.validate(); - - // const isAdmin = this.authorizationService.hasPermission(user, Permission.ADMIN_API); - - // return { - // isAdmin, - // licenseInfo: { - // key: isAdmin ? key : "REDACTED", - // seats, - // valid, - // validUntil - // } - // }; - // } - async licenseIncludesFeature(ctx: TraceContext, licenseFeature: LicenseFeature): Promise { traceAPIParams(ctx, { licenseFeature }); @@ -1536,11 +1516,8 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const context = await this.contextParser.handle(ctx, user, contextURL) as CommitContext; const prebuild = await this.prebuildManager.startPrebuild(ctx, { - contextURL, - cloneURL: project.cloneUrl, - commit: context.revision, + context, user, - branch: branchDetails[0].name, project }); diff --git a/components/server/ee/src/workspace/workspace-factory.ts b/components/server/ee/src/workspace/workspace-factory.ts index e33b9331e94d5e..9f29405bcc6178 100644 --- a/components/server/ee/src/workspace/workspace-factory.ts +++ b/components/server/ee/src/workspace/workspace-factory.ts @@ -8,14 +8,13 @@ import { v4 as uuidv4 } from 'uuid'; import { WorkspaceFactory } from "../../../src/workspace/workspace-factory"; import { injectable, inject } from "inversify"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; -import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, Project, PrebuiltWorkspace } from "@gitpod/gitpod-protocol"; +import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig, PrebuiltWorkspace, WorkspaceConfig, WorkspaceImageSource } from "@gitpod/gitpod-protocol"; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { LicenseEvaluator } from '@gitpod/licensor/lib'; import { Feature } from '@gitpod/licensor/lib/api'; import { ResponseError } from 'vscode-jsonrpc'; import { ErrorCodes } from '@gitpod/gitpod-protocol/lib/messaging/error'; import { HostContextProvider } from '../../../src/auth/host-context-provider'; -import { RepoURL } from '../../../src/repohost'; @injectable() export class WorkspaceFactoryEE extends WorkspaceFactory { @@ -51,7 +50,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { const { project, branch } = context; const commitContext: CommitContext = context.actual; - const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, commitContext.revision); + const existingPWS = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, CommitContext.computeHash(commitContext)); if (existingPWS) { const wsInstance = await this.db.trace({span}).findRunningInstance(existingPWS.buildWorkspaceId); if (wsInstance) { @@ -62,62 +61,28 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { const { config } = await this.configProvider.fetchConfig({span}, user, context.actual); const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config); - // Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on. + // Walk back the last prebuilds and check if they are valid ancestor. let ws; - for (const parent of (context.commitHistory || [])) { - const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent); - if (!parentPrebuild) { - continue; + if (context.commitHistory && context.commitHistory.length > 0) { + const recentPrebuilds = await this.db.trace({span}).findPrebuildsWithWorkpace(commitContext.repository.cloneUrl); + const match = recentPrebuilds.find(pb => this.isGoodBaseforIncrementalPrebuild(context, config, imageSource, pb.prebuild, pb.workspace)); + if (match) { + const incrementalPrebuildContext: PrebuiltWorkspaceContext = { + title: `Incremental prebuild of "${commitContext.title}"`, + originalContext: commitContext, + prebuiltWorkspace: match.prebuild, + } + ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL); + // Overwrite the config from the parent prebuild: + // `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild. + // Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now. + // However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...) + // To fix this, we overwrite the new prebuild's config with the most-recently fetched config. + // See also: https://github.com/gitpod-io/gitpod/issues/7475 + //TODO(sven) doing side effects on objects back and forth is complicated and error-prone. We should rather make sure we pass in the config when creating the prebuiltWorkspace. + ws.config = config; } - if (parentPrebuild.state !== 'available') { - continue; - } - log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild); - const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId); - if (!buildWorkspace) { - continue; - } - if (!!buildWorkspace.basedOnPrebuildId) { - continue; - } - if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) { - log.debug(`Skipping parent prebuild: Outdated image`, { - imageSource, - parentImageSource: buildWorkspace.imageSource, - }); - continue; - } - const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks - .map(task => Object.keys(task) - .filter(key => ['before', 'init', 'prebuild'].includes(key)) - // @ts-ignore - .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {})) - .filter(task => Object.keys(task).length > 0)); - const prebuildTasks = filterPrebuildTasks(config.tasks); - const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks); - if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) { - log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, { - prebuildTasks, - parentPrebuildTasks, - }); - continue; - } - const incrementalPrebuildContext: PrebuiltWorkspaceContext = { - title: `Incremental prebuild of "${commitContext.title}"`, - originalContext: commitContext, - prebuiltWorkspace: parentPrebuild, - } - ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL); - // Overwrite the config from the parent prebuild: - // `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild. - // Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now. - // However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...) - // To fix this, we overwrite the new prebuild's config with the most-recently fetched config. - // See also: https://github.com/gitpod-io/gitpod/issues/7475 - ws.config = config; - break; } - if (!ws) { // No suitable parent prebuild was found -- create a (fresh) full prebuild. ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL); @@ -130,21 +95,13 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { id: uuidv4(), buildWorkspaceId: ws.id, cloneURL: commitContext.repository.cloneUrl, - commit: commitContext.revision, + commit: CommitContext.computeHash(commitContext), state: "queued", creationTime: new Date().toISOString(), projectId: ws.projectId, branch }); - if (project) { - // do not await - this.storePrebuildInfo(ctx, project, pws, ws, user).catch(err => { - log.error(`failed to store prebuild info`, err); - TraceContext.setError({span}, err); - }); - } - log.debug({ userId: user.id, workspaceId: ws.id }, `Registered workspace prebuild: ${pws.id} for ${commitContext.repository.cloneUrl}:${commitContext.revision}`); return ws; @@ -156,43 +113,67 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { } } - protected async storePrebuildInfo(ctx: TraceContext, project: Project, pws: PrebuiltWorkspace, ws: Workspace, user: User) { - const span = TraceContext.startSpan("storePrebuildInfo", ctx); - const { userId, teamId, name: projectName, id: projectId } = project; - const parsedUrl = RepoURL.parseRepoUrl(project.cloneUrl); - if (!parsedUrl) { - return; + private async isGoodBaseforIncrementalPrebuild(context: StartPrebuildContext, config: WorkspaceConfig, imageSource: WorkspaceImageSource, candidatePrebuild: PrebuiltWorkspace, candidate: Workspace) { + if (!context.commitHistory || context.commitHistory.length === 0) { + return false; + } + if (!CommitContext.is(candidate.context)) { + return false; + } + + // we are only considering available prebuilds + if (candidatePrebuild.state !== 'available') { + return false; } - const { owner, repo, host } = parsedUrl; - const repositoryProvider = this.hostContextProvider.get(host)?.services?.repositoryProvider; - if (!repositoryProvider) { - return; + + // we are only considering full prebuilds + if (!!candidate.basedOnPrebuildId) { + return false; } - const commit = await repositoryProvider.getCommitInfo(user, owner, repo, pws.commit); - if (!commit) { - return; + + const candidateCtx = candidate.context; + if (candidateCtx.additionalRepositoryCheckoutInfo?.length !== context.additionalRepositoryCommitHistories?.length) { + // different number of repos + return false; + } + + if (!context.commitHistory.some(sha => sha === candidateCtx.revision)) { + return false; + } + + // check the commits are included in the commit history + for (const subRepo of candidateCtx.additionalRepositoryCheckoutInfo || []) { + const matchIngRepo = context.additionalRepositoryCommitHistories?.find(repo => repo.cloneUrl === subRepo.repository.cloneUrl); + if (!matchIngRepo || !matchIngRepo.commitHistory.some(sha => sha === subRepo.revision)) { + return false; + } + } + + // ensure the image source hasn't changed + if (JSON.stringify(imageSource) !== JSON.stringify(candidate.imageSource)) { + log.debug(`Skipping parent prebuild: Outdated image`, { + imageSource, + parentImageSource: candidate.imageSource, + }); + return false; + } + + // ensure the tasks haven't changed + const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks + .map(task => Object.keys(task) + .filter(key => ['before', 'init', 'prebuild'].includes(key)) + // @ts-ignore + .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {})) + .filter(task => Object.keys(task).length > 0)); + const prebuildTasks = filterPrebuildTasks(config.tasks); + const parentPrebuildTasks = filterPrebuildTasks(candidate.config.tasks); + if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) { + log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, { + prebuildTasks, + parentPrebuildTasks, + }); + return false; } - await this.db.trace({span}).storePrebuildInfo({ - id: pws.id, - buildWorkspaceId: pws.buildWorkspaceId, - basedOnPrebuildId: ws.basedOnPrebuildId, - teamId, - userId, - projectName, - projectId, - startedAt: pws.creationTime, - startedBy: "", // TODO - startedByAvatar: "", // TODO - cloneUrl: pws.cloneURL, - branch: pws.branch || "unknown", - changeAuthor: commit.author, - changeAuthorAvatar: commit.authorAvatarUrl, - changeDate: commit.authorDate || "", - changeHash: commit.sha, - changeTitle: commit.commitMessage, - // changePR - changeUrl: ws.contextURL, - }); } protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise { diff --git a/components/server/install-gh-app.sh b/components/server/install-gh-app.sh index e42c3158aa7912..33ab8f0f17ff35 100755 --- a/components/server/install-gh-app.sh +++ b/components/server/install-gh-app.sh @@ -3,7 +3,6 @@ # This script will patch the servers config map, install the app cert and restart the server components # It is best to add the envs to your environment variables using `gp env GH_APP_ID=....` and `gp env GH_APP_KEY="..."`. # See https://www.notion.so/gitpod/How-to-deploy-a-PR-with-a-working-GitHub-App-integration-d297a1ef2f7b4b3aa8483b2ae9b47da2 (internal) for more details. - # GH_APP_ID= # GH_APP_KEY="-----BEGIN RSA PRIVATE KEY----- # ... @@ -32,6 +31,7 @@ kubectl get cm server-config -o yaml > server-config.yml perl -0777 -i.original -pe "s/\"githubApp\":.+?\}/$LINE/igs" server-config.yml kubectl apply -f server-config.yml rm server-config.yml +rm server-config.yml.original echo 'updating the secret' kubectl delete secret server-github-app-cert diff --git a/components/server/src/auth/host-context-provider.ts b/components/server/src/auth/host-context-provider.ts index 355e254f2dc5f7..b5ee5544abfc8a 100644 --- a/components/server/src/auth/host-context-provider.ts +++ b/components/server/src/auth/host-context-provider.ts @@ -6,6 +6,8 @@ import { HostContext } from "./host-context"; import { AuthProviderParams } from "./auth-provider"; +import { CommitInfo, User } from "@gitpod/gitpod-protocol"; +import { RepoURL } from "../repohost"; export const HostContextProvider = Symbol("HostContextProvider"); @@ -16,6 +18,15 @@ export interface HostContextProvider { findByAuthProviderId(authProviderId: string): HostContext | undefined; } +export async function getCommitInfo(hostContextProvider: HostContextProvider, user: User, repoURL: string, commitSHA: string) { + const parsedRepo = RepoURL.parseRepoUrl(repoURL)!; + const hostCtx = hostContextProvider.get(parsedRepo.host); + let commitInfo: CommitInfo | undefined; + if (hostCtx?.services?.repositoryProvider) { + commitInfo = await hostCtx?.services?.repositoryProvider.getCommitInfo(user, parsedRepo.owner, parsedRepo.repo, commitSHA); + } + return commitInfo; +} export const HostContextProviderFactory = Symbol("HostContextProviderFactory"); diff --git a/components/server/src/auth/resource-access.ts b/components/server/src/auth/resource-access.ts index 59625822534287..26d32a580489f8 100644 --- a/components/server/src/auth/resource-access.ts +++ b/components/server/src/auth/resource-access.ts @@ -4,8 +4,10 @@ * See License-AGPL.txt in the project root for license information. */ -import { CommitContext, ContextURL, GitpodToken, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { CommitContext, GitpodToken, Repository, Snapshot, Team, TeamMemberInfo, Token, User, UserEnvVar, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; import { UnauthorizedError } from "../errors"; +import { RepoURL } from "../repohost"; import { HostContextProvider } from "./host-context-provider"; declare var resourceInstance: GuardedResource; @@ -477,27 +479,40 @@ export class RepositoryResourceGuard implements ResourceAccessGuard { // Check if user has at least read access to the repository const workspace = resource.kind === 'snapshot' ? resource.workspace : resource.subject; - const contextURL = ContextURL.getNormalizedURL(workspace); - if (!contextURL) { - throw new Error(`unable to parse ContextURL: ${contextURL}`); - } - const hostContext = this.hostContextProvider.get(contextURL.hostname); - if (!hostContext) { - throw new Error(`no HostContext found for hostname: ${contextURL.hostname}`); - } - const { authProvider } = hostContext; - const identity = User.getIdentity(this.user, authProvider.authProviderId); - if (!identity) { - throw UnauthorizedError.create(contextURL.hostname, authProvider.info.requirements?.default || [], "missing-identity"); - } - const { services } = hostContext; - if (!services) { - throw new Error(`no services found in HostContext for hostname: ${contextURL.hostname}`); - } - if (!CommitContext.is(workspace.context)) { - return false; + const repos: Repository[] = []; + if (CommitContext.is(workspace.context)) { + repos.push(workspace.context.repository); + for (const additionalRepo of workspace.context.additionalRepositoryCheckoutInfo || []) { + repos.push(additionalRepo.repository); + } } - const { owner, name: repo } = workspace.context.repository; - return services.repositoryProvider.hasReadAccess(this.user, owner, repo); + const result = await Promise.all( + repos.map( + async repo => { + const repoUrl = RepoURL.parseRepoUrl(repo.cloneUrl); + if (!repoUrl) { + log.error("Cannot parse repoURL", {repo}) + return false; + } + const hostContext = this.hostContextProvider.get(repoUrl.host) + if (!hostContext) { + throw new Error(`no HostContext found for hostname: ${repoUrl.host}`); + } + const { authProvider } = hostContext; + const identity = User.getIdentity(this.user, authProvider.authProviderId); + if (!identity) { + throw UnauthorizedError.create(repoUrl!.host, authProvider.info.requirements?.default || [], "missing-identity"); + } + const { services } = hostContext; + if (!services) { + throw new Error(`no services found in HostContext for hostname: ${repoUrl.host}`); + } + if (!CommitContext.is(workspace.context)) { + return false; + } + return services.repositoryProvider.hasReadAccess(this.user, repo.owner, repo.name); + } + )); + return result.every(b => b); } } \ No newline at end of file diff --git a/components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts b/components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts index e02c88912f6ffa..09c16a6fa233b1 100644 --- a/components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts +++ b/components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts @@ -107,4 +107,7 @@ export class BitbucketServerRepositoryProvider implements RepositoryProvider { return false; } + async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise { + return []; + } } diff --git a/components/server/src/bitbucket/bitbucket-context-parser.spec.ts b/components/server/src/bitbucket/bitbucket-context-parser.spec.ts index 9d4ef5ad832ac3..182b8778a5f6a4 100644 --- a/components/server/src/bitbucket/bitbucket-context-parser.spec.ts +++ b/components/server/src/bitbucket/bitbucket-context-parser.spec.ts @@ -471,13 +471,6 @@ class TestBitbucketContextParser { "title": "gitpod/integration-tests-forked-repository - master" }) } - - @test public async testFetchCommitHistory() { - const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100); - expect(result).to.deep.equal([ - 'da2119f51b0e744cb6b36399f8433b477a4174ef', - ]) - } } module.exports = new TestBitbucketContextParser(); diff --git a/components/server/src/bitbucket/bitbucket-context-parser.ts b/components/server/src/bitbucket/bitbucket-context-parser.ts index 36d36205a157ae..44fa4545ecc5bc 100644 --- a/components/server/src/bitbucket/bitbucket-context-parser.ts +++ b/components/server/src/bitbucket/bitbucket-context-parser.ts @@ -267,31 +267,4 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo return result; } - public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { - const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx); - try { - // TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled. - // The additional property 'page' may be helfpul. - const api = await this.api(user); - const { owner, repoName } = await this.parseURL(user, contextUrl); - const result = await api.repositories.listCommitsAt({ - workspace: owner, - repo_slug: repoName, - revision: sha, - pagelen: maxDepth, - }); - - const commits = result.data.values?.slice(1); - if (!commits) { - return undefined; - } - return commits.map((v: Schema.Commit) => v.hash!); - } catch (e) { - span.log({ error: e }); - log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e); - throw e; - } finally { - span.finish(); - } - } } diff --git a/components/server/src/bitbucket/bitbucket-repository-provider.spec.ts b/components/server/src/bitbucket/bitbucket-repository-provider.spec.ts index 89fa8a32ce353f..2d5be995a77812 100644 --- a/components/server/src/bitbucket/bitbucket-repository-provider.spec.ts +++ b/components/server/src/bitbucket/bitbucket-repository-provider.spec.ts @@ -72,6 +72,12 @@ class TestBitbucketRepositoryProvider { }); } + @test public async testFetchCommitHistory() { + const result = await this.repoProvider.getCommitHistory(this.user, 'gitpod', 'integration-tests', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100); + expect(result).to.deep.equal([ + 'da2119f51b0e744cb6b36399f8433b477a4174ef', + ]) + } } module.exports = new TestBitbucketRepositoryProvider(); diff --git a/components/server/src/bitbucket/bitbucket-repository-provider.ts b/components/server/src/bitbucket/bitbucket-repository-provider.ts index 8f0f5ff7c4ee42..dde1aa40fb2f85 100644 --- a/components/server/src/bitbucket/bitbucket-repository-provider.ts +++ b/components/server/src/bitbucket/bitbucket-repository-provider.ts @@ -5,6 +5,7 @@ */ import { Branch, CommitInfo, Repository, User } from "@gitpod/gitpod-protocol"; +import { Schema } from "bitbucket"; import { inject, injectable } from 'inversify'; import { URL } from "url"; import { RepoURL } from '../repohost/repo-url'; @@ -108,4 +109,22 @@ export class BitbucketRepositoryProvider implements RepositoryProvider { // FIXME(janx): Not implemented yet return false; } + + public async getCommitHistory(user: User, owner: string, repo: string, revision: string, maxDepth: number = 100): Promise { + const api = await this.apiFactory.create(user); + // TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled. + // The additional property 'page' may be helfpul. + const result = await api.repositories.listCommitsAt({ + workspace: owner, + repo_slug: repo, + revision: revision, + pagelen: maxDepth, + }); + + const commits = result.data.values?.slice(1); + if (!commits) { + return []; + } + return commits.map((v: Schema.Commit) => v.hash!); + } } diff --git a/components/server/src/config/configuration-service.ts b/components/server/src/config/configuration-service.ts index 4c66fbc74bfc02..ed8a79a2d4c015 100644 --- a/components/server/src/config/configuration-service.ts +++ b/components/server/src/config/configuration-service.ts @@ -9,21 +9,18 @@ import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { inject, injectable } from "inversify"; import { HostContextProvider } from "../auth/host-context-provider"; import { FileProvider } from "../repohost"; -import { ContextParser } from "../workspace/context-parser-service"; import { ConfigInferrer } from "./config-inferrer"; - @injectable() export class ConfigurationService { - @inject(ContextParser) protected contextParser: ContextParser; @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; // a static cache used to prefetch inferrer related files in parallel in advance private requestedPaths = new Set(); - async guessRepositoryConfiguration(ctx: TraceContext, user: User, contextURLOrContext: string | CommitContext): Promise { - const { fileProvider, commitContext } = await this.getRepositoryFileProviderAndCommitContext(ctx, user, contextURLOrContext); + async guessRepositoryConfiguration(ctx: TraceContext, user: User, context: CommitContext): Promise { + const { fileProvider, commitContext } = await this.getRepositoryFileProviderAndCommitContext(ctx, user, context); const cache: { [path: string]: Promise } = {}; const readFile = async (path: string) => { if (path in cache) { @@ -56,20 +53,13 @@ ${configString} `; } - async fetchRepositoryConfiguration(ctx: TraceContext, user: User, contextURL: string): Promise { - const { fileProvider, commitContext } = await this.getRepositoryFileProviderAndCommitContext(ctx, user, contextURL); + async fetchRepositoryConfiguration(ctx: TraceContext, user: User, context: CommitContext): Promise { + const { fileProvider, commitContext } = await this.getRepositoryFileProviderAndCommitContext(ctx, user, context); const configString = await fileProvider.getGitpodFileContent(commitContext, user); return configString; } - protected async getRepositoryFileProviderAndCommitContext(ctx: TraceContext, user: User, contextURLOrContext: string | CommitContext): Promise<{fileProvider: FileProvider, commitContext: CommitContext}> { - let commitContext: CommitContext; - if (typeof contextURLOrContext === 'string') { - const normalizedContextUrl = this.contextParser.normalizeContextURL(contextURLOrContext); - commitContext = (await this.contextParser.handle(ctx, user, normalizedContextUrl)) as CommitContext; - } else { - commitContext = contextURLOrContext; - } + protected async getRepositoryFileProviderAndCommitContext(ctx: TraceContext, user: User, commitContext: CommitContext): Promise<{fileProvider: FileProvider, commitContext: CommitContext}> { const { host } = commitContext.repository; const hostContext = this.hostContextProvider.get(host); if (!hostContext || !hostContext.services) { diff --git a/components/server/src/github/github-auth-provider.ts b/components/server/src/github/github-auth-provider.ts index 08fe4748c347bc..4a07cf0c99ab15 100644 --- a/components/server/src/github/github-auth-provider.ts +++ b/components/server/src/github/github-auth-provider.ts @@ -69,7 +69,7 @@ export class GitHubAuthProvider extends GenericAuthProvider { timeout: 5000, }, userAgent: this.USER_AGENT, - baseUrl: this.baseURL + baseUrl: this.baseURL, }); const fetchCurrentUser = async () => { const response = await api.users.getAuthenticated(); diff --git a/components/server/src/github/github-context-parser.spec.ts b/components/server/src/github/github-context-parser.spec.ts index 5141511b80236e..75381308d5bed8 100644 --- a/components/server/src/github/github-context-parser.spec.ts +++ b/components/server/src/github/github-context-parser.spec.ts @@ -577,14 +577,5 @@ class TestGithubContextParser { } ) } - - @test public async testFetchCommitHistory() { - const result = await this.parser.fetchCommitHistory({}, this.user, 'https://github.com/gitpod-io/gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100); - expect(result).to.deep.equal([ - '506e5aed317f28023994ecf8ca6ed91430e9c1a4', - 'f5b041513bfab914b5fbf7ae55788d9835004d76', - ]) - } - } module.exports = new TestGithubContextParser() // Only to circumvent no usage warning :-/ \ No newline at end of file diff --git a/components/server/src/github/github-context-parser.ts b/components/server/src/github/github-context-parser.ts index c67746e6409077..1ca4279d1363bf 100644 --- a/components/server/src/github/github-context-parser.ts +++ b/components/server/src/github/github-context-parser.ts @@ -426,55 +426,4 @@ export class GithubContextParser extends AbstractContextParser implements IConte `; } - public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { - const span = TraceContext.startSpan("GithubContextParser.fetchCommitHistory", ctx); - - try { - if (sha.length != 40) { - throw new Error(`Invalid commit ID ${sha}.`); - } - - // TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled. - // These additional history properties may be helfpul: - // totalCount, - // pageInfo { - // haxNextPage, - // }, - const { owner, repoName } = await this.parseURL(user, contextUrl); - const result: any = await this.githubQueryApi.runQuery(user, ` - query { - repository(name: "${repoName}", owner: "${owner}") { - object(oid: "${sha}") { - ... on Commit { - history(first: ${maxDepth}) { - edges { - node { - oid - } - } - } - } - } - } - } - `); - span.log({"request.finished": ""}); - - if (result.data.repository === null) { - throw await NotFoundError.create(await this.tokenHelper.getCurrentToken(user), user, this.config.host, owner, repoName); - } - - const commit = result.data.repository.object; - if (commit === null) { - throw new Error(`Couldn't find commit ${sha} in repository ${owner}/${repoName}.`); - } - - return commit.history.edges.slice(1).map((e: any) => e.node.oid) || []; - } catch (e) { - span.log({"error": e}); - throw e; - } finally { - span.finish(); - } - } } diff --git a/components/server/src/github/github-repository-provider.spec.ts b/components/server/src/github/github-repository-provider.spec.ts new file mode 100644 index 00000000000000..c3cfd1d60132e4 --- /dev/null +++ b/components/server/src/github/github-repository-provider.spec.ts @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2020 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +// Use asyncIterators with es2015 +if (typeof (Symbol as any).asyncIterator === 'undefined') { + (Symbol as any).asyncIterator = Symbol.asyncIterator || Symbol('asyncIterator'); +} +import "reflect-metadata"; + +import { suite, test, timeout, retries } from "mocha-typescript"; +import * as chai from 'chai'; +const expect = chai.expect; + +import { GitHubGraphQlEndpoint, GitHubRestApi } from './api'; +import { User } from "@gitpod/gitpod-protocol"; +import { ContainerModule, Container } from "inversify"; +import { Config } from "../config"; +import { DevData } from "../dev/dev-data"; +import { AuthProviderParams } from "../auth/auth-provider"; +import { TokenProvider } from "../user/token-provider"; +import { GitHubTokenHelper } from "./github-token-helper"; +import { HostContextProvider } from "../auth/host-context-provider"; +import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if"; +import { GithubRepositoryProvider } from "./github-repository-provider"; + +@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITHUB")) +class TestGithubContextRepositoryProvider { + + protected provider: GithubRepositoryProvider; + protected user: User; + + public before() { + const container = new Container(); + container.load(new ContainerModule((bind, unbind, isBound, rebind) => { + bind(Config).toConstantValue({ + // meant to appease DI, but Config is never actually used here + }); + bind(GithubRepositoryProvider).toSelf().inSingletonScope(); + bind(GitHubRestApi).toSelf().inSingletonScope(); + bind(GitHubGraphQlEndpoint).toSelf().inSingletonScope(); + bind(AuthProviderParams).toConstantValue(TestGithubContextRepositoryProvider.AUTH_HOST_CONFIG); + bind(GitHubTokenHelper).toSelf().inSingletonScope(); + bind(TokenProvider).toConstantValue({ + getTokenForHost: async (user: User, host: string) => { + return DevData.createGitHubTestToken(); + } + }); + bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider()); + })); + this.provider = container.get(GithubRepositoryProvider); + this.user = DevData.createTestUser(); + } + + static readonly AUTH_HOST_CONFIG: Partial = { + id: "Public-GitHub", + type: "GitHub", + verified: true, + description: "", + icon: "", + host: "github.com", + oauth: "not-used" as any + } + + @test public async testFetchCommitHistory() { + const result = await this.provider.getCommitHistory(this.user, 'gitpod-io', 'gitpod-test-repo', '409ac2de49a53d679989d438735f78204f441634', 100); + expect(result).to.deep.equal([ + '506e5aed317f28023994ecf8ca6ed91430e9c1a4', + 'f5b041513bfab914b5fbf7ae55788d9835004d76', + ]) + } + +} +module.exports = new TestGithubContextRepositoryProvider() // Only to circumvent no usage warning :-/ \ No newline at end of file diff --git a/components/server/src/github/github-repository-provider.ts b/components/server/src/github/github-repository-provider.ts index f0a11e0ce24c54..80a3df835d2417 100644 --- a/components/server/src/github/github-repository-provider.ts +++ b/components/server/src/github/github-repository-provider.ts @@ -103,8 +103,58 @@ export class GithubRepositoryProvider implements RepositoryProvider { } async getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise { - const commit = await this.github.getCommit(user, { repo, owner, ref }); - return commit; + try { + return await this.github.getCommit(user, { repo, owner, ref }); + } catch (error) { + console.error(error); + return undefined; + } + } + + public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise { + try { + if (ref.length != 40) { + throw new Error(`Invalid commit ID ${ref}.`); + } + + // TODO(janx): To get more results than GitHub API's max page size (seems to be 100), pagination should be handled. + // These additional history properties may be helfpul: + // totalCount, + // pageInfo { + // haxNextPage, + // }, + const result: any = await this.githubQueryApi.runQuery(user, ` + query { + repository(name: "${repo}", owner: "${owner}") { + object(oid: "${ref}") { + ... on Commit { + history(first: ${maxDepth}) { + edges { + node { + oid + } + } + } + } + } + } + } + `); + + if (result.data.repository === null) { + throw new Error(`couldn't find repository ${owner}/${repo} on ${this.github.baseURL}`); + } + + const commit = result.data.repository.object; + if (commit === null) { + throw new Error(`Couldn't find commit ${ref} in repository ${owner}/${repo}.`); + } + + return commit.history.edges.slice(1).map((e: any) => e.node.oid) || []; + } catch (e) { + console.error(e); + return []; + } } async getUserRepos(user: User): Promise { diff --git a/components/server/src/gitlab/gitlab-context-parser.spec.ts b/components/server/src/gitlab/gitlab-context-parser.spec.ts index c5a4211fbb3f35..bcce61257411f8 100644 --- a/components/server/src/gitlab/gitlab-context-parser.spec.ts +++ b/components/server/src/gitlab/gitlab-context-parser.spec.ts @@ -604,14 +604,6 @@ class TestGitlabContextParser { }) } - @test public async testFetchCommitHistory() { - const result = await this.parser.fetchCommitHistory({}, this.user, 'https://gitlab.com/AlexTugarev/gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100); - expect(result).to.deep.equal([ - '4447fbc4d46e6fd1ee41fb1b992702521ae078eb', - 'f2d9790f2752a794517b949c65a773eb864844cd', - ]) - } - } module.exports = new TestGitlabContextParser(); diff --git a/components/server/src/gitlab/gitlab-context-parser.ts b/components/server/src/gitlab/gitlab-context-parser.ts index 6e98d524181d08..ee98342827a436 100644 --- a/components/server/src/gitlab/gitlab-context-parser.ts +++ b/components/server/src/gitlab/gitlab-context-parser.ts @@ -394,23 +394,4 @@ export class GitlabContextParser extends AbstractContextParser implements IConte }; } - public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise { - // TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled. - const { owner, repoName } = await this.parseURL(user, contextUrl); - const projectId = `${owner}/${repoName}`; - const result = await this.gitlabApi.run(user, async g => { - return g.Commits.all(projectId, { - ref_name: sha, - per_page: maxDepth, - page: 1, - }); - }); - if (GitLab.ApiError.is(result)) { - if (result.message === 'GitLab responded with code 404') { - throw new Error(`Couldn't find commit #${sha} in repository ${projectId}.`); - } - throw result; - } - return result.slice(1).map((c: GitLab.Commit) => c.id); - } } \ No newline at end of file diff --git a/components/server/src/gitlab/gitlab-repository-provider.spec.ts b/components/server/src/gitlab/gitlab-repository-provider.spec.ts new file mode 100644 index 00000000000000..ea04a0ae0165bc --- /dev/null +++ b/components/server/src/gitlab/gitlab-repository-provider.spec.ts @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { User } from "@gitpod/gitpod-protocol"; +import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if"; +import { expect } from "chai"; +import { Container, ContainerModule } from "inversify"; +import { suite, retries, test, timeout } from "mocha-typescript"; +import { AuthProviderParams } from "../auth/auth-provider"; +import { HostContextProvider } from "../auth/host-context-provider"; +import { DevData } from "../dev/dev-data"; +import { TokenProvider } from "../user/token-provider"; +import { GitLabApi } from "./api"; +import { GitlabContextParser } from "./gitlab-context-parser"; +import { GitlabRepositoryProvider } from "./gitlab-repository-provider"; +import { GitLabTokenHelper } from "./gitlab-token-helper"; + +@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITLAB")) +class TestGitlabRepositoryProvider { + + static readonly AUTH_HOST_CONFIG: Partial = { + id: "Public-GitLab", + type: "GitLab", + verified: true, + description: "", + icon: "", + host: "gitlab.com", + } + + protected repositoryProvider: GitlabRepositoryProvider; + protected user: User; + + public before() { + const container = new Container(); + container.load(new ContainerModule((bind, unbind, isBound, rebind) => { + bind(GitlabContextParser).toSelf().inSingletonScope(); + bind(GitLabApi).toSelf().inSingletonScope(); + bind(AuthProviderParams).toConstantValue(TestGitlabRepositoryProvider.AUTH_HOST_CONFIG); + bind(GitLabTokenHelper).toSelf().inSingletonScope(); + bind(TokenProvider).toConstantValue({ + getTokenForHost: async () => DevData.createGitlabTestToken(), + getFreshPortAuthenticationToken: async (user: User, workspaceId: string) => DevData.createPortAuthTestToken(workspaceId), + }); + bind(HostContextProvider).toConstantValue(DevData.createDummyHostContextProvider()); + bind(GitlabRepositoryProvider).toSelf().inSingletonScope(); + })); + this.repositoryProvider = container.get(GitlabRepositoryProvider); + this.user = DevData.createTestUser(); + } + + @test public async testFetchCommitHistory() { + const result = await this.repositoryProvider.getCommitHistory(this.user, 'AlexTugarev', 'gp-test', '80948e8cc8f0e851e89a10bc7c2ee234d1a5fbe7', 100); + expect(result).to.deep.equal([ + '4447fbc4d46e6fd1ee41fb1b992702521ae078eb', + 'f2d9790f2752a794517b949c65a773eb864844cd', + ]) + } + +} + +module.exports = new TestGitlabRepositoryProvider(); \ No newline at end of file diff --git a/components/server/src/gitlab/gitlab-repository-provider.ts b/components/server/src/gitlab/gitlab-repository-provider.ts index 154f1bc82969a7..16b0e544500ddf 100644 --- a/components/server/src/gitlab/gitlab-repository-provider.ts +++ b/components/server/src/gitlab/gitlab-repository-provider.ts @@ -113,4 +113,23 @@ export class GitlabRepositoryProvider implements RepositoryProvider { return false; } } + + public async getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number = 100): Promise { + // TODO(janx): To get more results than GitLab API's max per_page (seems to be 100), pagination should be handled. + const projectId = `${owner}/${repo}`; + const result = await this.gitlab.run(user, async g => { + return g.Commits.all(projectId, { + ref_name: ref, + per_page: maxDepth, + page: 1, + }); + }); + if (GitLab.ApiError.is(result)) { + if (result.message === 'GitLab responded with code 404') { + throw new Error(`Couldn't find commit #${ref} in repository ${projectId}.`); + } + throw result; + } + return result.slice(1).map((c: GitLab.Commit) => c.id); + } } diff --git a/components/server/src/gitlab/gitlab-token-helper.ts b/components/server/src/gitlab/gitlab-token-helper.ts index 32311ab0e575a8..3c5334342e43f6 100644 --- a/components/server/src/gitlab/gitlab-token-helper.ts +++ b/components/server/src/gitlab/gitlab-token-helper.ts @@ -31,8 +31,8 @@ export class GitLabTokenHelper { if (this.containsScopes(token, requiredScopes)) { return token; } - } catch { - // no token + } catch (e) { + console.error(e); } if (requiredScopes.length === 0) { requiredScopes = GitLabScope.Requirements.DEFAULT diff --git a/components/server/src/projects/projects-service.ts b/components/server/src/projects/projects-service.ts index b0d7bf93ea7dfe..a4e4a5b40aec13 100644 --- a/components/server/src/projects/projects-service.ts +++ b/components/server/src/projects/projects-service.ts @@ -94,9 +94,7 @@ export class ProjectsService { changeHash: commit.sha, changeTitle: commit.commitMessage, changeAuthorAvatar: commit.authorAvatarUrl, - isDefault: repository.defaultBranch === branch.name, - changePR: "changePR", // todo: compute in repositoryProvider - changeUrl: "changeUrl", // todo: compute in repositoryProvider + isDefault: repository.defaultBranch === branch.name }); } result.sort((a, b) => (b.changeDate || "").localeCompare(a.changeDate || "")); diff --git a/components/server/src/repohost/repository-provider.ts b/components/server/src/repohost/repository-provider.ts index 48c056c19eca0b..3a739018efd871 100644 --- a/components/server/src/repohost/repository-provider.ts +++ b/components/server/src/repohost/repository-provider.ts @@ -15,4 +15,5 @@ export interface RepositoryProvider { getCommitInfo(user: User, owner: string, repo: string, ref: string): Promise; getUserRepos(user: User): Promise; hasReadAccess(user: User, owner: string, repo: string): Promise; + getCommitHistory(user: User, owner: string, repo: string, ref: string, maxDepth: number): Promise; } \ No newline at end of file diff --git a/components/server/src/workspace/context-parser-service.ts b/components/server/src/workspace/context-parser-service.ts index 7da56cd25bfd03..56f7f9ae3f5f05 100644 --- a/components/server/src/workspace/context-parser-service.ts +++ b/components/server/src/workspace/context-parser-service.ts @@ -4,17 +4,19 @@ * See License-AGPL.txt in the project root for license information. */ -import { WorkspaceContext, User } from "@gitpod/gitpod-protocol"; +import { WorkspaceContext, User, CommitContext, GitCheckoutInfo, PullRequestContext } from "@gitpod/gitpod-protocol"; import { injectable, multiInject, inject } from "inversify"; import { HostContextProvider } from "../auth/host-context-provider"; import { IPrefixContextParser, IContextParser } from "./context-parser"; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; +import { ConfigProvider, InvalidGitpodYMLError } from "./config-provider"; @injectable() export class ContextParser { @multiInject(IPrefixContextParser) protected readonly prefixParser: IPrefixContextParser[]; @multiInject(IContextParser) protected readonly contextParsers: IContextParser[]; @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; + @inject(ConfigProvider) protected readonly configProvider: ConfigProvider; protected get allContextParsers(): IContextParser[] { const result = [...this.contextParsers]; @@ -44,18 +46,9 @@ export class ContextParser { contextURL = this.normalizeContextURL(contextURL.substring(prefixResult.prefix.length)); } - for (const parser of this.allContextParsers) { - if (parser.canHandle(user, contextURL)) { - result = await parser.handle({ span }, user, contextURL); - break; - } - } - if (!result) { - throw new Error(`Couldn't parse context '${contextURL}'.`); - } + result = await this.internalHandleWithoutPrefix({ span }, user, contextURL); - // TODO: Make the parsers return the context with normalizedContextURL set - result.normalizedContextURL = contextURL; + result = await this.handleMultiRepositoryContext({ span }, user, result); if (prefixResult) { result = await prefixResult.parser.handle(user, prefixResult.prefix, result); @@ -70,6 +63,96 @@ export class ContextParser { return result; } + protected async internalHandleWithoutPrefix(ctx: TraceContext, user: User, nonPrefixedContextURL: string): Promise { + const span = TraceContext.startSpan("ContextParser.internalHandle", ctx); + try { + let result: WorkspaceContext | undefined; + + for (const parser of this.allContextParsers) { + if (parser.canHandle(user, nonPrefixedContextURL)) { + result = await parser.handle({ span }, user, nonPrefixedContextURL); + break; + } + } + if (!result) { + throw new Error(`Couldn't parse context '${nonPrefixedContextURL}'.`); + } + + // TODO: Make the parsers return the context with normalizedContextURL set + result.normalizedContextURL = nonPrefixedContextURL; + return result; + } finally { + span.finish(); + } + } + + protected buildUpstreamCloneUrl(context: CommitContext): string | undefined { + let upstreamCloneUrl: string | undefined = undefined; + if (PullRequestContext.is(context) && context.base) { + upstreamCloneUrl = context.base.repository.cloneUrl; + } else if (context.repository.fork) { + upstreamCloneUrl = context.repository.fork.parent.cloneUrl; + } + + if (context.repository.cloneUrl === upstreamCloneUrl) { + return undefined; + } + return upstreamCloneUrl; + } + + protected async handleMultiRepositoryContext(ctx: TraceContext, user: User, context: WorkspaceContext): Promise { + if (!CommitContext.is(context)) { + return context; + } + const span = TraceContext.startSpan("ContextParser.handleMultiRepositoryContext", ctx); + try { + let config = await this.configProvider.fetchConfig({ span }, user, context); + let mainRepoContext: WorkspaceContext | undefined; + if (config.config.mainConfiguration) { + mainRepoContext = await this.internalHandleWithoutPrefix({ span }, user, config.config.mainConfiguration); + if (!CommitContext.is(mainRepoContext)) { + throw new InvalidGitpodYMLError([`Cannot find main repository '${config.config.mainConfiguration}'.`]); + } + config = await this.configProvider.fetchConfig({ span }, user, mainRepoContext); + } + + if (config.config.additionalRepositories && config.config.additionalRepositories.length > 0) { + const subRepoCommits: GitCheckoutInfo[] = []; + for (const subRepo of config.config.additionalRepositories) { + let subContext = await this.internalHandleWithoutPrefix({ span }, user, subRepo.url) as CommitContext; + if (!CommitContext.is(subContext)) { + throw new InvalidGitpodYMLError([`Cannot find sub-repository '${subRepo.url}'.`]); + } + if (context.repository.cloneUrl === subContext.repository.cloneUrl) { + // if it's the repo from the original context we want to use that commit. + subContext = JSON.parse(JSON.stringify(context)); + } + + subRepoCommits.push({ + ... subContext, + checkoutLocation: (subRepo.checkoutLocation || subContext.repository.name), + upstreamRemoteURI: this.buildUpstreamCloneUrl(subContext), + // we want to create a local branch on all repos, in case it's a multi-repo change. If it's not there are no drawbacks anyway. + ref: context.ref, + refType: context.refType, + localBranch: context.localBranch + }); + } + context.additionalRepositoryCheckoutInfo = subRepoCommits; + } + // if the original contexturl was pointing to a subrepo we update the commit information with the mainContext. + if (mainRepoContext && CommitContext.is(mainRepoContext)) { + context.repository = mainRepoContext.repository; + context.revision = mainRepoContext.revision; + } + context.checkoutLocation = (config.config.checkoutLocation || context.repository.name); + context.upstreamRemoteURI = this.buildUpstreamCloneUrl(context); + return context; + } finally { + span.finish(); + } + } + protected findPrefix(user: User, context: string): { prefix: string, parser: IPrefixContextParser } | undefined { for (const parser of this.prefixParser) { const prefix = parser.findPrefix(user, context); diff --git a/components/server/src/workspace/context-parser.ts b/components/server/src/workspace/context-parser.ts index ea1956f3e8d80a..e086f59d1e8a19 100644 --- a/components/server/src/workspace/context-parser.ts +++ b/components/server/src/workspace/context-parser.ts @@ -14,7 +14,6 @@ export interface IContextParser { normalize?(contextUrl: string): string | undefined canHandle(user: User, contextUrl: string): boolean handle(ctx: TraceContext, user: User, contextUrl: string): Promise - fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise } export const IContextParser = Symbol("IContextParser") @@ -78,13 +77,6 @@ export abstract class AbstractContextParser implements IContextParser { } public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise; - - /** - * Fetches the commit history of a commit (used to find a relevant parent prebuild for incremental prebuilds). - * - * @returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log` - */ - public abstract fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string, maxDepth: number): Promise; } export interface URLParts { diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index 64cc885b53a0de..99c8be567e4d2d 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -1877,7 +1877,8 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { traceAPIParams(ctx, { cloneUrl }); const user = this.checkUser("fetchRepositoryConfiguration"); try { - return await this.configurationService.fetchRepositoryConfiguration(ctx, user, cloneUrl); + const context = await this.contextParser.handle(ctx, user, cloneUrl) as CommitContext; + return await this.configurationService.fetchRepositoryConfiguration(ctx, user, context); } catch (error) { if (UnauthorizedError.is(error)) { throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data); @@ -1898,7 +1899,8 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } try { - return await this.configurationService.fetchRepositoryConfiguration(ctx, user, project.cloneUrl); + const context = await this.contextParser.handle(ctx, user, project.cloneUrl) as CommitContext; + return await this.configurationService.fetchRepositoryConfiguration(ctx, user, context); } catch (error) { if (UnauthorizedError.is(error)) { throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data); @@ -1910,7 +1912,8 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { public async guessRepositoryConfiguration(ctx: TraceContext, cloneUrl: string): Promise { const user = this.checkUser("guessRepositoryConfiguration"); try { - return await this.configurationService.guessRepositoryConfiguration(ctx, user, cloneUrl); + const context = await this.contextParser.handle(ctx, user, cloneUrl) as CommitContext; + return await this.configurationService.guessRepositoryConfiguration(ctx, user, context); } catch (error) { if (UnauthorizedError.is(error)) { throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data); @@ -1930,7 +1933,8 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } try { - return await this.configurationService.guessRepositoryConfiguration(ctx, user, project.cloneUrl); + const context = await this.contextParser.handle(ctx, user, project.cloneUrl) as CommitContext; + return await this.configurationService.guessRepositoryConfiguration(ctx, user, context); } catch (error) { if (UnauthorizedError.is(error)) { throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data); diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts index 5e8ef51fc7ea2f..78405b668809a8 100644 --- a/components/server/src/workspace/workspace-starter.ts +++ b/components/server/src/workspace/workspace-starter.ts @@ -7,7 +7,7 @@ import { CloneTargetMode, FileDownloadInitializer, GitAuthMethod, GitConfig, GitInitializer, PrebuildInitializer, SnapshotInitializer, WorkspaceInitializer } from "@gitpod/content-service/lib"; import { CompositeInitializer, FromBackupInitializer } from "@gitpod/content-service/lib/initializer_pb"; import { DBUser, DBWithTracing, ProjectDB, TracedUserDB, TracedWorkspaceDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib'; -import { CommitContext, Disposable, GitpodToken, GitpodTokenType, IssueContext, NamedWorkspaceFeatureFlag, PullRequestContext, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ImageBuildLogInfo, ProjectEnvVar } from "@gitpod/gitpod-protocol"; +import { CommitContext, Disposable, GitpodToken, GitpodTokenType, GitCheckoutInfo, NamedWorkspaceFeatureFlag, RefType, SnapshotContext, StartWorkspaceResult, User, UserEnvVar, UserEnvVarValue, WithEnvvarsContext, WithPrebuild, Workspace, WorkspaceContext, WorkspaceImageSource, WorkspaceImageSourceDocker, WorkspaceImageSourceReference, WorkspaceInstance, WorkspaceInstanceConfiguration, WorkspaceInstanceStatus, WorkspaceProbeContext, Permission, HeadlessWorkspaceEvent, HeadlessWorkspaceEventType, DisposableCollection, AdditionalContentContext, ImageConfigFile, ProjectEnvVar, ImageBuildLogInfo } from "@gitpod/gitpod-protocol"; import { IAnalyticsWriter } from '@gitpod/gitpod-protocol/lib/analytics'; import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; @@ -37,6 +37,7 @@ import { IDEOption } from "@gitpod/gitpod-protocol/lib/ide-protocol"; import { Deferred } from "@gitpod/gitpod-protocol/lib/util/deferred"; import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints"; import { increaseFailedInstanceStartCounter, increaseSuccessfulInstanceStartCounter } from "../prometheus-metrics"; +import { ContextParser } from "./context-parser-service"; export interface StartWorkspaceOptions { rethrow?: boolean; @@ -65,6 +66,7 @@ export class WorkspaceStarter { @inject(TheiaPluginService) protected readonly theiaService: TheiaPluginService; @inject(OneTimeSecretServer) protected readonly otsServer: OneTimeSecretServer; @inject(ProjectDB) protected readonly projectDB: ProjectDB; + @inject(ContextParser) protected contextParser: ContextParser; public async startWorkspace(ctx: TraceContext, workspace: Workspace, user: User, userEnvVars: UserEnvVar[], projectEnvVars: ProjectEnvVar[], options?: StartWorkspaceOptions): Promise { const span = TraceContext.startSpan("WorkspaceStarter.startWorkspace", ctx); @@ -530,20 +532,27 @@ export class WorkspaceStarter { if (WorkspaceImageSourceDocker.is(imgsrc)) { let source: WorkspaceInitializer; const disp = new DisposableCollection(); - let checkoutLocation = this.getCheckoutLocation(workspace); + let checkoutLocation = CommitContext.is(workspace.context) && workspace.context.checkoutLocation || '.'; if (!AdditionalContentContext.hasDockerConfig(workspace.context, workspace.config) && imgsrc.dockerFileSource) { // TODO(se): we cannot change this initializer structure now because it is part of how baserefs are computed in image-builder. // Image builds should however just use the initialization if the workspace they are running for (i.e. the one from above). - const { git, disposable } = await this.createGitInitializer({ span }, workspace, { + + const { initializer, disposable } = await this.createCommitInitializer({span}, workspace, { ...imgsrc.dockerFileSource, + checkoutLocation: ".", title: "irrelevant", ref: undefined, }, user); disp.push(disposable); + let git: GitInitializer; + if (initializer instanceof CompositeInitializer) { + // we use the first git initializer for image builds only + git = initializer.getInitializerList()[0].getGit()!; + } else { + git = initializer; + } git.setCloneTaget(imgsrc.dockerFileSource.revision); git.setTargetMode(CloneTargetMode.REMOTE_COMMIT); - checkoutLocation = "." - git.setCheckoutLocation(checkoutLocation); source = new WorkspaceInitializer(); source.setGit(git); } else { @@ -1003,17 +1012,29 @@ export class WorkspaceStarter { const snapshot = new SnapshotInitializer(); snapshot.setSnapshot(context.snapshotBucketId); - const { git } = await this.createGitInitializer(traceCtx, workspace, context, user); + const { initializer } = await this.createCommitInitializer(traceCtx, workspace, context, user); const init = new PrebuildInitializer(); init.setPrebuild(snapshot); - init.setGit(git); + if (initializer instanceof CompositeInitializer) { + for (const myInit of initializer.getInitializerList()) { + if (myInit instanceof WorkspaceInitializer && myInit.hasGit()) { + init.addGit(myInit.getGit()); + } + } + } else { + init.addGit(initializer); + } result.setPrebuild(init); } else if (WorkspaceProbeContext.is(context)) { // workspace probes have no workspace initializer as they need no content } else if (CommitContext.is(context)) { - const { git, disposable } = await this.createGitInitializer(traceCtx, workspace, context, user); + const { initializer, disposable } = await this.createCommitInitializer(traceCtx, workspace, context, user); disp.push(disposable); - result.setGit(git); + if (initializer instanceof CompositeInitializer) { + result.setComposite(initializer); + } else { + result.setGit(initializer); + } } else { throw new Error("cannot create initializer for unkown context type"); } @@ -1036,11 +1057,13 @@ export class WorkspaceStarter { })); additionalInit.setFilesList(fileInfos); - additionalInit.setTargetLocation(this.getCheckoutLocation(workspace)); + if (CommitContext.is(context)) { + additionalInit.setTargetLocation(context.checkoutLocation || context.repository.name); + } // wire the protobuf structure - const newRoot = new WorkspaceInitializer(); const composite = new CompositeInitializer(); + const newRoot = new WorkspaceInitializer(); newRoot.setComposite(composite); composite.addInitializer(result); const wsInitializerForDownload = new WorkspaceInitializer(); @@ -1051,10 +1074,32 @@ export class WorkspaceStarter { return { initializer: result, disposable: disp }; } - protected async createGitInitializer(traceCtx: TraceContext, workspace: Workspace, context: CommitContext, user: User): Promise<{ git: GitInitializer, disposable: Disposable }> { - if (!CommitContext.is(context)) { - throw new Error("Unknown workspace context"); + protected async createCommitInitializer(ctx: TraceContext, workspace: Workspace, context: CommitContext, user: User): Promise<{initializer: GitInitializer | CompositeInitializer, disposable: Disposable}> { + const span = TraceContext.startSpan("createInitializerForCommit", ctx); + const mainGit = this.createGitInitializer({ span }, workspace, context, user); + if (!context.additionalRepositoryCheckoutInfo || context.additionalRepositoryCheckoutInfo.length === 0) { + return mainGit; + } + const subRepoInitializers = [mainGit]; + for (const subRepo of context.additionalRepositoryCheckoutInfo) { + subRepoInitializers.push(this.createGitInitializer({ span }, workspace, subRepo , user)); } + const inits = await Promise.all(subRepoInitializers); + const compositeInit = new CompositeInitializer(); + const compositeDisposable = new DisposableCollection(); + for (const r of inits) { + const wsinit = new WorkspaceInitializer(); + wsinit.setGit(r.initializer); + compositeInit.addInitializer(wsinit); + compositeDisposable.push(r.disposable); + } + return { + initializer: compositeInit, + disposable: compositeDisposable + }; + } + + protected async createGitInitializer(traceCtx: TraceContext, workspace: Workspace, context: GitCheckoutInfo, user: User): Promise<{initializer: GitInitializer, disposable: Disposable}> { const host = context.repository.host; const hostContext = this.hostContextProvider.get(host); if (!hostContext) { @@ -1080,14 +1125,13 @@ export class WorkspaceStarter { log.error({ workspaceId: workspace.id, userId: workspace.ownerId }, "cannot authenticate user for Git initializer", error); throw new Error("User is unauthorized!"); } - const cloneUrl = context.repository.cloneUrl || context.cloneUrl!; + const cloneUrl = context.repository.cloneUrl; var cloneTarget: string | undefined; var targetMode: CloneTargetMode; - const localBranchName = IssueContext.is(context) ? context.localBranch : undefined; - if (localBranchName) { + if (context.localBranch) { targetMode = CloneTargetMode.LOCAL_BRANCH; - cloneTarget = localBranchName; + cloneTarget = context.localBranch; } else if (RefType.getRefType(context) === 'tag') { targetMode = CloneTargetMode.REMOTE_COMMIT; cloneTarget = context.revision; @@ -1101,8 +1145,6 @@ export class WorkspaceStarter { targetMode = CloneTargetMode.REMOTE_HEAD; } - const upstreamRemoteURI = this.buildUpstreamCloneUrl(context); - const gitConfig = new GitConfig(); gitConfig.setAuthentication(GitAuthMethod.BASIC_AUTH_OTS); gitConfig.setAuthOts(tokenOTS); @@ -1123,40 +1165,22 @@ export class WorkspaceStarter { const result = new GitInitializer(); result.setConfig(gitConfig); - result.setCheckoutLocation(this.getCheckoutLocation(workspace)); + result.setCheckoutLocation(context.checkoutLocation || context.repository.name); if (!!cloneTarget) { result.setCloneTaget(cloneTarget); } result.setRemoteUri(cloneUrl); result.setTargetMode(targetMode); - if (!!upstreamRemoteURI) { - result.setUpstreamRemoteUri(upstreamRemoteURI); + if (!!context.upstreamRemoteURI) { + result.setUpstreamRemoteUri(context.upstreamRemoteURI); } return { - git: result, + initializer: result, disposable }; } - protected getCheckoutLocation(workspace: Workspace) { - return workspace.config.checkoutLocation || CommitContext.is(workspace.context) && workspace.context.repository.name || '.'; - } - - protected buildUpstreamCloneUrl(context: CommitContext): string | undefined { - let upstreamCloneUrl: string | undefined = undefined; - if (PullRequestContext.is(context) && context.base) { - upstreamCloneUrl = context.base.repository.cloneUrl; - } else if (context.repository.fork) { - upstreamCloneUrl = context.repository.fork.parent.cloneUrl; - } - - if (context.repository.cloneUrl === upstreamCloneUrl) { - return undefined; - } - return upstreamCloneUrl; - } - protected toWorkspaceFeatureFlags(featureFlags: NamedWorkspaceFeatureFlag[]): WorkspaceFeatureFlag[] { const result = featureFlags.map(name => { for (const key in WorkspaceFeatureFlag) { diff --git a/components/ws-daemon/pkg/content/service.go b/components/ws-daemon/pkg/content/service.go index ad1ca6cf51ec43..4eb93438a64f76 100644 --- a/components/ws-daemon/pkg/content/service.go +++ b/components/ws-daemon/pkg/content/service.go @@ -282,8 +282,10 @@ func getCheckoutLocation(req *api.InitWorkspaceRequest) string { } } if ir, ok := spec.(*csapi.WorkspaceInitializer_Prebuild); ok { - if ir.Prebuild != nil && ir.Prebuild.Git != nil { - return ir.Prebuild.Git.CheckoutLocation + if ir.Prebuild != nil { + if len(ir.Prebuild.Git) > 0 { + return ir.Prebuild.Git[0].CheckoutLocation + } } } return "" diff --git a/scripts/mysql.sh b/scripts/mysql.sh new file mode 100755 index 00000000000000..b4346aae67cc04 --- /dev/null +++ b/scripts/mysql.sh @@ -0,0 +1,3 @@ +#!/bin/bash +kubectl port-forward statefulset/mysql 3306:3306 & +mysql -h 127.0.0.1 -P 3306 -u gitpod -D gitpod --select-limit=200 --safe-updates --password="$(kubectl get secrets mysql -o jsonpath="{.data.password}" | base64 -d)" \ No newline at end of file