Skip to content

telemetry: add more fields to data #8067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 17, 2022
Merged

telemetry: add more fields to data #8067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions components/gitpod-db/src/typeorm/workspace-db-impl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
.addOrderBy('GREATEST(ws.creationTime, wsi.creationTime, wsi.startedTime, wsi.stoppedTime)', 'DESC')
.limit(options.limit || 10);
if (options.searchString) {
qb.andWhere("ws.description LIKE :searchString", {searchString: `%${options.searchString}%`});
qb.andWhere("ws.description LIKE :searchString", { searchString: `%${options.searchString}%` });
}
if (!options.includeHeadless) {
qb.andWhere("ws.type = 'regular'");
Expand Down Expand Up @@ -334,6 +334,15 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
return { total, rows };
}

public async getInstanceCount(type?: string): Promise<number> {
const workspaceInstanceRepo = await this.getWorkspaceInstanceRepo();
const queryBuilder = workspaceInstanceRepo.createQueryBuilder("wsi")
.leftJoinAndMapOne("wsi.workspace", DBWorkspace, "ws", "wsi.workspaceId = ws.id")
.where("ws.type = :type", { type: type ? type.toString() : "regular" }); // only regular workspaces by default

return await queryBuilder.getCount();
}

public async findRegularRunningInstances(userId?: string): Promise<WorkspaceInstance[]> {
const infos = await this.findRunningInstancesWithWorkspaces(undefined, userId);
return infos.filter(
Expand Down Expand Up @@ -578,7 +587,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {

public async findSnapshotsByWorkspaceId(workspaceId: string): Promise<Snapshot[]> {
const snapshots = await this.getSnapshotRepo();
return snapshots.find({where: {originalWorkspaceId: workspaceId}});
return snapshots.find({ where: { originalWorkspaceId: workspaceId } });
}

public async storePrebuiltWorkspace(pws: PrebuiltWorkspace): Promise<PrebuiltWorkspace> {
Expand All @@ -596,7 +605,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
}
const repo = await this.getPrebuiltWorkspaceRepo();
return await repo.createQueryBuilder('pws')
.where('pws.cloneURL = :cloneURL AND pws.commit LIKE :commit', { cloneURL, commit: commit+'%' })
.where('pws.cloneURL = :cloneURL AND pws.commit LIKE :commit', { cloneURL, commit: commit + '%' })
.orderBy('pws.creationTime', 'DESC')
.innerJoinAndMapOne('pws.workspace', DBWorkspace, 'ws', "pws.buildWorkspaceId = ws.id and ws.contentDeletedTime = ''")
.getOne();
Expand Down Expand Up @@ -737,7 +746,13 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
return { total, rows };
}

public async getWorkspaceCount(type?: String): Promise<Number> {
const workspaceRepo = await this.getWorkspaceRepo();
const queryBuilder = workspaceRepo.createQueryBuilder("ws")
.where("ws.type = :type", { type: type ? type.toString() : "regular" }); // only regular workspaces by default

return await queryBuilder.getCount();
}

public async findAllWorkspaceAndInstances(offset: number, limit: number, orderBy: keyof WorkspaceAndInstance, orderDir: "ASC" | "DESC", query?: AdminGetWorkspacesQuery, searchTerm?: string): Promise<{ total: number, rows: WorkspaceAndInstance[] }> {
let whereConditions = [];
Expand Down Expand Up @@ -827,7 +842,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
const workspaceRepo = await this.getWorkspaceRepo();
const workspace = await workspaceRepo.findOne(id);
if (!workspace) {
return;
return;
}

const instance = await this.findCurrentInstance(id);
Expand Down Expand Up @@ -890,7 +905,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
});
}

async findPrebuildInfos(prebuildIds: string[]): Promise<PrebuildInfo[]>{
async findPrebuildInfos(prebuildIds: string[]): Promise<PrebuildInfo[]> {
const repo = await this.getPrebuildInfoRepo();

const query = repo.createQueryBuilder('pi');
Expand All @@ -899,7 +914,7 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
if (filteredIds.length === 0) {
return [];
}
query.andWhere(`pi.prebuildId in (${ filteredIds.map(id => `'${id}'`).join(", ") })`)
query.andWhere(`pi.prebuildId in (${filteredIds.map(id => `'${id}'`).join(", ")})`)

const res = await query.getMany();
return res.map(r => r.info);
Expand Down
9 changes: 6 additions & 3 deletions components/gitpod-db/src/workspace-db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export interface WorkspacePortsAuthData {
workspace: WorkspaceAuthData;
}

export type WorkspaceInstanceSession = Pick<WorkspaceInstance, "id" | "startedTime"| "stoppingTime" | "stoppedTime">;
export type WorkspaceInstanceSession = Pick<WorkspaceInstance, "id" | "startedTime" | "stoppingTime" | "stoppedTime">;
export type WorkspaceSessionData = Pick<Workspace, "id" | "contextURL" | "context" | "type">;
export interface WorkspaceInstanceSessionWithWorkspace {
instance: WorkspaceInstanceSession;
Expand Down Expand Up @@ -67,7 +67,7 @@ export interface WorkspaceDB {

// Partial update: unconditional, single field updates. Enclose in a transaction if necessary
updateLastHeartbeat(instanceId: string, userId: string, newHeartbeat: Date, wasClosed?: boolean): Promise<void>;
getLastOwnerHeartbeatFor(instance: WorkspaceInstance): Promise<{ lastSeen:Date, wasClosed?: boolean} | undefined>;
getLastOwnerHeartbeatFor(instance: WorkspaceInstance): Promise<{ lastSeen: Date, wasClosed?: boolean } | undefined>;
getWorkspaceUsers(workspaceId: string, minLastSeen: number): Promise<WorkspaceInstanceUser[]>;
updateInstancePartial(instanceId: string, partial: DeepPartial<WorkspaceInstance>): Promise<WorkspaceInstance>;

Expand All @@ -84,12 +84,15 @@ export interface WorkspaceDB {
findAllWorkspaceAndInstances(offset: number, limit: number, orderBy: keyof WorkspaceAndInstance, orderDir: "ASC" | "DESC", query?: AdminGetWorkspacesQuery, searchTerm?: string): Promise<{ total: number, rows: WorkspaceAndInstance[] }>;
findWorkspaceAndInstance(id: string): Promise<WorkspaceAndInstance | undefined>;

getWorkspaceCount(type?: String): Promise<Number>;
getInstanceCount(type?: string): Promise<number>

findAllWorkspaceInstances(offset: number, limit: number, orderBy: keyof WorkspaceInstance, orderDir: "ASC" | "DESC", ownerId?: string, minCreationTime?: Date, maxCreationTime?: Date, onlyRunning?: boolean, type?: WorkspaceType): Promise<{ total: number, rows: WorkspaceInstance[] }>;

findRegularRunningInstances(userId?: string): Promise<WorkspaceInstance[]>;
findRunningInstancesWithWorkspaces(installation?: string, userId?: string, includeStopping?: boolean): Promise<RunningWorkspaceInfo[]>;

isWhitelisted(repositoryUrl : string): Promise<boolean>;
isWhitelisted(repositoryUrl: string): Promise<boolean>;
getFeaturedRepositories(): Promise<Partial<WhitelistedRepository>[]>;

findSnapshotById(snapshotId: string): Promise<Snapshot | undefined>;
Expand Down
7 changes: 7 additions & 0 deletions components/gitpod-protocol/src/installation-admin-protocol.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ export interface InstallationAdmin {
settings: InstallationAdminSettings;
}

export interface Data {
installationAdmin: InstallationAdmin
totalUsers: number
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As we add more telemetry data, We could continue adding them here thereby separating it from the InstallationAdmin (which integrates into the database).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same goes with the new Data struct for the golang side of things in installation-telemetry workspace

totalWorkspaces: number
totalInstances: number
}

export namespace InstallationAdmin {
export function createDefault(): InstallationAdmin {
return {
Expand Down
9 changes: 6 additions & 3 deletions components/installation-telemetry/cmd/send.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var sendCmd = &cobra.Command{
return err
}

if !data.Settings.SendTelemetry {
if !data.InstallationAdmin.Settings.SendTelemetry {
log.Info("installation-telemetry is not permitted to send - exiting")
return nil
}
Expand All @@ -51,10 +51,13 @@ var sendCmd = &cobra.Command{
}()

telemetry := analytics.Track{
UserId: data.ID,
UserId: data.InstallationAdmin.ID,
Event: "Installation telemetry",
Properties: analytics.NewProperties().
Set("version", versionId),
Set("version", versionId).
Set("totalUsers", data.TotalUsers).
Set("totalWorkspaces", data.TotalWorkspaces).
Set("totalInstances", data.TotalInstances),
}

client.Enqueue(telemetry)
Expand Down
13 changes: 10 additions & 3 deletions components/installation-telemetry/pkg/server/installationAdmin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,19 @@ type InstallationAdminSettings struct {
SendTelemetry bool `json:"sendTelemetry"`
}

type InstallationAdminData struct {
type Data struct {
InstallationAdmin InstallationAdmin `json:"installationAdmin"`
TotalUsers int64 `json:"totalUsers"`
TotalWorkspaces int64 `json:"totalWorkspaces"`
TotalInstances int64 `json:"totalInstances"`
}

type InstallationAdmin struct {
ID string `json:"id"`
Settings InstallationAdminSettings `json:"settings"`
}

func GetInstallationAdminData(config common.Config) (*InstallationAdminData, error) {
func GetInstallationAdminData(config common.Config) (*Data, error) {
resp, err := http.Get(fmt.Sprintf("%s/data", config.Server))
if err != nil {
return nil, err
Expand All @@ -35,7 +42,7 @@ func GetInstallationAdminData(config common.Config) (*InstallationAdminData, err
return nil, err
}

var data InstallationAdminData
var data Data
if err := json.Unmarshal(body, &data); err != nil {
return nil, err
}
Expand Down
12 changes: 10 additions & 2 deletions components/server/src/installation-admin/installation-admin-controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,25 @@

import { injectable, inject } from 'inversify';
import * as express from 'express';
import { InstallationAdminDB } from '@gitpod/gitpod-db/lib';
import { InstallationAdminDB, UserDB, WorkspaceDB } from '@gitpod/gitpod-db/lib';
import { Data } from '@gitpod/gitpod-protocol'

@injectable()
export class InstallationAdminController {
@inject(InstallationAdminDB) protected readonly installationAdminDb: InstallationAdminDB;
@inject(UserDB) protected readonly userDb: UserDB
@inject(WorkspaceDB) protected readonly workspaceDb: WorkspaceDB

public create(): express.Application {
const app = express();

app.get('/data', async (req: express.Request, res: express.Response) => {
const data = await this.installationAdminDb.getData();
const data: Data = {
installationAdmin: await this.installationAdminDb.getData(),
totalUsers: await this.userDb.getUserCount(true),
totalWorkspaces: await this.workspaceDb.getWorkspaceCount(),
totalInstances: await this.workspaceDb.getInstanceCount(),
} as Data;

res.status(200).json(data);
});
Expand Down