diff --git a/install/installer/Makefile b/install/installer/Makefile index 0bd53573fd6562..297b2494923fc7 100644 --- a/install/installer/Makefile +++ b/install/installer/Makefile @@ -34,3 +34,8 @@ versionManifest: @echo "Downloading version manifest for ${VERSION}" docker run -it --rm eu.gcr.io/gitpod-core-dev/build/versions:${VERSION} cat versions.yaml > ${VERSION_MANIFEST} .PHONY: versionManifest + +config-doc: + @echo "Building doc from Config struct for current version" + go run ./scripts/structdoc.go +.PHONY: config-doc diff --git a/install/installer/pkg/config/v1/config.go b/install/installer/pkg/config/v1/config.go index 2cec06b9f13c59..7c08e516d48db9 100644 --- a/install/installer/pkg/config/v1/config.go +++ b/install/installer/pkg/config/v1/config.go @@ -64,11 +64,14 @@ func (v version) Defaults(in interface{}) error { return nil } +// Config defines the v1 version structure of the gitpod config file type Config struct { - Kind InstallationKind `json:"kind" validate:"required,installation_kind"` - Domain string `json:"domain" validate:"required,fqdn"` - Metadata Metadata `json:"metadata"` - Repository string `json:"repository" validate:"required,ascii"` + // Installation type to run - for most users, this will be Full + Kind InstallationKind `json:"kind" validate:"required,installation_kind"` + // The domain to deploy to + Domain string `json:"domain" validate:"required,fqdn"` + Metadata Metadata `json:"metadata"` + Repository string `json:"repository" validate:"required,ascii"` Observability Observability `json:"observability"` Analytics *Analytics `json:"analytics,omitempty"` @@ -103,6 +106,7 @@ type Config struct { } type Metadata struct { + // Location for your objectStorage provider Region string `json:"region" validate:"required"` // InstallationShortname establishes the "identity" of the (application) cluster. InstallationShortname string `json:"shortname" validate:"required"` @@ -219,9 +223,12 @@ type Resources struct { } type WorkspaceRuntime struct { - FSShiftMethod FSShiftMethod `json:"fsShiftMethod" validate:"required,fs_shift_method"` - ContainerDRuntimeDir string `json:"containerdRuntimeDir" validate:"required,startswith=/"` - ContainerDSocket string `json:"containerdSocket" validate:"required,startswith=/"` + // File system + FSShiftMethod FSShiftMethod `json:"fsShiftMethod" validate:"required,fs_shift_method"` + // The location of containerd socket on the host machine + ContainerDRuntimeDir string `json:"containerdRuntimeDir" validate:"required,startswith=/"` + // The location of containerd socket on the host machine + ContainerDSocket string `json:"containerdSocket" validate:"required,startswith=/"` } type WorkspaceTemplates struct { diff --git a/install/installer/pkg/config/v1/config.md b/install/installer/pkg/config/v1/config.md new file mode 100644 index 00000000000000..e7388b45e69e44 --- /dev/null +++ b/install/installer/pkg/config/v1/config.md @@ -0,0 +1,88 @@ +# Config v1 + +Config defines the v1 version structure of the gitpod config file + + +## Supported parameters +| Property | Type | Required | Allowed| Description | +| --- | --- | --- | --- | --- | +|`kind`|string|N| `Meta`, `Workspace`, `Full` || +|`domain`|string|Y| | The domain to deploy to| +|`metadata.region`|string|Y| | Location for your objectStorage provider| +|`repository`|string|Y| || +|`observability.logLevel`|string|N| `trace`, `debug`, `info`, `warning`, `error`, `fatal`, `panic` |Taken from github.com/gitpod-io/gitpod/components/gitpod-protocol/src/util/logging.ts| +|`observability.tracing.endpoint`|string|N| || +|`observability.tracing.agentHost`|string|N| || +|`analytics.segmentKey`|string|N| || +|`analytics.writer`|string|N| || +|`database.inCluster`|bool|N| || +|`database.external.certificate.kind`|string|N| `secret` || +|`database.external.certificate.name`|string|Y| || +|`database.cloudSQL.serviceAccount.kind`|string|N| `secret` || +|`database.cloudSQL.serviceAccount.name`|string|Y| || +|`database.cloudSQL.instance`|string|Y| || +|`objectStorage.inCluster`|bool|N| || +|`objectStorage.s3.endpoint`|string|Y| || +|`objectStorage.s3.credentials.kind`|string|N| `secret` || +|`objectStorage.s3.credentials.name`|string|Y| || +|`objectStorage.cloudStorage.serviceAccount.kind`|string|N| `secret` || +|`objectStorage.cloudStorage.serviceAccount.name`|string|Y| || +|`objectStorage.cloudStorage.project`|string|Y| || +|`objectStorage.azure.credentials.kind`|string|N| `secret` || +|`objectStorage.azure.credentials.name`|string|Y| || +|`containerRegistry.inCluster`|bool|Y| || +|`containerRegistry.external.url`|string|Y| || +|`containerRegistry.external.certificate.kind`|string|N| `secret` || +|`containerRegistry.external.certificate.name`|string|Y| || +|`containerRegistry.s3storage.bucket`|string|Y| || +|`containerRegistry.s3storage.certificate.kind`|string|N| `secret` || +|`containerRegistry.s3storage.certificate.name`|string|Y| || +|`certificate.kind`|string|N| `secret` || +|`certificate.name`|string|Y| || +|`imagePullSecrets[ ].kind`|string|N| `secret` || +|`imagePullSecrets[ ].name`|string|Y| || +|`workspace.runtime.fsShiftMethod`|string|N| `fuse`, `shiftfs` || +|`workspace.runtime.containerdRuntimeDir`|string|Y| | The location of containerd socket on the host machine| +|`workspace.runtime.containerdSocket`|string|Y| | The location of containerd socket on the host machine| +|`workspace.resources.requests`||Y| | todo(sje): add custom validation to corev1.ResourceList| +|`workspace.resources.limits`||N| || +|`workspace.resources.dynamicLimits`||N| || +|`workspace.templates.default`||N| || +|`workspace.templates.prebuild`||N| || +|`workspace.templates.ghost`||N| || +|`workspace.templates.imagebuild`||N| || +|`workspace.templates.regular`||N| || +|`workspace.templates.probe`||N| || +|`workspace.maxLifetime`||Y| | MaxLifetime is the maximum time a workspace is allowed to run. After that, the workspace times out despite activity| +|`workspace.timeoutDefault`||N| | TimeoutDefault is the default timeout of a regular workspace| +|`workspace.timeoutExtended`||N| | TimeoutExtended is the workspace timeout that a user can extend to for one workspace| +|`workspace.timeoutAfterClose`||N| | TimeoutAfterClose is the time a workspace timed out after it has been closed (“closed” means that it does not get a heartbeat from an IDE anymore)| +|`openVSX.url`|string|N| || +|`authProviders[ ].kind`|string|N| `secret` || +|`authProviders[ ].name`|string|Y| || +|`blockNewUsers.enabled`|bool|N| || +|`blockNewUsers.passlist[ ]`|[]string|N| || +|`license.kind`|string|N| `secret` || +|`license.name`|string|Y| || +|`sshGatewayHostKey.kind`|string|N| `secret` || +|`sshGatewayHostKey.name`|string|Y| || +|`disableDefinitelyGp`|bool|N| || +|`apiVersion`|string|Y| |API version of the Gitpod config defintion. `v1` in this version of Config| + + +# Experimental config parameters v1 + +Additional config parameters that are in experimental state + +## Supported parameters +| Property | Type | Required | Allowed| Description | +| --- | --- | --- | --- | --- | +|`experimental.workspace.tracing.samplerType`|string|N| `const`, `probabilistic`, `rateLimiting`, `remote` |Values taken from https://github.com/jaegertracing/jaeger-client-go/blob/967f9c36f0fa5a2617c9a0993b03f9a3279fadc8/config/config.go#L71| +|`experimental.workspace.tracing.samplerParam`|float64|N| || +|`experimental.workspace.stage`|string|N| || +|`experimental.workspace.stage`|string|N| || +|`experimental.workspace.registryFacade`||N| || +|`experimental.webapp`|WebAppConfig|N| || +|`experimental.ide`|IDEConfig|N| || + + diff --git a/install/installer/scripts/README.md b/install/installer/scripts/README.md new file mode 100644 index 00000000000000..6cf25cb4f39e5f --- /dev/null +++ b/install/installer/scripts/README.md @@ -0,0 +1,30 @@ +# StructDoc script + +This directory contains a small script that would automatically generate a +markdown documentation for the supported parameters in the config file. The +script is intended to serve as an easy way to enable the creation of this doc +considering the heavily changing nature of the config. The script lets one also +create documentation for different versions of the Config file as well. + +This script essentially parses the AST of the `config` package to aquire the +structure names, fields and docs associated to them. The script is highly +customized for parsing the `Config` struct, so cannot be applied for any other +structs for generic doc generation. + +## Usage + +This script is intended to be run from the root directory of the installer + +``` sh +$ go run ./scripts/structdoc.go --version "v1" +INFO[0000] Generating doc for config version v1 +INFO[0000] The doc is written to the file pkg/config/v1/config.md + +# Alternatively one can also use the make target +# to create the doc for current version +$ make config-doc +Building doc from Config struct for current version +go run ./scripts/structdoc.go +INFO[0000] Generating doc for config version v1 +INFO[0000] The doc is written to the file pkg/config/v1/config.md +``` diff --git a/install/installer/scripts/structdoc.go b/install/installer/scripts/structdoc.go new file mode 100644 index 00000000000000..a31930308fef18 --- /dev/null +++ b/install/installer/scripts/structdoc.go @@ -0,0 +1,333 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/token" + "io/ioutil" + "path/filepath" + "strings" + + "github.com/fatih/structtag" + log "github.com/sirupsen/logrus" +) + +const ( + configDir = "./pkg/config" // todo(nvn): better ways to handle the config path +) + +var version string + +type configDoc struct { + configName string + doc string + fields map[string][]fieldSpec +} + +type fieldSpec struct { + name string + required bool + doc string + value string + allowedValues string +} + +// extractTags strips the tags of each struct field and returns json name of the +// field and if the field is a mandatory one +func extractTags(tag string) (result fieldSpec, err error) { + + // unfortunately structtag doesn't support multiple keys, + // so we have to handle this manually + tag = strings.Trim(tag, "`") + + tagObj, err := structtag.Parse(tag) // we assume at least JSON tag is always present + if err != nil { + return + } + + metadata, err := tagObj.Get("json") + if err != nil { + return + } + + result.name = metadata.Name + + reqInfo, err := tagObj.Get("validate") + if err != nil { + // bit of a hack to overwrite the value of error since we do + // not care if `validate` field is absent + err = nil + result.required = false + } else { + result.required = reqInfo.Name == "required" + } + + return +} + +func extractPkg(name string, dir string) (config configDoc, err error) { + fset := token.NewFileSet() + + pkgs, err := parser.ParseDir(fset, dir, nil, parser.ParseComments) + if err != nil { + return + } + + pkgInfo, ok := pkgs[name] + + if !ok { + err = fmt.Errorf("Could not extract pkg %s", name) + return + } + + pkgData := doc.New(pkgInfo, "./", 0) + + return extractStructInfo(pkgData.Types) +} + +func extractStructFields(structType *ast.StructType) (specs []fieldSpec, err error) { + var fieldInfo fieldSpec + if structType != nil && structType.Fields != nil { + + for _, field := range structType.Fields.List { + // we extract all the tags of the struct + if field.Tag != nil { + fieldInfo, err = extractTags(field.Tag.Value) + if err != nil { + return + } + + // we document experimental section separately + if fieldInfo.name == "experimental" { + continue + } + } + + switch xv := field.Type.(type) { + case *ast.StarExpr: + if si, ok := xv.X.(*ast.Ident); ok { + fieldInfo.value = si.Name + } + case *ast.Ident: + fieldInfo.value = xv.Name + case *ast.ArrayType: + fieldInfo.value = fmt.Sprintf("[]%s", xv.Elt) + } + + // Doc about the field can be provided as a comment + // above the field + if field.Doc != nil { + var comment string = "" + + // sometimes the comments are multi-line + for _, line := range field.Doc.List { + comment = fmt.Sprintf("%s %s", comment, strings.Trim(line.Text, "//")) + } + + fieldInfo.doc = comment + } + + specs = append(specs, fieldInfo) + } + } + + return +} + +func extractStructInfo(structTypes []*doc.Type) (configSpec configDoc, err error) { + configSpec.fields = map[string][]fieldSpec{} + for _, t := range structTypes { + + typeSpec := t.Decl.Specs[0].(*ast.TypeSpec) + + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + typename, aok := typeSpec.Type.(*ast.Ident) + if !aok { + continue + } + + allowed := []string{} + for _, con := range t.Consts[0].Decl.Specs { + value, ok := con.(*ast.ValueSpec) + if !ok { + continue + } + + for _, val := range value.Values { + bslit := val.(*ast.BasicLit) + + allowed = append(allowed, fmt.Sprintf("`%s`", strings.Trim(bslit.Value, "\""))) + } + } + + configSpec.fields[typeSpec.Name.Name] = []fieldSpec{ + { + name: typeSpec.Name.Name, + allowedValues: strings.Join(allowed, ", "), + value: typename.Name, + doc: t.Consts[0].Doc, + }, + } + + continue + + } + + structSpecs, err := extractStructFields(structType) + if err != nil { + return configSpec, err + } + + if t.Name == "Config" { + if strings.Contains(t.Doc, "experimental") { + // if we are dealing with experimental pkg we rename the config title + configSpec.configName = "Experimental config parameters" + configSpec.doc = "Additional config parameters that are in experimental state" + } else { + configSpec.configName = t.Name + configSpec.doc = t.Doc + // we hardcode the value for apiVersion since it is not present in + // Config struct + structSpecs = append(structSpecs, + fieldSpec{ + name: "apiVersion", + required: true, + value: "string", + doc: fmt.Sprintf("API version of the Gitpod config defintion."+ + " `%s` in this version of Config", version)}) + } + } + + configSpec.fields[typeSpec.Name.Name] = structSpecs + } + + return +} + +// parseConfigDir parses the AST of the config package and returns metadata +// about the `Config` struct +func parseConfigDir(fileDir string) (configSpec []configDoc, err error) { + // we basically parse the AST of the config package + configStruct, err := extractPkg("config", fileDir) + if err != nil { + return + } + + experimentalDir := fmt.Sprintf("%s/%s", fileDir, "experimental") + // we parse the AST of the experimental package since we have additional + // Config there + experimentalStruct, err := extractPkg("experimental", experimentalDir) + if err != nil { + return + } + + configSpec = []configDoc{configStruct, experimentalStruct} + + return +} + +func recurse(configSpec configDoc, field fieldSpec, parent string) []fieldSpec { + // check if field has type array + var arrayString, valuename string + if strings.Contains(field.value, "[]") { + arrayString = "[ ]" + valuename = strings.Trim(field.value, "[]") + } else { + valuename = field.value + } + + field.name = fmt.Sprintf("%s%s%s", parent, field.name, arrayString) + // results := []fieldSpec{field} + results := []fieldSpec{} + subFields := configSpec.fields[valuename] + + if len(subFields) < 1 { + // this means that this is a leaf node, terminating condition + return []fieldSpec{field} + } + + for _, sub := range subFields { + results = append(results, recurse(configSpec, sub, field.name+".")...) + } + + return results +} + +func generateMarkdown(configSpec configDoc, mddoc *strings.Builder) { + + var prefix string = "" + if strings.Contains(configSpec.configName, "Experimental") { + prefix = "experimental." + } + + mddoc.WriteString(fmt.Sprintf("# %s %s\n\n%s\n", configSpec.configName, version, configSpec.doc)) + mddoc.WriteString("\n## Supported parameters\n") + mddoc.WriteString("| Property | Type | Required | Allowed| Description |\n") + mddoc.WriteString("| --- | --- | --- | --- | --- |\n") + + results := []fieldSpec{} + fieldLists := configSpec.fields["Config"] + for _, field := range fieldLists { + results = append(results, recurse(configSpec, field, "")...) + } + + for _, res := range results { + reqd := "N" + if res.required { + reqd = "Y" + } + + if res.allowedValues != "" { + lastInd := strings.LastIndex(res.name, ".") + res.name = res.name[:lastInd] + + } + + mddoc.WriteString(fmt.Sprintf("|`%s%s`|%s|%s| %s |%s|\n", prefix, + res.name, res.value, reqd, res.allowedValues, strings.TrimSuffix(res.doc, + "\n"))) + } + + mddoc.WriteString("\n\n") +} + +func main() { + versionFlag := flag.String("version", "v1", "Config version for doc creation") + flag.Parse() + + version = *versionFlag + + log.Infof("Generating doc for config version %s", version) + + fileDir := fmt.Sprintf("%s/%s", configDir, version) + + // get the `Config` struct field info from `config` pkg + configSpec, err := parseConfigDir(fileDir) + if err != nil { + log.Fatal(err) + } + + // generate markdown for the doc + + mddoc := &strings.Builder{} + for _, spec := range configSpec { + generateMarkdown(spec, mddoc) + } + + // write the md file of name config.md in the same directory as config + mdfilename := filepath.Join(fileDir, "config.md") + + err = ioutil.WriteFile(mdfilename, []byte(mddoc.String()), 0644) + if err != nil { + log.Fatal(err) + } + + log.Infof("The doc is written to the file %s", mdfilename) +}