From 7e1c679682fdf00c281355dc4b7373ab0c086e07 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Thu, 4 Jun 2020 16:31:56 +0300 Subject: [PATCH 01/17] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a0e10b28e..5923b9d15 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.68.1", + "version": "0.68.3", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true, From d28f91783f2151babd1f0ca703bf98684c7f016e Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Fri, 5 Jun 2020 14:16:12 +0300 Subject: [PATCH 02/17] wip --- .../cli/commands/agent/install.cmd.js | 2 + lib/interface/cli/commands/hybrid/helper.js | 126 ++++++-- lib/interface/cli/commands/hybrid/init.cmd.js | 278 +++++++++--------- .../commands/hybrid/installation-process.js | 1 + .../cli/commands/hybrid/migration.js | 2 +- .../runtimeEnvironments/install.cmd.js | 3 +- 6 files changed, 246 insertions(+), 166 deletions(-) diff --git a/lib/interface/cli/commands/agent/install.cmd.js b/lib/interface/cli/commands/agent/install.cmd.js index ab8d360cd..96d0cb9b4 100644 --- a/lib/interface/cli/commands/agent/install.cmd.js +++ b/lib/interface/cli/commands/agent/install.cmd.js @@ -83,6 +83,7 @@ const installAgentCmd = new Command({ name, token, } = argv; const { + 'runtime-name': reName, 'kube-node-selector': kubeNodeSelector, 'dry-run': dryRun, 'in-cluster': inCluster, @@ -180,6 +181,7 @@ const installAgentCmd = new Command({ } if (installRuntime) { return installRuntimeCmd.handler({ + 'runtime-name': reName, 'runtime-kube-context-name': kubeContextName, 'runtime-kube-namespace': kubeNamespace, 'agent-name': name, diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js index 32a6f136f..d0acbb70d 100644 --- a/lib/interface/cli/commands/hybrid/helper.js +++ b/lib/interface/cli/commands/hybrid/helper.js @@ -3,7 +3,6 @@ const colors = require('colors'); const { to } = require('./../../../../logic/cli-config/errors/awaitTo'); const sdk = require('../../../../logic/sdk'); const _ = require('lodash'); -const { PIPELINE_CREATED, PIPELINE_EXECUTED } = require('./installation-process').events; const { status: STATUSES } = require('./installation-process'); const pipelinesRunCmd = require('../pipeline/run.cmd'); const { getAllNamespaces } = require('../../helpers/kubernetes'); @@ -11,6 +10,9 @@ const { followLogs } = require('../../helpers/logs'); const ProgressEvents = require('../../helpers/progressEvents'); const cliProgress = require('cli-progress'); const figlet = require('figlet'); +const { + components, Runner, Downloader, CommonProgressFormat, +} = require('./../../../../binary'); const INSTALLATION_DEFAULTS = { NAMESPACE: 'codefresh', @@ -21,6 +23,7 @@ const INSTALLATION_DEFAULTS = { CF_CONTEXT_NAME: 'cf-runner', }; +const maxRuntimeNameLength = 63; const DefaultLogFormatter = 'plain'; const defaultOpenIssueMessage = 'If you had any issues with this process please report them at: ' + @@ -55,7 +58,7 @@ async function getTestPipelineLink(pipelineName, pipeline) { return ''; } -async function createTestPipeline(runtimeName, pipelineName, pipelineCommands, progressReporter) { +async function createTestPipeline(runtimeName, pipelineName, pipelineCommands) { await _createRunnerProjectIfNotExists(); console.log(`Creating test pipeline with the name: "${colors.cyan(pipelineName)}" ` + `in project "${colors.cyan(INSTALLATION_DEFAULTS.PROJECT_NAME)}"`); @@ -83,10 +86,6 @@ async function createTestPipeline(runtimeName, pipelineName, pipelineCommands, p }, ); - if (progressReporter) { - await to(progressReporter.report(PIPELINE_CREATED, STATUSES.SUCCESS)); - } - const pipelineLink = await getTestPipelineLink(undefined, pipeline); console.log(`Created test pipeline with the name "${colors.cyan(pipelineName)}". Watch it here: ${colors.blue(pipelineLink)}`); @@ -107,9 +106,15 @@ async function getTestPipeline(pipelineName) { return null; } -async function updateTestPipelineRuntime(pipeline, runtimeName) { - // update pipeline runtime - const _pipeline = pipeline; +async function updateTestPipelineRuntime(pipeline, runtimeName, pipelineName) { + let _pipeline = pipeline; + if (!_pipeline) { + const testPipeline = await getTestPipeline(pipelineName); + if (!testPipeline) { + throw new Error('Could not get test pipeline'); + } + _pipeline = testPipeline; + } _pipeline.spec.runtimeEnvironment = { name: runtimeName, }; @@ -125,9 +130,9 @@ async function updateTestPipelineRuntime(pipeline, runtimeName) { ); } -async function executeTestPipeline(runtimeName, pipeline, progressReporter) { +async function executeTestPipeline(runtimeName, pipelineName) { const url = _.get(sdk, 'config.context.url', 'https://g.codefresh.io'); - const pipelineName = _.get(pipeline, 'metadata.name'); + console.log(`${colors.yellow('*NOTE* Running a pipeline for the first time might take longer than usual.')}`); const workflowId = await pipelinesRunCmd.handler({ name: pipelineName, @@ -139,24 +144,23 @@ async function executeTestPipeline(runtimeName, pipeline, progressReporter) { const buildLink = `${url}/build/${workflowId}`; console.log(`Executing pipeline "${colors.cyan(pipelineName)}", watch it at: ${colors.blue(buildLink)}`); await followLogs(workflowId); - if (progressReporter) { - await to(progressReporter.report(PIPELINE_EXECUTED, STATUSES.SUCCESS)); - } } function createErrorHandler(openIssueMessage = defaultOpenIssueMessage) { - return async (error, message, progressReporter, event) => { + return async (error, message, progressReporter, event, exitOnError = true) => { if (!error) { return; } - if (progressReporter) { + if (progressReporter && event) { await to(progressReporter.report(event, STATUSES.FAILURE)); } console.log(`${colors.red('Error:')} ${message}: ${prettyError(error)}`); - console.log(colors.green(openIssueMessage)); - process.exit(1); + if (exitOnError) { + console.log(colors.green(openIssueMessage)); + process.exit(1); + } }; } @@ -269,6 +273,88 @@ function drawCodefreshFiglet() { }); } +async function getDefaultRuntime(runtimes) { + let _runtimes = runtimes; + if (!_runtimes) { + _runtimes = await sdk.runtimeEnvs.list({ }); + } + const defaultRe = _.find(_runtimes, re => re.default); + + return defaultRe; +} + +async function getRecommendedKubeNamespace(kubeconfigPath, kubeContextName) { + const defaultName = INSTALLATION_DEFAULTS.NAMESPACE; + const [err, namespaces] = await to(getAllNamespaces(kubeconfigPath, kubeContextName)); + let name; + + if (err || !_.isArray(namespaces) || !_.find(namespaces, ns => ns === defaultName)) { + name = defaultName; // use the default name if there are no collisions + } else { + const namespacesSet = new Set(namespaces); // for fast lookup + let i = 1; + while (namespacesSet.has(`${defaultName}-${i}`)) { + i += 1; + } + name = `${defaultName}-${i}`; + } + + return name; +} + +async function _downloadVeonona() { + const downloader = new Downloader({ + progress: new cliProgress.SingleBar( + { + stopOnComplete: true, + format: CommonProgressFormat, + }, + cliProgress.Presets.shades_classic, + ), + }); + await downloader.download(components.venona); +} + +async function runClusterAcceptanceTests({ kubeNamespace, kubeConfigPath }) { + await _downloadVeonona(); + const componentRunner = new Runner(); + const cmd = ['test', '--log-formtter', DefaultLogFormatter]; + if (kubeNamespace) { + cmd.push('--kube-namespace'); + cmd.push(kubeNamespace); + } + if (kubeConfigPath) { + cmd.push('--kube-config-path'); + cmd.push(kubeConfigPath); + } + await componentRunner.run(components.venona, cmd); +} + +async function newRuntimeName(kubeContextName, kubeNamespace) { + const defaultName = `${kubeContextName}/${kubeNamespace}`.slice(0, maxRuntimeNameLength); + const runtimes = await sdk.runtimeEnvs.list({ }); + let name; + + if (!_.isArray(runtimes) || !_.find(runtimes, re => _.get(re, 'metadata.name') === defaultName)) { + name = defaultName; // use the default name if there are no collisions + } else { + const reNames = new Set(_.map(runtimes, re => _.get(re, 'metadata.name'))); // for fast lookup + let i = 1; + let suggestName; + // eslint-disable-next-line no-constant-condition + while (true) { + suggestName = `${defaultName.slice(0, maxRuntimeNameLength - 1 - i.toString().length)}_${i}`; + if (!reNames.has(suggestName)) { + break; + } + i += 1; + } + name = suggestName; + } + + return name; +} + module.exports = { getRelatedAgents, createErrorHandler, @@ -282,6 +368,10 @@ module.exports = { createProgressBar, getTestPipelineLink, drawCodefreshFiglet, + getDefaultRuntime, + getRecommendedKubeNamespace, + runClusterAcceptanceTests, + newRuntimeName, INSTALLATION_DEFAULTS, DefaultLogFormatter, }; diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index bce023b6d..0d1762ac3 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -4,8 +4,7 @@ const runnerRoot = require('../root/runner.cmd'); const inquirer = require('inquirer'); const colors = require('colors'); const _ = require('lodash'); -const cliProgress = require('cli-progress'); -const { getAllKubeContexts, getKubeContext, getAllNamespaces } = require('../../helpers/kubernetes'); +const { getAllKubeContexts, getKubeContext } = require('../../helpers/kubernetes'); const installAgent = require('../agent/install.cmd'); const installMonitoring = require('../monitor/install.cmd'); const createContext = require('../auth/create-context.cmd'); @@ -15,71 +14,22 @@ const DEFAULTS = require('../../defaults'); const sdk = require('../../../../logic/sdk'); const installationProgress = require('./installation-process'); const { to } = require('./../../../../logic/cli-config/errors/awaitTo'); -const { createErrorHandler, DefaultLogFormatter } = require('./helper'); +const { createErrorHandler } = require('./helper'); const { - getTestPipeline, createTestPipeline, executeTestPipeline, updateTestPipelineRuntime, drawCodefreshFiglet, + getDefaultRuntime, + getRecommendedKubeNamespace, + runClusterAcceptanceTests, + newRuntimeName, INSTALLATION_DEFAULTS, } = require('./helper'); -const { - components, Runner, Downloader, CommonProgressFormat, -} = require('./../../../../binary'); +const InstallationPlan = require('./InstallationPlan'); const handleError = createErrorHandler(`\nIf you had any issues with the installation please report them at: ${colors.blue('https://github.com/codefresh-io/cli/issues/new')}`); -async function createAndRunTestPipeline(runtimeName, errHandler, progressReporter) { - let testPipeline; - const [getPipelineErr, _testPipeline] = await to(getTestPipeline(INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME)); - testPipeline = _testPipeline; - await errHandler(getPipelineErr, 'Could not get test pipeline', progressReporter, installationProgress.events.PIPELINE_CREATED); - if (!testPipeline) { - // eslint-disable-next-line no-shadow - const [createPipelineErr, _testPipeline] = await to(createTestPipeline( - runtimeName, - INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, - ['echo hello codefresh runner!'], - progressReporter, - )); - await errHandler(createPipelineErr, 'Failed to create test pipeline', progressReporter, installationProgress.events.PIPELINE_CREATED); - testPipeline = _testPipeline; - } else { - const [updatePipelineErr] = await to(updateTestPipelineRuntime(testPipeline, runtimeName)); - if (updatePipelineErr) { - console.log(colors.yellow('*warning* failed to update test pipeline runtime, you can' + - ' change it manually if you want to run it again on this runtime')); - } - } - console.log(`${colors.yellow('*NOTE* Running a pipeline for the first time might take longer than usual.')}`); - const [runPipelineErr] = await to(executeTestPipeline( - runtimeName, - testPipeline, - progressReporter, - )); - await errHandler(runPipelineErr, 'Failed to run test pipeline', progressReporter, installationProgress.events.PIPELINE_EXECUTED); -} - -async function getRecommendedKubeNamespace(kubeconfigPath, kubeContextName) { - const defaultName = INSTALLATION_DEFAULTS.NAMESPACE; - const [err, namespaces] = await to(getAllNamespaces(kubeconfigPath, kubeContextName)); - let name; - - if (err || !_.isArray(namespaces) || !_.find(namespaces, ns => ns === defaultName)) { - name = defaultName; // use the default name if there are no collisions - } else { - const namespacesSet = new Set(namespaces); // for fast lookup - let i = 1; - while (namespacesSet.has(`${defaultName}-${i}`)) { - i += 1; - } - name = `${defaultName}-${i}`; - } - - return name; -} - async function isNewAccount() { const [pipelines, err] = await to(sdk.pipelines.list({ })); if (!err && _.isArray(_.get(pipelines, 'docs'))) { @@ -89,6 +39,20 @@ async function isNewAccount() { return false; } +function printInstallationOptionsSummary({ + kubeContextName, + kubeNamespace, + shouldMakeDefaultRe, + shouldExecutePipeline, +}) { + console.log(`\n${colors.green('Installation options summary:')} + 1. Kubernetes Context: ${colors.cyan(kubeContextName)} + 2. Kubernetes Namespace: ${colors.cyan(kubeNamespace)} + 3. Set this as default account runtime-environment: ${colors.cyan(!!shouldMakeDefaultRe)} + 4. Execute demo pipeline after install: ${colors.cyan(!!shouldExecutePipeline)} + `); +} + const initCmd = new Command({ root: false, parent: runnerRoot, @@ -186,7 +150,7 @@ const initCmd = new Command({ } if (noQuestions) { - // set defaults + // use defaults kubeContextName = getKubeContext(kubeConfigPath); kubeNamespace = await getRecommendedKubeNamespace(kubeConfigPath, kubeContextName); shouldMakeDefaultRe = INSTALLATION_DEFAULTS.MAKE_DEFAULT_RE; @@ -220,21 +184,14 @@ const initCmd = new Command({ if (_.isUndefined(shouldMakeDefaultRe)) { if (_.get(sdk, 'config.context.isNoAuth') || await isNewAccount()) { - // if this is a new account, don't ask and set this runtime as default + // don't ask and set this runtime as default if it's a new account shouldMakeDefaultRe = true; } else { let message = 'Set this as the default runtime environment for your Codefresh account? (Y/N)'; - - const [listReErr, runtimes] = await to(sdk.runtimeEnvs.list({ })); - if (listReErr) { - console.debug('Failed to fetch runtimes'); - } else { - const defaultRe = _.find(runtimes, re => re.default); - if (defaultRe) { - message = `Change the current default runtime "${colors.cyan(defaultRe.metadata.name)}" to new runtime ?`; - } + const [, defaultRe] = await to(getDefaultRuntime()); + if (defaultRe) { + message = `Change the current default runtime "${colors.cyan(defaultRe.metadata.name)}" to new runtime ?`; } - questions.push({ type: 'confirm', name: 'shouldMakeDefaultRe', @@ -260,12 +217,25 @@ const initCmd = new Command({ shouldExecutePipeline = _.isUndefined(shouldExecutePipeline) ? answers.shouldExecutePipeline : shouldExecutePipeline; } - console.log(`\n${colors.green('Installation options summary:')} -1. Kubernetes Context: ${colors.cyan(kubeContextName)} -2. Kubernetes Namespace: ${colors.cyan(kubeNamespace)} -3. Set this as default account runtime-environment: ${colors.cyan(!!shouldMakeDefaultRe)} -4. Execute demo pipeline after install: ${colors.cyan(!!shouldExecutePipeline)} -`); + printInstallationOptionsSummary({ + kubeContextName, + kubeNamespace, + shouldMakeDefaultRe, + shouldExecutePipeline, + }); + + if (token) { + // Create a new context and switch to that context + const [err] = await to(createContext.handler({ + apiKey: token, + name: INSTALLATION_DEFAULTS.CF_CONTEXT_NAME, + url, + })); + await handleError(err, 'Failed to use the provided token'); + const config = await getConfigForSdk(); + await sdk.configure(config); + console.log(`A Codefresh context named '${INSTALLATION_DEFAULTS.CF_CONTEXT_NAME}' was added to your "cfconfig" file.`); + } const [, progress] = await to(async () => installationProgress.create(sdk['runner-installation'], { options: { @@ -275,92 +245,108 @@ const initCmd = new Command({ shouldExecutePipeline, }, })); - const progressReporter = installationProgress.buildReporter(sdk['runner-installation'], progress); - - const downloader = new Downloader({ - progress: new cliProgress.SingleBar( - { - stopOnComplete: true, - format: CommonProgressFormat, - }, - cliProgress.Presets.shades_classic, - ), - }); - await downloader.download(components.venona); - const componentRunner = new Runner(); + const installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); if (skipClusterTest) { - console.log('Skipping cluster requirements tests.'); + console.log('Skipping cluster requirements tests...'); } else { - const cmd = ['test', '--log-formtter', DefaultLogFormatter]; - if (kubeNamespace) { - cmd.push('--kube-namespace'); - cmd.push(kubeNamespace); - } - if (kubeConfigPath) { - cmd.push('--kube-config-path'); - cmd.push(kubeConfigPath); - } - const [err] = await to(componentRunner.run(components.venona, cmd)); - await handleError(err, 'Failed to run cluster test'); + installationPlan.addStep({ + name: 'Cluster acceptance tests', + func: runClusterAcceptanceTests, + arg: { kubeNamespace, kubeConfigPath }, + errMessage: 'Failed to run cluster test', + successMessage: 'Cluster acceptance tests ran successfully', + installationEvent: installationProgress.events.ACCEPTANCE_TESTS_RAN, + }); } - if (token) { - // Create a new context and switch to that context - const createContextOptions = { - apiKey: token, - name: INSTALLATION_DEFAULTS.CF_CONTEXT_NAME, - url, - }; - const [err] = await to(createContext.handler(createContextOptions)); - await handleError(err, 'Failed to use the provided token'); - const config = await getConfigForSdk(); - await sdk.configure(config); - console.log(`A Codefresh context named '${INSTALLATION_DEFAULTS.CF_CONTEXT_NAME}' was added to your "cfconfig" file.`); - } + // get suggested reName + const runtimeName = await newRuntimeName(kubeContextName, kubeNamespace); // Install runner and runtime - const agentInstallOptions = { - name, - 'kube-context-name': kubeContextName, - 'kube-node-selector': kubeNodeSelector, - 'kube-namespace': kubeNamespace, - tolerations, - 'kube-config-path': kubeConfigPath, - 'install-runtime': true, - verbose, - 'make-default-runtime': shouldMakeDefaultRe, - 'storage-class-name': storageClassName, - terminateProcess: false, - 'set-value': setValue, - 'set-file': setFile, - 'agent-kube-context-name': kubeContextName, - 'agent-kube-namespace': kubeNamespace, - }; - const [runnerErr, runtimeName] = await to(installAgent.handler(agentInstallOptions)); - await handleError(runnerErr, 'Runner installation failed', progressReporter, installationProgress.events.RUNNER_INSTALLED); - await to(progressReporter.report(installationProgress.events.RUNNER_INSTALLED, installationProgress.status.SUCCESS)); + installationPlan.addStep({ + name: 'Install agent and runtime', + func: installAgent.handler, + arg: { + name, + 'runtime-name': runtimeName, + 'kube-context-name': kubeContextName, + 'kube-node-selector': kubeNodeSelector, + 'kube-namespace': kubeNamespace, + tolerations, + 'kube-config-path': kubeConfigPath, + 'install-runtime': true, + verbose, + 'make-default-runtime': shouldMakeDefaultRe, + 'storage-class-name': storageClassName, + terminateProcess: false, + 'set-value': setValue, + 'set-file': setFile, + 'agent-kube-context-name': kubeContextName, + 'agent-kube-namespace': kubeNamespace, + }, + errMessage: 'Runner installation failed', + successMessage: 'Successfully installed codefresh runner', + installationEvent: installationProgress.events.RUNNER_INSTALLED, + }); // Install monitoring - const monitorInstallOptions = { - 'kube-config-path': kubeConfigPath, - 'cluster-id': kubeContextName, - 'kube-context-name': kubeContextName, - 'kube-namespace': kubeNamespace, - token: _.get(sdk, 'config.context.token'), - verbose, - noExit: true, // to prevent if from calling inner: process.exit() - }; - const [monitorErr] = await to(installMonitoring.handler(monitorInstallOptions)); - await handleError(monitorErr, 'Monitor installation failed', progressReporter, installationProgress.events.MONITOR_INSTALLED); - await to(progressReporter.report(installationProgress.events.MONITOR_INSTALLED, installationProgress.status.SUCCESS)); + installationPlan.addStep({ + name: 'Install cluster monitoring', + func: installMonitoring.handler, + arg: { + 'kube-config-path': kubeConfigPath, + 'cluster-id': kubeContextName, + 'kube-context-name': kubeContextName, + 'kube-namespace': kubeNamespace, + token: _.get(sdk, 'config.context.token'), + verbose, + noExit: true, // to prevent if from calling inner: process.exit() + }, + errMessage: 'Monitor installation failed', + successMessage: 'Successfully installed cluster monitoring', + installationEvent: installationProgress.events.MONITOR_INSTALLED, + }); // Post Installation if (shouldExecutePipeline) { - await createAndRunTestPipeline(runtimeName, handleError, progressReporter); + const pipelines = await sdk.pipelines.list({ id: `${INSTALLATION_DEFAULTS.PROJECT_NAME}/${INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME}` }); + const testPipelineExists = !!_.get(pipelines, 'docs.length'); + if (!testPipelineExists) { + installationPlan.addStep({ + name: 'Create test pipeline', + func: createTestPipeline, + args: [ + runtimeName, + INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, + ['echo hello codefresh runner!'], + ], + errMessage: 'Failed to create test pipeline', + installationEvent: installationProgress.events.PIPELINE_CREATED, + }); + } else { + installationPlan.addStep({ + name: 'Update test pipeline runtime', + func: updateTestPipelineRuntime, + args: [undefined, runtimeName, INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME], + errMessage: colors.yellow('*warning* could not update test pipeline runtime, you can' + + ' change it manually if you want to run it again on this runtime'), + successMessage: 'Updated test pipeline runtime', + exitOnError: false, + }); + } + installationPlan.addStep({ + name: 'Execute test pipeline', + func: executeTestPipeline, + args: [runtimeName, INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME], + errMessage: 'Failed to execute test pipeline', + installationEvent: installationProgress.events.PIPELINE_EXECUTED, + }); } + await installationPlan.execute(); + console.log(colors.green('\nRunner Status:')); await getAgents.handler({}); console.log(colors.green(`\nGo to ${colors.blue('https://g.codefresh.io/kubernetes/monitor/services')} to view your cluster in codefresh dashbaord`)); diff --git a/lib/interface/cli/commands/hybrid/installation-process.js b/lib/interface/cli/commands/hybrid/installation-process.js index d5d966f96..385f5495d 100644 --- a/lib/interface/cli/commands/hybrid/installation-process.js +++ b/lib/interface/cli/commands/hybrid/installation-process.js @@ -31,6 +31,7 @@ module.exports = { create, buildReporter, events: { + ACCEPTANCE_TESTS_RAN: 'acceptance-tests-ran', RUNNER_INSTALLED: 'runner-installed', MONITOR_INSTALLED: 'monitor-installed', PIPELINE_EXECUTED: 'demo-pipeline-executed', diff --git a/lib/interface/cli/commands/hybrid/migration.js b/lib/interface/cli/commands/hybrid/migration.js index 73c895eba..445416ba8 100644 --- a/lib/interface/cli/commands/hybrid/migration.js +++ b/lib/interface/cli/commands/hybrid/migration.js @@ -41,7 +41,7 @@ async function createAndRunTestPipeline(runtimeName, errHandler) { } const [runPipelineErr] = await to(executeTestPipeline( runtimeName, - testPipeline, + testPipeline.metadata.name, )); await errHandler(runPipelineErr, 'Failed to run test pipeline'); } diff --git a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js index 35102a517..33a751a19 100644 --- a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js +++ b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js @@ -118,6 +118,7 @@ const installRuntimeCmd = new Command({ const { 'storage-class-name': storageClassName, 'agent-name': agentName, + 'runtime-name': reName, 'dry-run': dryRun, 'in-cluster': inCluster, 'kube-node-selector': kubeNodeSelector, @@ -167,7 +168,7 @@ const installRuntimeCmd = new Command({ kubeContextName = getKubeContext(kubeConfigPath); } const clusterName = kubeContextName || getKubeContext(kubeConfigPath); - const runtimeName = await newRuntimeName(kubeContextName, kubeNamespace); + const runtimeName = reName || await newRuntimeName(kubeContextName, kubeNamespace); if (!token) { // eslint-disable-next-line prefer-destructuring From 1dad3ce7861f3b494b5f5f6350fb58c6e859f2c2 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Fri, 5 Jun 2020 14:16:30 +0300 Subject: [PATCH 03/17] wip --- .../cli/commands/hybrid/InstallationPlan.js | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 lib/interface/cli/commands/hybrid/InstallationPlan.js diff --git a/lib/interface/cli/commands/hybrid/InstallationPlan.js b/lib/interface/cli/commands/hybrid/InstallationPlan.js new file mode 100644 index 000000000..f4ace9f21 --- /dev/null +++ b/lib/interface/cli/commands/hybrid/InstallationPlan.js @@ -0,0 +1,72 @@ +/* eslint-disable no-await-in-loop */ +const installationProgress = require('./installation-process'); +const { to } = require('./../../../../logic/cli-config/errors/awaitTo'); +const colors = require('colors'); + +class InstallationPlan { + constructor({ steps, progressReporter, errHandler }) { + this.steps = steps || []; + this.progressReporter = progressReporter; + this.errHandler = errHandler; + } + + addStep({ + name, + func, + args, + arg, + errMessage, + successMessage, + installationEvent, + exitOnError = true, + }) { + this.steps.push({ + name, + func, + args, + arg, + errMessage, + successMessage, + installationEvent, + exitOnError, + }); + } + + async execute() { + while (this.steps.length) { + const { + name, + func, + args, + arg, + errMessage, + successMessage, + installationEvent, + exitOnError, + } = this.steps.shift(); + const _args = args || [arg]; + + if (name) { + console.log(`executing step: ${colors.cyan(name)}`); + } + + const [stepErr] = await to(func(..._args)); + if (stepErr) { + await this.errHandler(stepErr, errMessage, this.progressReporter, installationEvent, exitOnError); + } else { + if (successMessage) { + console.log(successMessage); + } + if (installationEvent) { + await to(this.progressReporter.report(installationEvent, installationProgress.status.SUCCESS)); + } + } + } + } + + getPlan() { + return this.steps; + } +} + +module.exports = InstallationPlan; From 5733e92b46cdec25490eac3fc18194e54269e1f8 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Fri, 5 Jun 2020 14:56:56 +0300 Subject: [PATCH 04/17] fixed pipeline run name --- lib/interface/cli/commands/hybrid/helper.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js index d0acbb70d..6d2f72e07 100644 --- a/lib/interface/cli/commands/hybrid/helper.js +++ b/lib/interface/cli/commands/hybrid/helper.js @@ -132,10 +132,14 @@ async function updateTestPipelineRuntime(pipeline, runtimeName, pipelineName) { async function executeTestPipeline(runtimeName, pipelineName) { const url = _.get(sdk, 'config.context.url', 'https://g.codefresh.io'); + const projectPrefix = `${INSTALLATION_DEFAULTS.PROJECT_NAME}/`; console.log(`${colors.yellow('*NOTE* Running a pipeline for the first time might take longer than usual.')}`); - + let _pipelineName = pipelineName; + if (!pipelineName.includes(projectPrefix)) { + _pipelineName = `${projectPrefix}${pipelineName}`; + } const workflowId = await pipelinesRunCmd.handler({ - name: pipelineName, + name: _pipelineName, exitProcess: false, annotation: [], 'runtime-name': runtimeName, From d0c84ed26be475684740e49ac6d3222a3a35bf1a Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Sun, 7 Jun 2020 18:15:00 +0300 Subject: [PATCH 05/17] wip --- .../cli/commands/hybrid/InstallationPlan.js | 161 ++++++++-- lib/interface/cli/commands/hybrid/helper.js | 222 ++++++++++++++ lib/interface/cli/commands/hybrid/init.cmd.js | 284 +++++++++++++++--- 3 files changed, 589 insertions(+), 78 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/InstallationPlan.js b/lib/interface/cli/commands/hybrid/InstallationPlan.js index f4ace9f21..259a22464 100644 --- a/lib/interface/cli/commands/hybrid/InstallationPlan.js +++ b/lib/interface/cli/commands/hybrid/InstallationPlan.js @@ -1,13 +1,50 @@ /* eslint-disable no-await-in-loop */ +const _ = require('lodash'); const installationProgress = require('./installation-process'); const { to } = require('./../../../../logic/cli-config/errors/awaitTo'); const colors = require('colors'); +const { + resolve, join, +} = require('path'); +const { + homedir, +} = require('os'); +const { + existsSync, mkdirSync, readFileSync, writeFileSync, +} = require('fs'); + +const CODEFRESH_PATH = resolve(homedir(), '.Codefresh'); +const RUNNER_INSTALLATIONS_PATH = join(CODEFRESH_PATH, 'installations'); +const STEP_STATUSES = { + SUCCESS: 'success', + FAILURE: 'failure', + PENDING: 'pending', + SKIPPED: 'skipped', +}; + +function _ensureDirectory(location) { + if (!existsSync(location)) { + mkdirSync(location); + } +} class InstallationPlan { - constructor({ steps, progressReporter, errHandler }) { - this.steps = steps || []; + constructor({ + pendingSteps, + finishedSteps, + progressReporter, + errHandler, + context, + }) { + this.state = { + pendingSteps: pendingSteps || [], + finishedSteps: finishedSteps || [], + context: context || {}, + }; + this.completedSteps = {}; this.progressReporter = progressReporter; this.errHandler = errHandler; + _ensureDirectory(RUNNER_INSTALLATIONS_PATH); } addStep({ @@ -19,22 +56,10 @@ class InstallationPlan { successMessage, installationEvent, exitOnError = true, + condition = true, }) { - this.steps.push({ - name, - func, - args, - arg, - errMessage, - successMessage, - installationEvent, - exitOnError, - }); - } - - async execute() { - while (this.steps.length) { - const { + if (!this.completedSteps[name]) { + this.state.pendingSteps.push({ name, func, args, @@ -43,29 +68,107 @@ class InstallationPlan { successMessage, installationEvent, exitOnError, - } = this.steps.shift(); - const _args = args || [arg]; + condition, + status: STEP_STATUSES.PENDING, + }); + } + } + + reset() { + this.state.finishedSteps.forEach((step) => { + if (step.status === STEP_STATUSES.SUCCESS || step.status === STEP_STATUSES.SKIPPED) { + this.completedSteps[step.name] = step; + } + }); + + this.state.pendingSteps = []; + } + + async execute() { + while (this.state.pendingSteps.length) { + const step = this.state.pendingSteps.shift(); + this.state.finishedSteps.push(step); + const _args = step.args || [step.arg]; + + if (!step.condition || (_.isFunction(step.condition) && !await step.condition())) { + console.log(`skipping step: ${colors.cyan(step.name)}`); + step.status = STEP_STATUSES.SKIPPED; + this._writeState(); + // eslint-disable-next-line no-continue + continue; + } - if (name) { - console.log(`executing step: ${colors.cyan(name)}`); + if (step.name) { + console.log(`executing step: ${colors.cyan(step.name)}`); } - const [stepErr] = await to(func(..._args)); + const [stepErr] = await to(step.func(..._args)); if (stepErr) { - await this.errHandler(stepErr, errMessage, this.progressReporter, installationEvent, exitOnError); + step.status = STEP_STATUSES.FAILURE; + this._writeState(); + await this.errHandler( + stepErr, + step.errMessage || `Failed to ${step.name}`, + this.progressReporter, step.installationEvent, + step.exitOnError, + ); } else { - if (successMessage) { - console.log(successMessage); + step.status = STEP_STATUSES.SUCCESS; + this._writeState(); + if (step.successMessage) { + console.log(step.successMessage); } - if (installationEvent) { - await to(this.progressReporter.report(installationEvent, installationProgress.status.SUCCESS)); + if (step.installationEvent && this.progressReporter) { + await to(this.progressReporter.report(step.installationEvent, installationProgress.status.SUCCESS)); } } } } - getPlan() { - return this.steps; + setProgressReporter(progressReporter) { + this.progressReporter = progressReporter; + } + + setErrorHandler(errHandler) { + this.errHandler = errHandler; + } + + addContext(key, value) { + this.state.context[key] = value; + } + + getContext(key) { + return this.state.context[key]; + } + + _writeState() { + writeFileSync(join(RUNNER_INSTALLATIONS_PATH, 'installation_state.json'), JSON.stringify(this.state)); + } + + static restorePreviousState() { + try { + const data = readFileSync(join(RUNNER_INSTALLATIONS_PATH, 'installation_state.json')); + const oldState = JSON.parse(data.toString('utf-8')); + return new InstallationPlan({ ...oldState }); + } catch (err) { + return undefined; + } + } + + printState() { + this.state.finishedSteps.forEach((step) => { + if (step.status === STEP_STATUSES.SUCCESS) { + console.log(`${colors.green('✓')} ${step.name}`); + } else if (step.status === STEP_STATUSES.SKIPPED) { + console.log(`${colors.cyan('⤳')} ${step.name}`); + } else if (step.status === STEP_STATUSES.FAILURE) { + console.log(`${colors.red('✘')} ${step.name}`); + } + }); + + this.state.pendingSteps.forEach((step) => { + console.log(`${colors.white('◼')} ${step.name}`); + }); } } diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js index 6d2f72e07..090854b57 100644 --- a/lib/interface/cli/commands/hybrid/helper.js +++ b/lib/interface/cli/commands/hybrid/helper.js @@ -21,6 +21,8 @@ const INSTALLATION_DEFAULTS = { DEMO_PIPELINE_NAME: 'Codefresh-Runner Demo', PROJECT_NAME: 'Runner', CF_CONTEXT_NAME: 'cf-runner', + STORAGE_CLASS_PREFIX: 'dind-local-volumes-runner', + RESUME_OLD_INSTALLATION: true, }; const maxRuntimeNameLength = 63; @@ -334,6 +336,200 @@ async function runClusterAcceptanceTests({ kubeNamespace, kubeConfigPath }) { await componentRunner.run(components.venona, cmd); } +async function installAgent({ + apiHost, // --api-host + agentId, // --agnetId + kubeContextName, // kube-context-name + kubeNamespace, // --kube-namespace + token, // --agentToken + kubeNodeSelector, // --kube-node-selector + dryRun, // --dryRun + inCluster, // -inCluster + kubernetesRunnerType, // --kubernetes-runner-type + tolerations, // --tolerations + venonaVersion, // --venona-version + kubeConfigPath, // --kube-config-path + skipVersionCheck, // --skip-version-check + verbose, // --verbose + logFormatting = DefaultLogFormatter, // --log-formtter +}) { + await _downloadVeonona(); + const componentRunner = new Runner(); + const cmd = [ + 'install', + 'agent', + '--agentId', + agentId, + '--api-host', + apiHost, + '--kube-context-name', + kubeContextName, + '--kube-namespace', + kubeNamespace, + '--log-formtter', + logFormatting, + ]; + if (kubeNodeSelector) { + cmd.push('--kube-node-selector'); + cmd.push(kubeNodeSelector); + } + + if (token) { + cmd.push(`--agentToken=${token}`); + } + if (dryRun) { + cmd.push('--dry-run'); + } + if (inCluster) { + cmd.push('--in-cluster'); + } + if (kubernetesRunnerType) { + cmd.push(`--kubernetes-runner-type=${kubernetesRunnerType}`); + } + if (tolerations) { + cmd.push(`--tolerations=${tolerations}`); + } + if (venonaVersion) { + cmd.push(`--venona-version=${venonaVersion}`); + } + if (kubeConfigPath) { + cmd.push(`--kube-config-path=${kubeConfigPath}`); + } + if (skipVersionCheck) { + cmd.push('--skip-version-check'); + } + if (verbose) { + cmd.push('--verbose'); + } + + await componentRunner.run(components.venona, cmd); +} + +async function installRuntime({ + apiHost, // --api-host + token, // --agent-token + kubeContextName, // kube-context-name + kubeNamespace, // --kube-namespace + dryRun, // --dryRun + inCluster, // -inCluster + kubernetesRunnerType, // --kubernetes-runner-type + kubeConfigPath, // --kube-config-path + verbose, // --verbose + name, // --runtimeEnvironmentName + setValue, // --set-value + setFile, // --set-file + kubeNodeSelector, // --kube-node-selector + storageClassName, // --storage-class + logFormatting = DefaultLogFormatter, // --log-formtter +}) { + await _downloadVeonona(); + const componentRunner = new Runner(); + const cmd = [ + 'install', + 'runtime', + '--codefreshToken', + token, + '--api-host', + apiHost, + '--kube-context-name', + kubeContextName, + '--kube-namespace', + kubeNamespace, + '--log-formtter', + logFormatting, + ]; + + if (dryRun) { + cmd.push('--dry-run'); + } + if (inCluster) { + cmd.push('--in-cluster'); + } + if (kubernetesRunnerType) { + cmd.push(`--kubernetes-runner-type=${kubernetesRunnerType}`); + } + if (kubeConfigPath) { + cmd.push(`--kube-config-path=${kubeConfigPath}`); + } + if (verbose) { + cmd.push('--verbose'); + } + if (setValue) { + cmd.push(`--set-value=${setValue}`); + } + if (setFile) { + cmd.push(`--set-file=${setFile}`); + } + if (kubeNodeSelector) { + cmd.push(`--kube-node-selector=${kubeNodeSelector}`); + } + if (name) { + cmd.push('--runtimeName'); + cmd.push(name); + } + if (storageClassName) { + cmd.push(`--storage-class=${storageClassName}`); + } + + await componentRunner.run(components.venona, cmd); +} + +async function attachRuntime({ + kubeServiceAccount, // kube-service-account + kubeContextName, // kube-context-name + kubeNamespace, // --kube-namespace + kubeConfigPath, // --kube-config-path + agentKubeContextName, // --kube-context-name-agent + agentKubeNamespace, // --kube-namespace-agent + agentKubeConfigPath, // --kube-config-path-agent + restartAgent, // --restart-agent + verbose, // --verbose + runtimeName, // --runtimeName + logFormatting = DefaultLogFormatter, // --log-formtter +}) { + await _downloadVeonona(); + const componentRunner = new Runner(); + const cmd = [ + 'attach', + '--kube-context-name', + kubeContextName, + '--kube-namespace', + kubeNamespace, + '--runtime-name', + runtimeName, + '--log-formtter', + logFormatting, + ]; + + if (kubeServiceAccount) { + cmd.push(`--kube-service-account=${kubeServiceAccount}`); + } + + if (kubeConfigPath) { + cmd.push(`--kube-config-path=${kubeConfigPath}`); + } + if (agentKubeContextName) { + cmd.push(`--kube-context-name-agent=${agentKubeContextName}`); + } + if (agentKubeNamespace) { + cmd.push(`--kube-namespace-agent=${agentKubeNamespace}`); + } + if (agentKubeConfigPath) { + cmd.push(`--kube-config-path-agent=${agentKubeConfigPath}`); + } + if (verbose) { + cmd.push('--verbose'); + } + if (restartAgent) { + cmd.push('--restart-agent'); + } + if (logFormatting) { + cmd.push(`--log-formtter=${logFormatting}`); + } + + await componentRunner.run(components.venona, cmd); +} + async function newRuntimeName(kubeContextName, kubeNamespace) { const defaultName = `${kubeContextName}/${kubeNamespace}`.slice(0, maxRuntimeNameLength); const runtimes = await sdk.runtimeEnvs.list({ }); @@ -359,6 +555,28 @@ async function newRuntimeName(kubeContextName, kubeNamespace) { return name; } +async function newAgentName(kubeContextName, kubeNamespace, agents) { + const defaultName = `${kubeContextName}_${kubeNamespace}`; + if (!agents) { + // eslint-disable-next-line no-param-reassign + agents = await sdk.agents.list({ }); + } + let name; + + if (!_.isArray(agents) || !_.find(agents, a => a.name === defaultName)) { + name = defaultName; // use the default name if there are no collisions + } else { + const agentsNames = new Set(_.map(agents, a => a.name)); // for fast lookup + let i = 1; + while (agentsNames.has(`${defaultName}_${i}`)) { + i += 1; + } + name = `${defaultName}_${i}`; + } + + return name; +} + module.exports = { getRelatedAgents, createErrorHandler, @@ -375,7 +593,11 @@ module.exports = { getDefaultRuntime, getRecommendedKubeNamespace, runClusterAcceptanceTests, + installAgent, + installRuntime, + attachRuntime, newRuntimeName, + newAgentName, INSTALLATION_DEFAULTS, DefaultLogFormatter, }; diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index 0d1762ac3..f0fc1f657 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -5,8 +5,8 @@ const inquirer = require('inquirer'); const colors = require('colors'); const _ = require('lodash'); const { getAllKubeContexts, getKubeContext } = require('../../helpers/kubernetes'); -const installAgent = require('../agent/install.cmd'); const installMonitoring = require('../monitor/install.cmd'); +const createClusterCmd = require('../cluster/create.cmd'); const createContext = require('../auth/create-context.cmd'); const getAgents = require('../agent/get.cmd'); const { getConfigForSdk } = require('../../commad-line-interface'); @@ -23,7 +23,11 @@ const { getDefaultRuntime, getRecommendedKubeNamespace, runClusterAcceptanceTests, + installAgent, + installRuntime, + attachRuntime, newRuntimeName, + newAgentName, INSTALLATION_DEFAULTS, } = require('./helper'); const InstallationPlan = require('./InstallationPlan'); @@ -122,6 +126,27 @@ const initCmd = new Command({ describe: 'Print logs', }), handler: async (argv) => { + let resumedInstallation = false; + + const oldInstallationPlan = InstallationPlan.restorePreviousState(); + if (oldInstallationPlan) { + console.log(colors.cyan('Previous installation state:')); + oldInstallationPlan.printState(); + const answer = await inquirer.prompt({ + type: 'confirm', + name: 'resumeInstallation', + default: INSTALLATION_DEFAULTS.RESUME_OLD_INSTALLATION, + message: 'Detected previous incomplete installation, do you want to resume this installation?', + }); + resumedInstallation = answer.resumeInstallation; + } + + let _argv = argv; + if (resumedInstallation) { + _argv = oldInstallationPlan.getContext('argv'); // restore previous installation environment + oldInstallationPlan.reset(); + } + const { 'kube-node-selector': kubeNodeSelector, tolerations, @@ -134,17 +159,18 @@ const initCmd = new Command({ 'set-value': setValue, 'set-file': setFile, 'skip-cluster-test': skipClusterTest, - } = argv; + } = _argv; let { 'kube-context-name': kubeContextName, 'kube-namespace': kubeNamespace, 'set-default-runtime': shouldMakeDefaultRe, 'exec-demo-pipeline': shouldExecutePipeline, - } = argv; + } = _argv; + if (_.get(sdk, 'config.context.isNoAuth') && !token) { console.log('Not authenticated as a Codefresh account: '); console.log('In order to install a Codefresh Runner you need to provide ' + - `an authentication token which can be generated here: ${colors.blue(`${argv.url}/user/settings`)}` + + `an authentication token which can be generated here: ${colors.blue(`${_argv.url}/user/settings`)}` + '\nAfter getting the token you may run this command again with the [--token] option or use the \'codefresh auth\' command to create an authenticated context.'); process.exit(1); } @@ -155,7 +181,7 @@ const initCmd = new Command({ kubeNamespace = await getRecommendedKubeNamespace(kubeConfigPath, kubeContextName); shouldMakeDefaultRe = INSTALLATION_DEFAULTS.MAKE_DEFAULT_RE; shouldExecutePipeline = INSTALLATION_DEFAULTS.RUN_DEMO_PIPELINE; - } else { + } else if (!resumedInstallation) { console.log(colors.green('This installer will guide you through the Codefresh Runner installation process')); if (!kubeContextName) { const contexts = getAllKubeContexts(kubeConfigPath); @@ -246,54 +272,206 @@ const initCmd = new Command({ }, })); const progressReporter = installationProgress.buildReporter(sdk['runner-installation'], progress); - const installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); + + let installationPlan; + if (resumedInstallation) { + installationPlan = oldInstallationPlan; + } else { + installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); + } + + // save the answers for backup + _argv['kube-context-name'] = kubeContextName; + _argv['kube-namespace'] = kubeNamespace; + _argv['set-default-runtime'] = shouldMakeDefaultRe; + _argv['exec-demo-pipeline'] = shouldExecutePipeline; + installationPlan.addContext('argv', _argv); if (skipClusterTest) { console.log('Skipping cluster requirements tests...'); } else { installationPlan.addStep({ - name: 'Cluster acceptance tests', + name: 'run cluster acceptance tests', func: runClusterAcceptanceTests, arg: { kubeNamespace, kubeConfigPath }, errMessage: 'Failed to run cluster test', - successMessage: 'Cluster acceptance tests ran successfully', installationEvent: installationProgress.events.ACCEPTANCE_TESTS_RAN, }); } - // get suggested reName - const runtimeName = await newRuntimeName(kubeContextName, kubeNamespace); + // generate new agent name + installationPlan.addContext('agentName', name); + installationPlan.addStep({ + name: 'generate new agent name', + func: async (contextName, namespace) => { + const agentName = await newAgentName(contextName, namespace); + installationPlan.addContext('agentName', agentName); + }, + args: [kubeContextName, kubeNamespace], + condition: async () => !installationPlan.getContext('agentName'), + }); - // Install runner and runtime + // create new agent installationPlan.addStep({ - name: 'Install agent and runtime', - func: installAgent.handler, - arg: { - name, - 'runtime-name': runtimeName, - 'kube-context-name': kubeContextName, - 'kube-node-selector': kubeNodeSelector, - 'kube-namespace': kubeNamespace, - tolerations, - 'kube-config-path': kubeConfigPath, - 'install-runtime': true, - verbose, - 'make-default-runtime': shouldMakeDefaultRe, - 'storage-class-name': storageClassName, - terminateProcess: false, - 'set-value': setValue, - 'set-file': setFile, - 'agent-kube-context-name': kubeContextName, - 'agent-kube-namespace': kubeNamespace, + name: 'create new agent', + func: async () => { + const agentName = installationPlan.getContext('agentName'); + const agent = await sdk.agents.create({ name: agentName }); + installationPlan.addContext('agent', agent); + const { token: agentToken } = agent; + console.log(`A Codefresh Runner with the name: ${colors.cyan(agentName)} has been created.` + + `\n${colors.yellow('*IMPORTANT*')} Make sure to copy your access token now and st` + + 'ore it in a safe location. You won’t be able to see it again.'); + console.log(agentToken); + installationPlan.addContext('agentToken', agentToken); + }, + condition: async () => !installationPlan.getContext('agentToken'), + }); + + // install agent + installationPlan.addStep({ + name: 'install new agent', + func: async () => { + await installAgent({ + apiHost: sdk.config.context.url, + kubeContextName, + kubeNamespace, + token: installationPlan.getContext('agentToken'), + kubeNodeSelector, + tolerations, + kubeConfigPath, + verbose, + agentId: installationPlan.getContext('agentName'), + }); + }, + }); + + // generate new runtime name + installationPlan.addStep({ + name: 'generate new runtime name', + func: async (contextName, namespace) => { + const reName = await newRuntimeName(contextName, namespace); + installationPlan.addContext('runtimeName', reName); + }, + args: [kubeContextName, kubeNamespace], + condition: async () => !installationPlan.getContext('runtimeName'), + }); + + // create new runtime + installationPlan.addStep({ + name: 'create new runtime', + func: async () => { + await sdk.cluster.create({ + namespace: kubeNamespace, + storageClassName: storageClassName || `${INSTALLATION_DEFAULTS.STORAGE_CLASS_PREFIX}-${kubeNamespace}`, + clusterName: kubeContextName, + runtimeEnvironmentName: installationPlan.getContext('runtimeName'), + agent: true, + }); + }, + }); + + // set runtime as default + installationPlan.addStep({ + name: 'set new runtime as default', + func: async () => { + const reName = installationPlan.getContext('runtimeName'); + const re = await sdk.runtimeEnvs.get({ + name: reName, + }); + await sdk.runtimeEnvs.setDefault({ account: re.accountId, name: re.metadata.name }); + console.log(`Runtime environment "${colors.cyan(reName)}" has been set as the default runtime`); + }, + condition: shouldMakeDefaultRe, + }); + + // add cluster integration + installationPlan.addStep({ + name: 'add cluster integration', + func: async () => { + await createClusterCmd.handler({ + 'kube-context': kubeContextName, + namespace: kubeNamespace, + 'behind-firewall': true, + serviceaccount: 'default', + terminateProcess: false, + }); + }, + condition: async () => { + const clusters = await sdk.clusters.list() || []; + if (clusters.find(cluster => cluster.selector === kubeContextName)) { + return false; // cluster already exists + } + return true; + }, + }); + + // install runtime on cluster + installationPlan.addStep({ + name: 'install runtime', + func: async () => { + await installRuntime({ + apiHost: sdk.config.context.url, + name: installationPlan.getContext('runtimeName'), + kubeContextName, + kubeNamespace, + token: sdk.config.context.token, + kubeConfigPath, + verbose, + kubeNodeSelector, + setValue, + setFile, + storageClassName, + }); + }, + }); + + // update agent with new runtime + installationPlan.addStep({ + name: 'update agent with new runtime', + func: async () => { + const reName = installationPlan.getContext('runtimeName'); + const agent = installationPlan.getContext('agent'); + const rt = await sdk.runtimeEnvs.get({ name: reName }); + if (!rt) { + throw new Error(`runtime ${reName} does not exist on the account`); + } + if (!rt.metadata.agent) { + throw new Error('cannot attach non hybrid runtime'); + } + const runtimes = _.get(agent, 'runtimes', []); + const existingRT = _.find(runtimes, value => value === reName); + if (!existingRT) { + runtimes.push(reName); + await sdk.agents.update({ agentId: agent.id, runtimes }); + } else { + throw new Error(`Runtime ${name} already attached`); + } + }, + }); + + // attach runtime to agent + installationPlan.addStep({ + name: 'attach runtime to agent', + func: async () => { + await attachRuntime({ + kubeContextName, // kube-context-name + kubeNamespace, // --kube-namespace + kubeConfigPath, // --kube-config-path + agentKubeContextName: kubeContextName, // --kube-context-name-agent + agentKubeNamespace: kubeNamespace, // --kube-namespace-agent + agentKubeConfigPath: kubeConfigPath, // --kube-config-path-agent + restartAgent: true, // --restart-agent + verbose, // --verbose + runtimeName: installationPlan.getContext('runtimeName'), // --runtimeName + }); }, - errMessage: 'Runner installation failed', - successMessage: 'Successfully installed codefresh runner', installationEvent: installationProgress.events.RUNNER_INSTALLED, }); - // Install monitoring + // install monitoring installationPlan.addStep({ - name: 'Install cluster monitoring', + name: 'install cluster monitoring', func: installMonitoring.handler, arg: { 'kube-config-path': kubeConfigPath, @@ -304,7 +482,6 @@ const initCmd = new Command({ verbose, noExit: true, // to prevent if from calling inner: process.exit() }, - errMessage: 'Monitor installation failed', successMessage: 'Successfully installed cluster monitoring', installationEvent: installationProgress.events.MONITOR_INSTALLED, }); @@ -315,21 +492,26 @@ const initCmd = new Command({ const testPipelineExists = !!_.get(pipelines, 'docs.length'); if (!testPipelineExists) { installationPlan.addStep({ - name: 'Create test pipeline', - func: createTestPipeline, - args: [ - runtimeName, - INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, - ['echo hello codefresh runner!'], - ], - errMessage: 'Failed to create test pipeline', + name: 'create test pipeline', + func: async () => { + await createTestPipeline( + installationPlan.getContext('runtimeName'), + INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, + ['echo hello codefresh runner!'], + ); + }, installationEvent: installationProgress.events.PIPELINE_CREATED, }); } else { installationPlan.addStep({ - name: 'Update test pipeline runtime', - func: updateTestPipelineRuntime, - args: [undefined, runtimeName, INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME], + name: 'update test pipeline runtime', + func: async () => { + await updateTestPipelineRuntime( + undefined, + installationPlan.getContext('runtimeName'), + INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, + ); + }, errMessage: colors.yellow('*warning* could not update test pipeline runtime, you can' + ' change it manually if you want to run it again on this runtime'), successMessage: 'Updated test pipeline runtime', @@ -337,9 +519,13 @@ const initCmd = new Command({ }); } installationPlan.addStep({ - name: 'Execute test pipeline', - func: executeTestPipeline, - args: [runtimeName, INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME], + name: 'execute test pipeline', + func: async () => { + await executeTestPipeline( + installationPlan.getContext('runtimeName'), + INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME, + ); + }, errMessage: 'Failed to execute test pipeline', installationEvent: installationProgress.events.PIPELINE_EXECUTED, }); From 4b2d5703b329b122e1196d25b467e0d9963055d3 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Mon, 8 Jun 2020 13:39:42 +0300 Subject: [PATCH 06/17] wip --- .../cli/commands/hybrid/InstallationPlan.js | 19 ++++++++++++++++--- lib/interface/cli/commands/hybrid/init.cmd.js | 4 +++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/InstallationPlan.js b/lib/interface/cli/commands/hybrid/InstallationPlan.js index 259a22464..374bfe78f 100644 --- a/lib/interface/cli/commands/hybrid/InstallationPlan.js +++ b/lib/interface/cli/commands/hybrid/InstallationPlan.js @@ -10,11 +10,12 @@ const { homedir, } = require('os'); const { - existsSync, mkdirSync, readFileSync, writeFileSync, + existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, } = require('fs'); const CODEFRESH_PATH = resolve(homedir(), '.Codefresh'); const RUNNER_INSTALLATIONS_PATH = join(CODEFRESH_PATH, 'installations'); +const INSTALLATION_STATE_FILE_PATH = join(RUNNER_INSTALLATIONS_PATH, 'installation_state.json'); const STEP_STATUSES = { SUCCESS: 'success', FAILURE: 'failure', @@ -75,10 +76,12 @@ class InstallationPlan { } reset() { - this.state.finishedSteps.forEach((step) => { + this.state.finishedSteps = this.state.finishedSteps.filter((step) => { if (step.status === STEP_STATUSES.SUCCESS || step.status === STEP_STATUSES.SKIPPED) { this.completedSteps[step.name] = step; + return true; } + return false; }); this.state.pendingSteps = []; @@ -123,6 +126,8 @@ class InstallationPlan { } } } + + InstallationPlan._cleanup(); } setProgressReporter(progressReporter) { @@ -142,7 +147,15 @@ class InstallationPlan { } _writeState() { - writeFileSync(join(RUNNER_INSTALLATIONS_PATH, 'installation_state.json'), JSON.stringify(this.state)); + writeFileSync(INSTALLATION_STATE_FILE_PATH, JSON.stringify(this.state)); + } + + static _cleanup() { + try { + unlinkSync(INSTALLATION_STATE_FILE_PATH); + } catch (err) { + console.warn('** could not delete installation state file **'); + } } static restorePreviousState() { diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index f0fc1f657..cc51863a1 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -143,7 +143,7 @@ const initCmd = new Command({ let _argv = argv; if (resumedInstallation) { - _argv = oldInstallationPlan.getContext('argv'); // restore previous installation environment + _argv = Object.assign(oldInstallationPlan.getContext('argv'), _argv); // restore previous installation environment oldInstallationPlan.reset(); } @@ -276,6 +276,8 @@ const initCmd = new Command({ let installationPlan; if (resumedInstallation) { installationPlan = oldInstallationPlan; + installationPlan.setErrorHandler(handleError); + installationPlan.setProgressReporter(progressReporter); } else { installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); } From 4ab17cbfcd1c7b6b76c9fab7213d9aceacf176da Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Mon, 8 Jun 2020 19:51:13 +0300 Subject: [PATCH 07/17] added build-node-selector --- lib/interface/cli/commands/hybrid/helper.js | 17 +++++++++++++++++ lib/interface/cli/commands/hybrid/init.cmd.js | 16 +++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js index 090854b57..b4078e533 100644 --- a/lib/interface/cli/commands/hybrid/helper.js +++ b/lib/interface/cli/commands/hybrid/helper.js @@ -577,6 +577,22 @@ async function newAgentName(kubeContextName, kubeNamespace, agents) { return name; } +function parseNodeSelector(nodeSelectorStr) { + if (nodeSelectorStr) { + const kubeNodeSelectorObj = {}; + const nsSplitParts = nodeSelectorStr.split(','); + nsSplitParts.forEach((nsPart) => { + const nsRecordSplit = nsPart.split('='); + if (nsRecordSplit.length !== 2) { + throw new Error('invalid kube-node-selector parameter'); + } + // eslint-disable-next-line prefer-destructuring + kubeNodeSelectorObj[nsRecordSplit[0]] = nsRecordSplit[1]; + }); + return kubeNodeSelectorObj; + } +} + module.exports = { getRelatedAgents, createErrorHandler, @@ -598,6 +614,7 @@ module.exports = { attachRuntime, newRuntimeName, newAgentName, + parseNodeSelector, INSTALLATION_DEFAULTS, DefaultLogFormatter, }; diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index cc51863a1..c042a1687 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -28,6 +28,7 @@ const { attachRuntime, newRuntimeName, newAgentName, + parseNodeSelector, INSTALLATION_DEFAULTS, } = require('./helper'); const InstallationPlan = require('./InstallationPlan'); @@ -87,6 +88,9 @@ const initCmd = new Command({ .option('kube-node-selector', { describe: 'The kubernetes node selector "key=value" to be used by venona build resources (default is no node selector) (string)', }) + .option('build-node-selector', { + describe: 'The kubernetes node selector "key=value" to be used by the codefresh build resources (default is no node selector)', + }) .option('yes', { describe: 'Use installation defaults (don\'t ask any questions)', alias: 'y', @@ -144,11 +148,11 @@ const initCmd = new Command({ let _argv = argv; if (resumedInstallation) { _argv = Object.assign(oldInstallationPlan.getContext('argv'), _argv); // restore previous installation environment - oldInstallationPlan.reset(); } const { 'kube-node-selector': kubeNodeSelector, + 'build-node-selector': buildNodeSelector, tolerations, 'kube-config-path': kubeConfigPath, 'storage-class-name': storageClassName, @@ -363,13 +367,19 @@ const initCmd = new Command({ installationPlan.addStep({ name: 'create new runtime', func: async () => { - await sdk.cluster.create({ + const runtimeCreateOpt = { namespace: kubeNamespace, storageClassName: storageClassName || `${INSTALLATION_DEFAULTS.STORAGE_CLASS_PREFIX}-${kubeNamespace}`, clusterName: kubeContextName, runtimeEnvironmentName: installationPlan.getContext('runtimeName'), agent: true, - }); + }; + + if (buildNodeSelector) { + runtimeCreateOpt.nodeSelector = parseNodeSelector(buildNodeSelector); + } + + await sdk.cluster.create(runtimeCreateOpt); }, }); From 7dade20d5e879fe47ea5483705cfaacda2405652 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 14:45:51 +0300 Subject: [PATCH 08/17] wip --- lib/interface/cli/commands/hybrid/migration.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/migration.js b/lib/interface/cli/commands/hybrid/migration.js index 445416ba8..ae5ee1a59 100644 --- a/lib/interface/cli/commands/hybrid/migration.js +++ b/lib/interface/cli/commands/hybrid/migration.js @@ -81,7 +81,7 @@ async function migrate({ process.exit(1); } - // delete old agent + // delete old agent and runtime console.log(`Running migration script on runtime: ${colors.cyan(runtimeName)}`); const [migrateScriptErr, migrateScriptExitCode] = await to(sdk.agents.migrate({ kubeContextName, diff --git a/package.json b/package.json index 5923b9d15..3e82014c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.68.3", + "version": "0.68.4", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true, From 2b9698ff6f0d0f620edd9ecc42fc0e23cd986a2d Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 14:46:34 +0300 Subject: [PATCH 09/17] wip --- lib/interface/cli/commands/hybrid/InstallationPlan.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/InstallationPlan.js b/lib/interface/cli/commands/hybrid/InstallationPlan.js index 76ee3f097..ff40ece5e 100644 --- a/lib/interface/cli/commands/hybrid/InstallationPlan.js +++ b/lib/interface/cli/commands/hybrid/InstallationPlan.js @@ -42,11 +42,7 @@ class InstallationPlan { finishedSteps: finishedSteps || [], context: context || {}, }; -<<<<<<< HEAD - this.completedSteps = {}; -======= this.completedSteps = {}; // used for fast lookup of completed steps ->>>>>>> e4939d36e6684e606b50a3d9441d20d4e4793de8 this.progressReporter = progressReporter; this.errHandler = errHandler; _ensureDirectory(RUNNER_INSTALLATIONS_PATH); From c920a6100aeeff7f1ed642b94893d200d5f17917 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 14:50:47 +0300 Subject: [PATCH 10/17] wip --- lib/interface/cli/commands/hybrid/migration.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/migration.js b/lib/interface/cli/commands/hybrid/migration.js index ae5ee1a59..95d67a188 100644 --- a/lib/interface/cli/commands/hybrid/migration.js +++ b/lib/interface/cli/commands/hybrid/migration.js @@ -113,8 +113,8 @@ async function migrate({ oldConfig.nodeSelector = `${key}=${oldConfig.nodeSelector[key]}`; } - // install new agent - console.log(`Creating new codefresh agent with name: ${colors.cyan(newAgentName)}`); + // install new agent and runtime + console.log(`Creating new codefresh runner with name: ${colors.cyan(newAgentName)}`); const agentInstallOptions = { name: newAgentName, 'kube-context-name': kubeContextName, @@ -122,7 +122,8 @@ async function migrate({ 'kube-namespace': kubeNamespace, tolerations: JSON.stringify(oldConfig.tolerations), 'kube-config-path': kubeConfigPath, - 'install-runtime': false, + 'install-runtime': true, + 'runtime-name': runtimeName, verbose, 'make-default-runtime': shouldMakeDefaultRe, 'storage-class-name': storageClassName, From c41bf9440251063fa1f3b50437c1f8e80842ba68 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 15:55:10 +0300 Subject: [PATCH 11/17] wip --- .../cli/commands/agent/install.cmd.js | 2 + .../cli/commands/hybrid/migration.js | 21 +---- .../runtimeEnvironments/install.cmd.js | 80 ++++++++++--------- 3 files changed, 45 insertions(+), 58 deletions(-) diff --git a/lib/interface/cli/commands/agent/install.cmd.js b/lib/interface/cli/commands/agent/install.cmd.js index 96d0cb9b4..652cefdd9 100644 --- a/lib/interface/cli/commands/agent/install.cmd.js +++ b/lib/interface/cli/commands/agent/install.cmd.js @@ -84,6 +84,7 @@ const installAgentCmd = new Command({ } = argv; const { 'runtime-name': reName, + 'skip-re-creation': skipRuntimeCreation, 'kube-node-selector': kubeNodeSelector, 'dry-run': dryRun, 'in-cluster': inCluster, @@ -182,6 +183,7 @@ const installAgentCmd = new Command({ if (installRuntime) { return installRuntimeCmd.handler({ 'runtime-name': reName, + 'skip-re-creation': skipRuntimeCreation, 'runtime-kube-context-name': kubeContextName, 'runtime-kube-namespace': kubeNamespace, 'agent-name': name, diff --git a/lib/interface/cli/commands/hybrid/migration.js b/lib/interface/cli/commands/hybrid/migration.js index 95d67a188..546aa6b0d 100644 --- a/lib/interface/cli/commands/hybrid/migration.js +++ b/lib/interface/cli/commands/hybrid/migration.js @@ -124,6 +124,7 @@ async function migrate({ 'kube-config-path': kubeConfigPath, 'install-runtime': true, 'runtime-name': runtimeName, + 'skip-re-creation': true, verbose, 'make-default-runtime': shouldMakeDefaultRe, 'storage-class-name': storageClassName, @@ -132,25 +133,7 @@ async function migrate({ 'set-file': setFile, }; const [agentInstallErr] = await to(installAgent.handler(agentInstallOptions)); - handleError(agentInstallErr, 'Failed to install new agent'); - - // attach old runtime to new agent - console.log(`Attaching runtime: ${colors.cyan(runtimeName)} to agent: ${colors.cyan(newAgentName)}`); - const [attachRuntimeErr] = await to(attachRuntime.handler({ - 'agent-name': newAgentName, - 'runtime-name': runtimeName, - 'runtime-kube-context-name': kubeContextName, - 'runtime-kube-namespace': kubeNamespace, - 'runtime-kube-serviceaccount': 'venona', - 'runtime-kube-config-path': kubeConfigPath, - 'agent-kube-context-name': kubeContextName, - 'agent-kube-namespace': kubeNamespace, - 'agent-kube-config-path': kubeConfigPath, - 'restart-agent': true, - terminateProcess: false, - verbose, - })); - handleError(attachRuntimeErr, 'Failed to attach the old runtime to the new agent'); + handleError(agentInstallErr, 'Failed to install new agent and runtime'); // Install new monitoring components console.log('Installing monitoring components'); diff --git a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js index 33a751a19..133d5e6ff 100644 --- a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js +++ b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js @@ -119,6 +119,7 @@ const installRuntimeCmd = new Command({ 'storage-class-name': storageClassName, 'agent-name': agentName, 'runtime-name': reName, + 'skip-re-creation': skipRuntimeCreation, 'dry-run': dryRun, 'in-cluster': inCluster, 'kube-node-selector': kubeNodeSelector, @@ -176,54 +177,55 @@ const installRuntimeCmd = new Command({ } // create RE in codefresh - await sdk.cluster.create({ - namespace: kubeNamespace, - storageClassName: storageClassName || `${defaultStorageClassPrefix}-${kubeNamespace}`, - runnerType: kubernetesRunnerType, - nodeSelector: kubeNodeSelectorObj, - annotations: buildAnnotations, - clusterName, - runtimeEnvironmentName: runtimeName, - agent: true, - }); - console.log(`Runtime environment "${colors.cyan(runtimeName)}" has been created`); - if (shouldMakeDefaultRe) { - const re = await sdk.runtimeEnvs.get({ - name: runtimeName, + if (!skipRuntimeCreation) { + await sdk.cluster.create({ + namespace: kubeNamespace, + storageClassName: storageClassName || `${defaultStorageClassPrefix}-${kubeNamespace}`, + runnerType: kubernetesRunnerType, + nodeSelector: kubeNodeSelectorObj, + annotations: buildAnnotations, + clusterName, + runtimeEnvironmentName: runtimeName, + agent: true, }); - await sdk.runtimeEnvs.setDefault({ account: re.accountId, name: re.metadata.name }); - console.log(`Runtime environment "${colors.cyan(runtimeName)}" has been set as the default runtime`); - } - - // check if cluster already exists - let clusterExists = false; - try { - const clusters = await sdk.clusters.list() || []; - if (clusters.find(cluster => cluster.selector === kubeContextName)) { - clusterExists = true; + console.log(`Runtime environment "${colors.cyan(runtimeName)}" has been created`); + if (shouldMakeDefaultRe) { + const re = await sdk.runtimeEnvs.get({ + name: runtimeName, + }); + await sdk.runtimeEnvs.setDefault({ account: re.accountId, name: re.metadata.name }); + console.log(`Runtime environment "${colors.cyan(runtimeName)}" has been set as the default runtime`); } - } catch (error) { - console.log(`Failed to fetch account clusters, cause: ${error.message}`); - } - // create the cluster in codefresh if does not exists - if (!clusterExists) { - console.log(`Adding cluster "${colors.cyan(kubeContextName)}" integration to your Codefresh account`); + // check if cluster already exists + let clusterExists = false; try { - await createClusterCmd.handler({ - 'kube-context': kubeContextName, - namespace: kubeNamespace, - 'behind-firewall': true, - serviceaccount: clusterServiceAccount || 'default', - terminateProcess: false, - }); + const clusters = await sdk.clusters.list() || []; + if (clusters.find(cluster => cluster.selector === kubeContextName)) { + clusterExists = true; + } } catch (error) { - console.log(`Failed to register cluster on Codefresh, cause: ${error.message}`); + console.log(`Failed to fetch account clusters, cause: ${error.message}`); + } + + // create the cluster in codefresh if does not exists + if (!clusterExists) { + console.log(`Adding cluster "${colors.cyan(kubeContextName)}" integration to your Codefresh account`); + try { + await createClusterCmd.handler({ + 'kube-context': kubeContextName, + namespace: kubeNamespace, + 'behind-firewall': true, + serviceaccount: clusterServiceAccount || 'default', + terminateProcess: false, + }); + } catch (error) { + console.log(`Failed to register cluster on Codefresh, cause: ${error.message}`); + } } } // install RE on cluster - const runtimeEvents = new ProgressEvents(); const runtimeFormat = 'downloading runtime installer [{bar}] {percentage}% | {value}/{total}'; const runtimmrProgressBar = new cliProgress.SingleBar({ stopOnComplete: true, format: runtimeFormat }, cliProgress.Presets.shades_classic); From 172896f48e5adec23f78457e436d18c799d1fbe6 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 15:56:32 +0300 Subject: [PATCH 12/17] wip --- lib/interface/cli/commands/hybrid/init.cmd.js | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index 49b191181..c40aa7cd0 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -136,10 +136,7 @@ const initCmd = new Command({ if (oldInstallationPlan) { console.log(colors.cyan('Previous installation state:')); oldInstallationPlan.printState(); -<<<<<<< HEAD -======= oldInstallationPlan.reset(); // needs to be called after printState() ->>>>>>> e4939d36e6684e606b50a3d9441d20d4e4793de8 const answer = await inquirer.prompt({ type: 'confirm', name: 'resumeInstallation', @@ -257,22 +254,6 @@ const initCmd = new Command({ shouldMakeDefaultRe, shouldExecutePipeline, }); -<<<<<<< HEAD - - if (token) { - // Create a new context and switch to that context - const [err] = await to(createContext.handler({ - apiKey: token, - name: INSTALLATION_DEFAULTS.CF_CONTEXT_NAME, - url, - })); - await handleError(err, 'Failed to use the provided token'); - const config = await getConfigForSdk(); - await sdk.configure(config); - console.log(`A Codefresh context named '${INSTALLATION_DEFAULTS.CF_CONTEXT_NAME}' was added to your "cfconfig" file.`); - } -======= ->>>>>>> e4939d36e6684e606b50a3d9441d20d4e4793de8 const [, progress] = await to(async () => installationProgress.create(sdk['runner-installation'], { options: { @@ -289,43 +270,6 @@ const initCmd = new Command({ installationPlan = oldInstallationPlan; installationPlan.setErrorHandler(handleError); installationPlan.setProgressReporter(progressReporter); -<<<<<<< HEAD - } else { - installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); - } - - // save the answers for backup - _argv['kube-context-name'] = kubeContextName; - _argv['kube-namespace'] = kubeNamespace; - _argv['set-default-runtime'] = shouldMakeDefaultRe; - _argv['exec-demo-pipeline'] = shouldExecutePipeline; - installationPlan.addContext('argv', _argv); - - if (skipClusterTest) { - console.log('Skipping cluster requirements tests...'); - } else { - installationPlan.addStep({ - name: 'run cluster acceptance tests', - func: runClusterAcceptanceTests, - arg: { kubeNamespace, kubeConfigPath }, - errMessage: 'Failed to run cluster test', - installationEvent: installationProgress.events.ACCEPTANCE_TESTS_RAN, - }); - } - - // generate new agent name - installationPlan.addContext('agentName', name); - installationPlan.addStep({ - name: 'generate new agent name', - func: async (contextName, namespace) => { - const agentName = await newAgentName(contextName, namespace); - installationPlan.addContext('agentName', agentName); - }, - args: [kubeContextName, kubeNamespace], - condition: async () => !installationPlan.getContext('agentName'), - }); - -======= } else { installationPlan = new InstallationPlan({ progressReporter, errHandler: handleError }); } @@ -375,7 +319,6 @@ const initCmd = new Command({ condition: async () => !installationPlan.getContext('agentName'), }); ->>>>>>> e4939d36e6684e606b50a3d9441d20d4e4793de8 // create new agent installationPlan.addStep({ name: 'create new agent', From b0efb12eb880af449871ad985bd64f935a2b8f7f Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 18:00:47 +0300 Subject: [PATCH 13/17] wip --- .../cli/commands/agent/install.cmd.js | 2 ++ .../cli/commands/hybrid/migration.js | 19 +++++++++++++++++-- .../runtimeEnvironments/install.cmd.js | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/interface/cli/commands/agent/install.cmd.js b/lib/interface/cli/commands/agent/install.cmd.js index 652cefdd9..d5133141d 100644 --- a/lib/interface/cli/commands/agent/install.cmd.js +++ b/lib/interface/cli/commands/agent/install.cmd.js @@ -86,6 +86,7 @@ const installAgentCmd = new Command({ 'runtime-name': reName, 'skip-re-creation': skipRuntimeCreation, 'kube-node-selector': kubeNodeSelector, + 'build-node-selector': buildNodeSelector, 'dry-run': dryRun, 'in-cluster': inCluster, 'kubernetes-runner-type': kubernetesRunnerType, @@ -191,6 +192,7 @@ const installAgentCmd = new Command({ 'attach-runtime': true, 'restart-agent': true, 'make-default-runtime': shouldMakeDefaultRe, + 'kube-node-selector': buildNodeSelector, 'storage-class-name': storageClassName, 'set-value': setValue, 'set-file': setFile, diff --git a/lib/interface/cli/commands/hybrid/migration.js b/lib/interface/cli/commands/hybrid/migration.js index 546aa6b0d..27413650b 100644 --- a/lib/interface/cli/commands/hybrid/migration.js +++ b/lib/interface/cli/commands/hybrid/migration.js @@ -2,7 +2,6 @@ const _ = require('lodash'); const { to } = require('./../../../../logic/cli-config/errors/awaitTo'); const { sdk } = require('../../../../logic'); const installAgent = require('../agent/install.cmd'); -const attachRuntime = require('../runtimeEnvironments/attach.cmd'); const installMonitoring = require('../monitor/install.cmd'); const colors = require('colors'); const inquirer = require('inquirer'); @@ -61,6 +60,10 @@ async function migrate({ agents, }) { const newAgentName = agentName || await getNewAgentName(kubeContextName, kubeNamespace, agents); + const [getRuntimeErr, runtime] = await to(sdk.runtimeEnvs.get({ name: runtimeName })); + handleError(getRuntimeErr, `Failed to get runtime with name "${runtimeName}"`); + const oldNodeSelector = _.get(runtime, 'runtimeScheduler.cluster.nodeSelector'); + const oldStorageClassName = _.get(runtime, 'dockerDaemonScheduler.pvcs.dind.storageClassName'); // prompt migration process confirmation console.log(`${colors.red('This migration process will do the following:')}`); @@ -81,6 +84,14 @@ async function migrate({ process.exit(1); } + // prepare old runtime + if (oldStorageClassName && oldStorageClassName.startsWith('dind-local-volumes-venona')) { + // need to replace to start with 'dind-local-volumes-runner' + const newRe = _.set(runtime, 'dockerDaemonScheduler.pvcs.dind.storageClassName', oldStorageClassName.replace('venona', 'runner')); + const [err] = await to(sdk.runtimeEnvs.update({ name: runtimeName }, newRe)); + handleError(err, 'Failed to update runtime storage class name'); + } + // delete old agent and runtime console.log(`Running migration script on runtime: ${colors.cyan(runtimeName)}`); const [migrateScriptErr, migrateScriptExitCode] = await to(sdk.agents.migrate({ @@ -119,7 +130,11 @@ async function migrate({ name: newAgentName, 'kube-context-name': kubeContextName, 'kube-node-selector': oldConfig.nodeSelector, + 'build-node-selector': oldNodeSelector, 'kube-namespace': kubeNamespace, + 'agent-kube-namespace': kubeNamespace, + 'agent-kube-context-name': kubeContextName, + 'agent-kube-config-path': kubeConfigPath, tolerations: JSON.stringify(oldConfig.tolerations), 'kube-config-path': kubeConfigPath, 'install-runtime': true, @@ -127,7 +142,7 @@ async function migrate({ 'skip-re-creation': true, verbose, 'make-default-runtime': shouldMakeDefaultRe, - 'storage-class-name': storageClassName, + 'storage-class-name': storageClassName || oldStorageClassName, terminateProcess: false, 'set-value': setValue, 'set-file': setFile, diff --git a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js index 133d5e6ff..2cef1be23 100644 --- a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js +++ b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js @@ -258,7 +258,7 @@ const installRuntimeCmd = new Command({ setFile, terminateProcess: !attachRuntime, events: runtimeEvents, - storageClassName, + storageClassName: storageClassName && storageClassName.startsWith('dind-local-volumes') ? undefined : storageClassName, logFormatting: DefaultLogFormatter, }); // attach RE to agent in codefresh From 2c13dba67438ae84c9c5fc0333130ac996b44331 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 18:07:07 +0300 Subject: [PATCH 14/17] wip --- .../runtimeEnvironments/install.cmd.js | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js index 2cef1be23..dbba3961c 100644 --- a/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js +++ b/lib/interface/cli/commands/runtimeEnvironments/install.cmd.js @@ -196,32 +196,32 @@ const installRuntimeCmd = new Command({ await sdk.runtimeEnvs.setDefault({ account: re.accountId, name: re.metadata.name }); console.log(`Runtime environment "${colors.cyan(runtimeName)}" has been set as the default runtime`); } + } - // check if cluster already exists - let clusterExists = false; - try { - const clusters = await sdk.clusters.list() || []; - if (clusters.find(cluster => cluster.selector === kubeContextName)) { - clusterExists = true; - } - } catch (error) { - console.log(`Failed to fetch account clusters, cause: ${error.message}`); + // check if cluster already exists + let clusterExists = false; + try { + const clusters = await sdk.clusters.list() || []; + if (clusters.find(cluster => cluster.selector === kubeContextName)) { + clusterExists = true; } + } catch (error) { + console.log(`Failed to fetch account clusters, cause: ${error.message}`); + } - // create the cluster in codefresh if does not exists - if (!clusterExists) { - console.log(`Adding cluster "${colors.cyan(kubeContextName)}" integration to your Codefresh account`); - try { - await createClusterCmd.handler({ - 'kube-context': kubeContextName, - namespace: kubeNamespace, - 'behind-firewall': true, - serviceaccount: clusterServiceAccount || 'default', - terminateProcess: false, - }); - } catch (error) { - console.log(`Failed to register cluster on Codefresh, cause: ${error.message}`); - } + // create the cluster in codefresh if does not exists + if (!clusterExists) { + console.log(`Adding cluster "${colors.cyan(kubeContextName)}" integration to your Codefresh account`); + try { + await createClusterCmd.handler({ + 'kube-context': kubeContextName, + namespace: kubeNamespace, + 'behind-firewall': true, + serviceaccount: clusterServiceAccount || 'default', + terminateProcess: false, + }); + } catch (error) { + console.log(`Failed to register cluster on Codefresh, cause: ${error.message}`); } } From cebbe18d9da9269eb2f43939926414df4e7ee002 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 18:15:07 +0300 Subject: [PATCH 15/17] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3e82014c2..14649d822 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.68.4", + "version": "0.68.5", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true, From 6ae3f77e81f8807c9fdaef6060307ff93f841114 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 10 Jun 2020 18:19:23 +0300 Subject: [PATCH 16/17] trigger From 92c51b6b07b5f1e40953569c4e5d9dc7ae1e5947 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Thu, 11 Jun 2020 08:26:39 +0300 Subject: [PATCH 17/17] trigger