From fc2c6e9818e6e6dec42abead185b23c42d87b218 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sun, 26 Jun 2022 15:00:00 +0200 Subject: [PATCH 1/4] Add `husky init` to `devcontainer.json` --- .devcontainer.json | 3 +- .github/workflows/test.yml | 2 + .golangci.yml | 3 - .goreleaser.yml | 6 +- .husky/hooks/pre-commit | 3 + environment/setup.sh | 8 +- go.mod | 7 +- go.sum | 15 +- iterative/resource_task.go | 14 +- .../testdata/script_template_cloud_aws.golden | 8 +- .../script_template_cloud_azure.golden | 8 +- .../testdata/script_template_cloud_gcp.golden | 8 +- .../script_template_cloud_invalid.golden | 8 +- iterative/utils/analytics.go | 405 ++++++++++++++++++ iterative/utils/analytics_test.go | 89 ++++ main.go | 3 + 16 files changed, 560 insertions(+), 30 deletions(-) create mode 100644 .husky/hooks/pre-commit create mode 100644 iterative/utils/analytics.go create mode 100644 iterative/utils/analytics_test.go diff --git a/.devcontainer.json b/.devcontainer.json index 9650257d..6d35ab71 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,7 +1,8 @@ { "name": "terraform", "extensions": ["hashicorp.terraform"], - "postCreateCommand": "/bin/bash -c 'curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && sudo apt-add-repository \"deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main\" && sudo apt-get update && sudo apt-get install packer terraform'", + "postCreateCommand": "go install github.com/hashicorp/terraform@v1 github.com/hashicorp/packer@v1 github.com/automation-co/husky@v0.2 && husky init", + "image": "mcr.microsoft.com/vscode/devcontainers/go:1", "hostRequirements": { "cpus": 4, "memory": "8gb", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 390b34c2..a99acc8a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v2 - uses: golangci/golangci-lint-action@v2 + with: + only-new-issues: true test: runs-on: ubuntu-latest steps: diff --git a/.golangci.yml b/.golangci.yml index eb01cdc0..26d35f94 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,8 +1,5 @@ run: timeout: 10m - skip-dirs: - - iterative - - cml output: format: github-actions severity: diff --git a/.goreleaser.yml b/.goreleaser.yml index 0c7f3835..31a2e035 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -5,7 +5,11 @@ builds: flags: - -trimpath ldflags: - - '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}}' + - -s -w + - -X terraform-provider-iterative/iterative/utils.Version={{.Version}} + - -X main.version={{.Version}} + - -X main.commit={{.Commit}} + - -X main.date={{.Date}} goos: # - freebsd - windows diff --git a/.husky/hooks/pre-commit b/.husky/hooks/pre-commit new file mode 100644 index 00000000..b8aaa327 --- /dev/null +++ b/.husky/hooks/pre-commit @@ -0,0 +1,3 @@ +#!/bin/sh +golangci-lint run +go test -v -short ./... diff --git a/environment/setup.sh b/environment/setup.sh index 41d6d0f8..8f3c7c01 100644 --- a/environment/setup.sh +++ b/environment/setup.sh @@ -1,7 +1,7 @@ #/bin/sh FILE=/var/log/cml_stack.log if [ ! -f "$FILE" ]; then - DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND=noninteractive PS4='tpi:setup.sh: ' set -x echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes @@ -11,15 +11,15 @@ if [ ! -f "$FILE" ]; then sudo add-apt-repository universe -y sudo add-apt-repository ppa:git-core/ppa -y - sudo apt update && sudo apt-get install -y software-properties-common build-essential git + sudo apt update && sudo apt-get install -y software-properties-common build-essential git acpid sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh sudo usermod -aG docker ubuntu sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock get_ecr_helper="curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login" - chmod_ecr_help="chmod 755 /usr/bin/docker-credential-ecr-login" - sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_help && $chmod_ecr_help" + chmod_ecr_helper="chmod 755 /usr/bin/docker-credential-ecr-login" + sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_helper && $chmod_ecr_helper" curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" diff --git a/go.mod b/go.mod index 1700d2ed..35ff1230 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d - github.com/alessio/shellescape v1.4.1 // indirect + github.com/alessio/shellescape v1.4.1 github.com/aohorodnyk/uid v1.1.0 github.com/aws/aws-sdk-go-v2 v1.11.0 github.com/aws/aws-sdk-go-v2/config v1.8.3 @@ -24,13 +24,18 @@ require ( github.com/docker/go-units v0.4.0 github.com/gobwas/glob v0.2.3 github.com/google/go-github/v42 v42.0.0 + github.com/google/go-github/v45 v45.2.0 + github.com/google/uuid v1.3.0 github.com/hashicorp/go-getter v1.5.11 // indirect github.com/hashicorp/hcl/v2 v2.8.0 // indirect github.com/hashicorp/terraform-plugin-sdk/v2 v2.8.0 github.com/rclone/rclone v1.57.0 github.com/sebdah/goldie/v2 v2.5.3 + github.com/shirou/gopsutil v3.21.11+incompatible github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/wessie/appdirs v0.0.0-20141031215813-6573e894f8e2 + github.com/yusufpapurcu/wmi v1.2.2 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f google.golang.org/api v0.54.0 diff --git a/go.sum b/go.sum index 407f550c..c70346c3 100644 --- a/go.sum +++ b/go.sum @@ -369,8 +369,9 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -448,13 +449,16 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v39 v39.0.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec= github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg= +github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= +github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -866,6 +870,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA= github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -972,6 +978,8 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/wessie/appdirs v0.0.0-20141031215813-6573e894f8e2 h1:fKKg46hH3QFC22OrV5dRuXUtpWds4bM+9du/O0xAnHc= +github.com/wessie/appdirs v0.0.0-20141031215813-6573e894f8e2/go.mod h1:c0hW4EpVkKCEjosF5u1Lm+x5twh0RpPx05T9XGlyN8w= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo= @@ -989,6 +997,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= @@ -1378,7 +1388,6 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= diff --git a/iterative/resource_task.go b/iterative/resource_task.go index 349d9776..78bc660d 100644 --- a/iterative/resource_task.go +++ b/iterative/resource_task.go @@ -190,11 +190,11 @@ func resourceTaskCreate(ctx context.Context, d *schema.ResourceData, m interface task, err := resourceTaskBuild(ctx, d, m) if err != nil { + utils.SendJitsuEvent("task/apply", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Error) } d.SetId(task.GetIdentifier(ctx).Long()) - if err := task.Create(ctx); err != nil { diags = diagnostic(diags, err, diag.Error) if err := task.Delete(ctx); err != nil { @@ -205,32 +205,38 @@ func resourceTaskCreate(ctx context.Context, d *schema.ResourceData, m interface } } + utils.SendJitsuEvent("task/apply", err, utils.ResourceData(d)) return } func resourceTaskRead(ctx context.Context, d *schema.ResourceData, m interface{}) (diags diag.Diagnostics) { task, err := resourceTaskBuild(ctx, d, m) if err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } if err := task.Read(ctx); err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } if keyPair, err := task.GetKeyPair(ctx); err != nil { if err != common.NotImplementedError { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } } else { publicKey, err := keyPair.PublicString() if err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } d.Set("ssh_public_key", publicKey) privateKey, err := keyPair.PrivateString() if err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } d.Set("ssh_private_key", privateKey) @@ -255,12 +261,14 @@ func resourceTaskRead(ctx context.Context, d *schema.ResourceData, m interface{} status, err := task.Status(ctx) if err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } d.Set("status", status) logs, err := task.Logs(ctx) if err != nil { + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Warning) } @@ -272,6 +280,7 @@ func resourceTaskRead(ctx context.Context, d *schema.ResourceData, m interface{} logger.Info("logs") logger.Info("status") + utils.SendJitsuEvent("task/read", err, utils.ResourceData(d)) return diags } @@ -280,13 +289,16 @@ func resourceTaskDelete(ctx context.Context, d *schema.ResourceData, m interface task, err := resourceTaskBuild(ctx, d, m) if err != nil { + utils.SendJitsuEvent("task/destroy", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Error) } if err := task.Delete(ctx); err != nil { + utils.SendJitsuEvent("task/destroy", err, utils.ResourceData(d)) return diagnostic(diags, err, diag.Error) } + utils.SendJitsuEvent("task/destroy", err, utils.ResourceData(d)) return } diff --git a/iterative/testdata/script_template_cloud_aws.golden b/iterative/testdata/script_template_cloud_aws.golden index aec12e4b..6312f9bc 100644 --- a/iterative/testdata/script_template_cloud_aws.golden +++ b/iterative/testdata/script_template_cloud_aws.golden @@ -2,7 +2,7 @@ sudo systemctl is-enabled cml.service && return 0 FILE=/var/log/cml_stack.log if [ ! -f "$FILE" ]; then - DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND=noninteractive PS4='tpi:setup.sh: ' set -x echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes @@ -12,15 +12,15 @@ if [ ! -f "$FILE" ]; then sudo add-apt-repository universe -y sudo add-apt-repository ppa:git-core/ppa -y - sudo apt update && sudo apt-get install -y software-properties-common build-essential git + sudo apt update && sudo apt-get install -y software-properties-common build-essential git acpid sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh sudo usermod -aG docker ubuntu sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock get_ecr_helper="curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login" - chmod_ecr_help="chmod 755 /usr/bin/docker-credential-ecr-login" - sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_help && $chmod_ecr_help" + chmod_ecr_helper="chmod 755 /usr/bin/docker-credential-ecr-login" + sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_helper && $chmod_ecr_helper" curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" diff --git a/iterative/testdata/script_template_cloud_azure.golden b/iterative/testdata/script_template_cloud_azure.golden index 4be38a40..23527f46 100644 --- a/iterative/testdata/script_template_cloud_azure.golden +++ b/iterative/testdata/script_template_cloud_azure.golden @@ -2,7 +2,7 @@ sudo systemctl is-enabled cml.service && return 0 FILE=/var/log/cml_stack.log if [ ! -f "$FILE" ]; then - DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND=noninteractive PS4='tpi:setup.sh: ' set -x echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes @@ -12,15 +12,15 @@ if [ ! -f "$FILE" ]; then sudo add-apt-repository universe -y sudo add-apt-repository ppa:git-core/ppa -y - sudo apt update && sudo apt-get install -y software-properties-common build-essential git + sudo apt update && sudo apt-get install -y software-properties-common build-essential git acpid sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh sudo usermod -aG docker ubuntu sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock get_ecr_helper="curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login" - chmod_ecr_help="chmod 755 /usr/bin/docker-credential-ecr-login" - sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_help && $chmod_ecr_help" + chmod_ecr_helper="chmod 755 /usr/bin/docker-credential-ecr-login" + sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_helper && $chmod_ecr_helper" curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" diff --git a/iterative/testdata/script_template_cloud_gcp.golden b/iterative/testdata/script_template_cloud_gcp.golden index c4aad36f..47ab1704 100644 --- a/iterative/testdata/script_template_cloud_gcp.golden +++ b/iterative/testdata/script_template_cloud_gcp.golden @@ -2,7 +2,7 @@ sudo systemctl is-enabled cml.service && return 0 FILE=/var/log/cml_stack.log if [ ! -f "$FILE" ]; then - DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND=noninteractive PS4='tpi:setup.sh: ' set -x echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes @@ -12,15 +12,15 @@ if [ ! -f "$FILE" ]; then sudo add-apt-repository universe -y sudo add-apt-repository ppa:git-core/ppa -y - sudo apt update && sudo apt-get install -y software-properties-common build-essential git + sudo apt update && sudo apt-get install -y software-properties-common build-essential git acpid sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh sudo usermod -aG docker ubuntu sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock get_ecr_helper="curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login" - chmod_ecr_help="chmod 755 /usr/bin/docker-credential-ecr-login" - sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_help && $chmod_ecr_help" + chmod_ecr_helper="chmod 755 /usr/bin/docker-credential-ecr-login" + sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_helper && $chmod_ecr_helper" curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" diff --git a/iterative/testdata/script_template_cloud_invalid.golden b/iterative/testdata/script_template_cloud_invalid.golden index 929040f4..f1b5e3ef 100644 --- a/iterative/testdata/script_template_cloud_invalid.golden +++ b/iterative/testdata/script_template_cloud_invalid.golden @@ -2,7 +2,7 @@ sudo systemctl is-enabled cml.service && return 0 FILE=/var/log/cml_stack.log if [ ! -f "$FILE" ]; then - DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND=noninteractive PS4='tpi:setup.sh: ' set -x echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes @@ -12,15 +12,15 @@ if [ ! -f "$FILE" ]; then sudo add-apt-repository universe -y sudo add-apt-repository ppa:git-core/ppa -y - sudo apt update && sudo apt-get install -y software-properties-common build-essential git + sudo apt update && sudo apt-get install -y software-properties-common build-essential git acpid sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh sudo usermod -aG docker ubuntu sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock get_ecr_helper="curl https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.5.0/linux-amd64/docker-credential-ecr-login --output /usr/bin/docker-credential-ecr-login" - chmod_ecr_help="chmod 755 /usr/bin/docker-credential-ecr-login" - sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_help && $chmod_ecr_help" + chmod_ecr_helper="chmod 755 /usr/bin/docker-credential-ecr-login" + sudo systemd-run --same-dir --no-block --service-type=exec bash -c "$get_ecr_helper && $chmod_ecr_helper" curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" diff --git a/iterative/utils/analytics.go b/iterative/utils/analytics.go new file mode 100644 index 00000000..380cbc49 --- /dev/null +++ b/iterative/utils/analytics.go @@ -0,0 +1,405 @@ +package utils + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime/debug" + "sync" + "time" + + "github.com/google/go-github/v45/github" + "github.com/google/uuid" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/shirou/gopsutil/host" + "github.com/sirupsen/logrus" + "github.com/wessie/appdirs" + + "golang.org/x/crypto/scrypt" +) + +const ( + Timeout = 5 * time.Second + Endpoint = "https://telemetry.cml.dev/api/v1/s2s/event?ip_policy=strict" + Token = "s2s.jtyjusrpsww4k9b76rrjri.bl62fbzrb7nd9n6vn5bpqt" +) + +var ( + Version string = "0.0.0" + wg sync.WaitGroup +) + +func getenv(key, defaultValue string) string { + value := os.Getenv(key) + if len(value) == 0 { + return defaultValue + } + return value +} + +func deterministic(data string) (*uuid.UUID, error) { + ns := uuid.NewSHA1(uuid.NameSpaceDNS, []byte("iterative.ai")) + + seed, err := ns.MarshalBinary() + if err != nil { + return nil, err + } + + dk, err := scrypt.Key([]byte(data), seed, 1<<16, 8, 1, 8) + if err != nil { + return nil, err + } + + id := uuid.NewSHA1(ns, []byte(hex.EncodeToString(dk))) + return &id, nil +} + +func SystemInfo() map[string]interface{} { + hostStat, _ := host.Info() + return map[string]interface{}{ + "os_name": hostStat.OS, + "platform_version": hostStat.PlatformVersion, + } +} + +func TaskDuration(logs string) float64 { + regex := regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}`) + matches := regex.FindAllString(logs, -1) + taskDuration := 0.0 + + if len(matches) > 1 { + layout := "2006-01-02 15:04:05" + t1, _ := time.Parse(layout, matches[len(matches)-1]) + t0, _ := time.Parse(layout, matches[0]) + taskDuration = t1.Sub(t0).Seconds() + } + + return taskDuration +} + +func IsCI() bool { + return len(guessCI()) > 0 +} + +func guessCI() string { + if _, ok := os.LookupEnv("GITHUB_SERVER_URL"); ok { + return "github" + } + + if _, ok := os.LookupEnv("CI_SERVER_URL"); ok { + return "gitlab" + } + + if _, ok := os.LookupEnv("BITBUCKET_WORKSPACE"); ok { + return "bitbucket" + } + + if _, ok := os.LookupEnv("TF_BUILD"); ok { + return "azure" + } + + if _, ok := os.LookupEnv("CI"); ok { + return "unknown" + } + + return "" +} + +func TerraformVersion() (string, error) { + var out bytes.Buffer + cmd := exec.Command("terraform", "--version") + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return "", err + } + + regex := regexp.MustCompile(`v\d+\.\d+\.\d+`) + version := regex.FindString(out.String()) + return version, nil +} + +func GroupId() (string, error) { + if !IsCI() { + return "", nil + } + + rawId := "CI" + ci := guessCI() + + if ci == "github" { + rawId = fmt.Sprintf("%s/%s", + os.Getenv("GITHUB_SERVER_URL"), + os.Getenv("GITHUB_REPOSITORY_OWNER")) + } else if ci == "gitlab" { + rawId = fmt.Sprintf("%s/%s", + os.Getenv("CI_SERVER_URL"), + os.Getenv("CI_PROJECT_ROOT_NAMESPACE")) + } else if ci == "bitbucket" { + rawId = os.Getenv("BITBUCKET_WORKSPACE") + } + + id, err := deterministic(rawId) + if err != nil { + return "", err + } + + return id.String(), nil +} + +func UserId() (string, error) { + if IsCI() { + ci := guessCI() + var rawId string + + if ci == "github" { + client := github.NewClient(nil) + user, _, err := client.Users.Get(context.Background(), os.Getenv("GITHUB_ACTOR")) + if err != nil { + return "", err + } + name := "" + if user.Name != nil { + name = *user.Name + } + + rawId = fmt.Sprintf("%s %s %d", name, *user.Login, *user.ID) + } else if ci == "gitlab" { + rawId = fmt.Sprintf("%s %s %s", + os.Getenv("GITLAB_USER_NAME"), + os.Getenv("GITLAB_USER_LOGIN"), + os.Getenv("GITLAB_USER_ID")) + } else if ci == "bitbucket" { + rawId = os.Getenv("BITBUCKET_STEP_TRIGGERER_UUID") + } else { + var out bytes.Buffer + cmd := exec.Command("git", "log", "-1", "--pretty=format:'%ae'") + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return "", err + } + + rawId = out.String() + } + + id, err := deterministic(rawId) + if err != nil { + return "", err + } + + return id.String(), nil + } + + id := uuid.New().String() + old := appdirs.UserConfigDir("dvc/user_id", "iterative", "", false) + _, errorOld := os.Stat(old) + + new := appdirs.UserConfigDir("iterative/telemetry", "", "", false) + _, errorNew := os.Stat(new) + + if os.IsNotExist(errorNew) { + if !os.IsNotExist(errorOld) { + jsonFile, jsonErr := os.Open(old) + if jsonErr != nil { + return "", jsonErr + } + + byteValue, err := ioutil.ReadAll(jsonFile) + if err != nil { + return "", err + } + var data map[string]interface{} + err = json.Unmarshal([]byte(byteValue), &data) + if err != nil { + return "", err + } + id = data["user_id"].(string) + + defer jsonFile.Close() + } + + err := os.MkdirAll(filepath.Dir(new), 0644) + if err != nil { + return "", err + } + err = ioutil.WriteFile(new, []byte(id), 0644) + if err != nil { + return "", err + } + } else { + dat, err := ioutil.ReadFile(new) + if err != nil { + return "", err + } + id = string(dat[:]) + } + + if os.IsNotExist(errorOld) && id != "do-not-track" { + err := os.MkdirAll(filepath.Dir(old), 0644) + if err != nil { + return "", err + } + data := map[string]interface{}{ + "user_id": id, + } + file, err := json.MarshalIndent(data, "", " ") + if err != nil { + return "", err + } + err = ioutil.WriteFile(old, file, 0644) + if err != nil { + return "", err + } + } + + return id, nil +} + +func ResourceData(d *schema.ResourceData) map[string]interface{} { + if d == nil { + return map[string]interface{}{} + } + + tpiLogs := d.Get("logs").([]interface{}) + logs := "" + for _, log := range tpiLogs { + logs += log.(string) + } + spot := d.Get("spot").(float64) + return map[string]interface{}{ + "cloud": d.Get("cloud").(string), + "cloud_region": d.Get("region").(string), + "cloud_machine": d.Get("machine").(string), + "cloud_disk_size": d.Get("disk_size").(int), + "cloud_spot": spot, + "cloud_spot_auto": spot == 0.0, + "task_status": d.Get("status").(map[string]interface{}), + "task_duration": TaskDuration(logs), + "task_resumed": len(tpiLogs) > 1, + } +} + +func JitsuEventPayload(action string, e error, extra map[string]interface{}) (map[string]interface{}, error) { + systemInfo := SystemInfo() + + err := "" + if e != nil { + err = reflect.TypeOf(e).String() + } + + userId, uErr := UserId() + if uErr != nil { + return nil, uErr + } + + groupId, gErr := GroupId() + if gErr != nil { + return nil, gErr + } + + tfVer, tfVerErr := TerraformVersion() + if tfVerErr != nil { + return nil, tfVerErr + } + + extra["ci"] = guessCI() + extra["terraform_version"] = tfVer + + payload := map[string]interface{}{ + "user_id": userId, + "group_id": groupId, + "action": action, + "interface": "cli", + "tool_name": "tpi", + "tool_source": "terraform", + "tool_version": Version, + "os_name": systemInfo["os_name"], + "os_version": systemInfo["platform_version"], + "backend": extra["cloud"], + "error": err, + "extra": extra, + } + + return payload, nil +} + +func SendJitsuEvent(action string, e error, extra map[string]interface{}) { + for _, env := range []string{"ITERATIVE_DO_NOT_TRACK"} { + if _, ok := os.LookupEnv(env); ok { + logrus.Debugf("analytics: %s environment variable is set; doing nothing", env) + return + } + } + + payload, err := JitsuEventPayload(action, e, extra) + if err != nil { + logrus.Debugf("analytics: Failure generating Jitsu Event Payload; doing nothing") + return + } + + if payload["user_id"] == "do-not-track" { + logrus.Debugf("analytics: user_id %s is set; doing nothing", payload["user_id"]) + return + } + + go send(payload) //nolint:errcheck + wg.Add(1) +} + +func send(event interface{}) error { + defer wg.Done() + + body, err := json.Marshal(event) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, getenv("TPI_ANALYTICS_ENDPOINT", Endpoint), bytes.NewBuffer(body)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Auth-Token", getenv("TPI_ANALYTICS_TOKEN", Token)) + + client := &http.Client{Timeout: Timeout} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.New("server returned: " + resp.Status) + } + + return nil +} + +func WaitForAnalyticsAndHandlePanics() { + r := recover() + + if r != nil { + extra := map[string]interface{}{"stack": debug.Stack()} + SendJitsuEvent("panic", fmt.Errorf("panic: %v", r), extra) + } + + wg.Wait() + + if r != nil { + panic(r) + } +} diff --git a/iterative/utils/analytics_test.go b/iterative/utils/analytics_test.go new file mode 100644 index 00000000..e85d9161 --- /dev/null +++ b/iterative/utils/analytics_test.go @@ -0,0 +1,89 @@ +package utils + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/wessie/appdirs" +) + +func TestVersion(t *testing.T) { + assert.Equal(t, Version, "0.0.0") +} + +func TestTerraformVersion(t *testing.T) { + ver, _ := TerraformVersion() + assert.Equal(t, strings.HasPrefix(ver, "v"), true) +} + +func TestSystemInfo(t *testing.T) { + info := SystemInfo() + assert.NotNil(t, info["os_name"]) + assert.NotNil(t, info["platform_version"]) +} + +func TestUserId(t *testing.T) { + old := appdirs.UserConfigDir("dvc/user_id", "iterative", "", false) + new := appdirs.UserConfigDir("iterative/telemetry", "", "", false) + + userId := "00000000-0000-0000-0000-000000000000" + data := map[string]interface{}{ + "user_id": userId, + } + json, _ := json.MarshalIndent(data, "", " ") + + _ = os.MkdirAll(filepath.Dir(old), 0644) + _ = ioutil.WriteFile(old, json, 0644) + + id, _ := UserId() + assert.Equal(t, len(id) == 36, true) + + if !IsCI() { + assert.Equal(t, userId == id, true) + + _, err := os.Stat(new) + assert.Equal(t, !os.IsNotExist(err), true) + } +} + +func TestResourceData(t *testing.T) { + cloud := "aws" + region := "us-west" + machine := "xl" + disk_size := 30 + spot := 0.0 + status := map[string]interface{}{ + "running": 0, + "failed": 0, + } + logs := make([]interface{}, 0) + logs = append(logs, "2022-04-24 20:25:07 Started tpi-task.service.\n2022-04-24 20:26:07 hello.\n") + logs = append(logs, "2022-04-24 20:27:07 Started tpi-task.service.\n") + + d := generateSchemaData(t, map[string]interface{}{ + "cloud": cloud, + "region": region, + "machine": machine, + "disk_size": disk_size, + "spot": spot, + "status": status, + "logs": logs, + }) + + data := ResourceData(d) + + assert.Equal(t, cloud, data["cloud"].(string)) + assert.Equal(t, region, data["cloud_region"].(string)) + assert.Equal(t, machine, data["cloud_machine"].(string)) + assert.Equal(t, disk_size, data["cloud_disk_size"].(int)) + assert.Equal(t, spot, data["cloud_spot"].(float64)) + assert.Equal(t, true, data["cloud_spot_auto"].(bool)) + assert.Equal(t, status, data["task_status"].(map[string]interface{})) + assert.Equal(t, 120.0, data["task_duration"].(float64)) + assert.Equal(t, true, data["task_resumed"].(bool)) +} diff --git a/main.go b/main.go index e45dcc2d..dc65dc85 100644 --- a/main.go +++ b/main.go @@ -10,11 +10,14 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/plugin" "terraform-provider-iterative/iterative" + "terraform-provider-iterative/iterative/utils" "terraform-provider-iterative/task" "terraform-provider-iterative/task/common" ) func main() { + defer utils.WaitForAnalyticsAndHandlePanics() + if identifier := os.Getenv("TPI_TASK_IDENTIFIER"); identifier != "" { provider := os.Getenv("TPI_TASK_CLOUD_PROVIDER") region := os.Getenv("TPI_TASK_CLOUD_REGION") From 8543c1f0536a69f03e1b4275c100d0dcb29ed981 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 2 Jul 2022 00:08:27 +0100 Subject: [PATCH 2/4] Update .devcontainer.json --- .devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer.json b/.devcontainer.json index 6d35ab71..d2c9ae23 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,7 +1,7 @@ { "name": "terraform", "extensions": ["hashicorp.terraform"], - "postCreateCommand": "go install github.com/hashicorp/terraform@v1 github.com/hashicorp/packer@v1 github.com/automation-co/husky@v0.2 && husky init", + "postCreateCommand": "for tool in packer_1.8.2 terraform_1.2.4; do curl https://releases.hashicorp.com/${tool%_*}/${tool#*_}/${tool}_linux_amd64.zip | zcat | dd of=/usr/bin/${tool%_*} && chmod 755 /usr/bin/${tool%_*}; done && go install github.com/automation-co/husky@v0.2 && husky init", "image": "mcr.microsoft.com/vscode/devcontainers/go:1", "hostRequirements": { "cpus": 4, From 53b1cd347c60b06a3d96fc38b8e2b0384eb1b4bb Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 2 Jul 2022 00:12:46 +0100 Subject: [PATCH 3/4] Delete .restyled.yaml --- .restyled.yaml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .restyled.yaml diff --git a/.restyled.yaml b/.restyled.yaml deleted file mode 100644 index 17478695..00000000 --- a/.restyled.yaml +++ /dev/null @@ -1,2 +0,0 @@ -exclude: - - "**/*.golden" From a79dfd8a241aa925ed59661f08345a95df6928a3 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 2 Jul 2022 00:33:26 +0100 Subject: [PATCH 4/4] Update .devcontainer.json --- .devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer.json b/.devcontainer.json index d2c9ae23..877248f4 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,7 +1,7 @@ { "name": "terraform", "extensions": ["hashicorp.terraform"], - "postCreateCommand": "for tool in packer_1.8.2 terraform_1.2.4; do curl https://releases.hashicorp.com/${tool%_*}/${tool#*_}/${tool}_linux_amd64.zip | zcat | dd of=/usr/bin/${tool%_*} && chmod 755 /usr/bin/${tool%_*}; done && go install github.com/automation-co/husky@v0.2 && husky init", + "postCreateCommand": "for tool in packer_1.8.2 terraform_1.2.4; do curl https://releases.hashicorp.com/${tool%_*}/${tool#*_}/${tool}_linux_amd64.zip | zcat | dd of=/usr/bin/${tool%_*} && chmod 755 /usr/bin/${tool%_*}; done && go install github.com/automation-co/husky@v0.2 && husky install", "image": "mcr.microsoft.com/vscode/devcontainers/go:1", "hostRequirements": { "cpus": 4,