PMM-7 Modernize go tools

.devcontainer/setup.py

@@ -24,16 +24,14 @@ def install_packages():
     """Installs required and useful RPM packages."""
 
     run_commands([
-        "dnf install -y gcc git make pkgconfig \
+        "dnf install -y --enablerepo=ol9_codeready_builder\
+            gcc git make pkgconfig \
             vim \
-            mc psmisc procps lsof which iproute diffutils \
+            psmisc procps lsof which iproute diffutils \
             man man-pages \
             openssl-devel \
-            wget \
-            krb5-devel",
-
-        "dnf install -y ansible-lint glibc-static --enablerepo=ol9_codeready_builder"
-
+            ansible-lint glibc-static \
+            krb5-devel"
     ])
 
 
@@ -63,14 +61,6 @@ def install_go():
     ])
 
 
-def make_init():
-    """Runs make init."""
-
-    run_commands([
-        "make init",
-    ])
-
-
 def setup():
     """Runs various setup commands."""
     run_commands([
@@ -87,7 +77,6 @@ def setup():
 def main():
     install_packages()
     install_go()
-    make_init()
 
     # do basic setup
     setup()

.github/dependabot.yml

@@ -68,24 +68,6 @@ updates:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
-  - package-ecosystem: "gomod"
-    directory: "/tools"
-    target-branch: "v3"
-    schedule:
-      interval: "daily"
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
-
-  - package-ecosystem: "gomod"
-    directory: "/api-tests/tools"
-    target-branch: "v3"
-    schedule:
-      interval: "daily"
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
-
   - package-ecosystem: "docker"
     directory: "/"
     target-branch: "v3"

.github/workflows/main.yml

@@ -47,12 +47,7 @@ jobs:
             ${{ runner.os }}-go-modules-
 
       - name: Download Go modules
-        run: |
-          pushd tools && go mod download -x
-          popd        && go mod download -x
-
-      - name: Install development tools
-        run: make init
+        run: go mod download -x
 
       - name: Generate files
         run: make gen
@@ -63,8 +58,7 @@ jobs:
       - name: Check files are formatted and no source code changes
         run: |
           make format
-          pushd tools && go mod tidy -v
-          popd        && go mod tidy -v
+          go mod tidy -v
           git status
           git diff --exit-code
 
@@ -77,16 +71,16 @@ jobs:
       - name: Run check-license
         run: |
           # run license checker on configured files
-          bin/license-eye -c .licenserc.yaml header check
+          go tool license-eye -c .licenserc.yaml header check
 
       - name: Run go-sumtype
-        run: bin/go-sumtype ./...
+        run: go tool go-sumtype ./...
 
       - name: Run API linter
         env:
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
-          if out=$(bin/buf lint -v api); code="$?"; test "$code" -eq 0; then
+          if out=$(go tool buf lint -v api); code="$?"; test "$code" -eq 0; then
             echo "$out"
             exit 0
           fi
@@ -99,7 +93,7 @@ jobs:
           fi
 
           # One may need to suppress passing to reviewdog because of https://github.com/reviewdog/reviewdog/issues/1696
-          echo "$out" | bin/reviewdog -f=buf -reporter=github-pr-review -fail-level=error
+          echo "$out" | go tool reviewdog -f=buf -reporter=github-pr-review -fail-level=error
 
       - name: Run code linters
         uses: reviewdog/action-golangci-lint@f9bba13753278f6a73b27a56a3ffb1bfda90ed71 # v2.8.0
@@ -116,7 +110,7 @@ jobs:
         env:
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
-          if out=$(bin/go-consistent -pedantic -exclude "tests" ./...); exit_code=$?; [ $exit_code -eq 0 ]; then
+          if out=$(go tool go-consistent -pedantic -exclude "tests" ./...); exit_code=$?; [ $exit_code -eq 0 ]; then
             echo "$out"
             exit 0
           fi
@@ -126,7 +120,7 @@ jobs:
             exit $exit_code
           fi
 
-          echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error
+          echo "$out" | go tool reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error
 
       - name: Test common API
         run: make test-common

Makefile.include

@@ -10,11 +10,7 @@ help:                 ## Display this help message
 	@echo Check the docker-compose.yml file to see which environment variables are available.
 
 init:                 ## Install tools
-	rm -rf bin/*
-	cd tools && go generate -x -tags=tools
-
-	# Install golangci-lint
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v2.3.0 # Version should match specified in CI
+	@echo "Tools are now managed via go.mod and accessed with 'go tool <toolname>'"
 
 release:              ## Build release versions of all components
 	make -C agent release
@@ -41,7 +37,7 @@ clean:               ## Remove generated files
 
 gen-mocks:
 	find . -name mock_*.go -delete
-	./bin/mockery --config .mockery.yaml
+	go tool mockery --config .mockery.yaml
 
 test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
 	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
@@ -50,24 +46,24 @@ api-test:             ## Run API tests on dev env.
 	go test -count=1 -race -p 1 -v ./api-tests/... -pmm.server-insecure-tls
 
 check:                ## Run required checkers and linters
-	bin/buf lint -v api
-	LOG_LEVEL=error bin/golangci-lint run
-	bin/go-sumtype ./...
-	bin/go-consistent -pedantic ./...
+	go tool buf lint -v api
+	LOG_LEVEL=error go tool -modfile=golangci-lint.mod golangci-lint run
+	go tool go-sumtype ./...
+	go tool go-consistent -pedantic ./...
 
 check-license:          ## Run license header checks against source files
-	bin/license-eye -c .licenserc.yaml header check
+	go tool license-eye -c .licenserc.yaml header check
 
 check-all: check-license check    ## Run golangci linter to check for changes against main
-	bin/golangci-lint run -c=.golangci.yml --new-from-rev=v3
+	go tool -modfile=golangci-lint.mod golangci-lint run -c=.golangci.yml --new-from-rev=v3
 
 FILES = $(shell find . -type f -name '*.go')
 
 format:               ## Format source code
 	make -C api format
-	bin/gofumpt -l -w $(FILES)
-	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
-	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
+	go tool gofumpt -l -w $(FILES)
+	go tool goimports -local github.com/percona/pmm -l -w $(FILES)
+	go tool gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
 
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf

agent/Makefile

@@ -48,9 +48,8 @@ clean:                          ## Remove generated files
 
 # generate stub models for perfschema QAN agent; hidden from help as it is not generally useful
 gen-init:
-	go build -modfile=tools/go.mod -o ../bin/reform-db gopkg.in/reform.v1/reform-db
 	mkdir tmp-mysql
-	../bin/reform-db -db-driver=mysql -db-source='root:root-password@tcp(127.0.0.1:3306)/performance_schema' init tmp-mysql
+	go tool reform-db -db-driver=mysql -db-source='root:root-password@tcp(127.0.0.1:3306)/performance_schema' init tmp-mysql
 
 install:                        ## Install pmm-admin binary
 	go build -v -ldflags "$(VERSION_FLAGS)" -o $(GOBIN)/pmm-agent
@@ -88,10 +87,10 @@ fuzz-postgres-parser:   ## Run fuzzer for agents/postgres/parser package
 
 bench:                          ## Run benchmarks
 	go test -bench=. -benchtime=1s -count=5 -cpu=1 -timeout=30m -failfast github.com/percona/pmm/agent/agents/mysql/slowlog/parser | tee slowlog_parser_new.bench
-	../bin/benchstat slowlog_parser_old.bench slowlog_parser_new.bench
+	go tool benchstat slowlog_parser_old.bench slowlog_parser_new.bench
 
 	go test -bench=. -benchtime=1s -count=5 -cpu=1 -timeout=30m -failfast github.com/percona/pmm/agent/agents/postgres/parser | tee postgres_parser_new.bench
-	../bin/benchstat postgres_parser_old.bench postgres_parser_new.bench
+	go tool benchstat postgres_parser_old.bench postgres_parser_new.bench
 
 RUN_FLAGS = --config-file=pmm-agent-dev.yaml
 

agent/agents/mysql/perfschema/models.go

@@ -16,7 +16,7 @@ package perfschema
 
 import "gopkg.in/reform.v1/parse"
 
-//go:generate ../../../../bin/reform
+//go:generate go tool reform
 
 // eventsStatementsSummaryByDigest represents a row in performance_schema.events_statements_summary_by_digest table.
 //

agent/agents/postgres/pgstatmonitor/models.go

@@ -16,7 +16,7 @@ package pgstatmonitor
 
 import "fmt"
 
-//go:generate ../../../../bin/reform
+//go:generate go tool reform
 
 // pgStatDatabase represents a row in pg_stat_database view.
 //

agent/agents/postgres/pgstatstatements/models.go

@@ -14,7 +14,7 @@
 
 package pgstatstatements
 
-//go:generate ../../../../bin/reform
+//go:generate go tool reform
 
 // pgStatDatabase represents a row in pg_stat_database view.
 //

api-tests/Makefile

@@ -1,7 +1,7 @@
 all: build
 
 init:           ## Installs development tools
-	cd tools && go generate -x -tags=tools
+	@echo "Tools are now managed via root go.mod and accessed with 'go tool <toolname>'"
 
 build:
 	go install -v ./...
@@ -13,14 +13,14 @@ build:
 
 run:
 	go test -count=1 -p 1 -v ./... 2>&1 | tee pmm-api-tests-output.txt
-	cat pmm-api-tests-output.txt | bin/go-junit-report > pmm-api-tests-junit-report.xml
+	cat pmm-api-tests-output.txt | go tool go-junit-report > pmm-api-tests-junit-report.xml
 
 run-dev:
 	go test -count=1 -p 1 -v ./...
 
 run-race:
 	go test -count=1 -p 1 -v -race ./... 2>&1 | tee pmm-api-tests-output.txt
-	cat pmm-api-tests-output.txt | bin/go-junit-report > pmm-api-tests-junit-report.xml
+	cat pmm-api-tests-output.txt | go tool go-junit-report > pmm-api-tests-junit-report.xml
 
 clean:
 	rm -f ./pmm-api-tests-output.txt

api/Makefile

@@ -3,9 +3,9 @@ SWAGGER_UI_VERSION?=latest
 
 gen:            ## Generate PMM API
 	# generated by descriptors target
-	../bin/buf breaking --against descriptor.bin .
+	go tool buf breaking --against descriptor.bin .
 
-	../bin/buf generate -v .
+	go tool buf generate -v .
 
 	SPECS="\
 		agentlocal/v1 \
@@ -24,10 +24,10 @@ gen:            ## Generate PMM API
 		platform/v1"; \
 	for API in $$SPECS; do \
 		set -x ; \
-		../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
-		../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
-		../bin/swagger validate $$API/json/$$(basename $$API).json ; \
-		../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
+		go tool swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
+		go tool swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
+		go tool swagger validate $$API/json/$$(basename $$API).json ; \
+		go tool swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
 			--additional-initialism=aws \
 			--additional-initialism=db \
 			--additional-initialism=ok \
@@ -43,7 +43,7 @@ gen:            ## Generate PMM API
 
 	# generate public API spec, omit agentlocal and inventory (always private),
 	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
-	../bin/swagger mixin --output=swagger/swagger.json \
+	go tool swagger mixin --output=swagger/swagger.json \
 		swagger/header.json \
 		server/v1/json/v1.json \
 		user/v1/json/v1.json \
@@ -55,14 +55,14 @@ gen:            ## Generate PMM API
 		backup/v1/json/v1.json \
 		qan/v1/json/v1.json \
 		platform/v1/json/v1.json
-	../bin/swagger validate swagger/swagger.json
+	go tool swagger validate swagger/swagger.json
 
 	# It looks like the order is already correct, so no need to run this
 	# TODO: check if swagger-order can be removed
-	# ../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json
+	# go tool swagger-order --output=swagger/swagger.json swagger/swagger.json
 
 	# generate API spec with all PMM Server APIs (omit agentlocal)
-	../bin/swagger mixin --output=swagger/swagger-dev.json \
+	go tool swagger mixin --output=swagger/swagger-dev.json \
 		swagger/header-dev.json \
 		server/v1/json/v1.json \
 		user/v1/json/v1.json \
@@ -77,13 +77,13 @@ gen:            ## Generate PMM API
 		qan/v1/json/v1.json \
 		platform/v1/json/v1.json
 
-	../bin/swagger validate swagger/swagger-dev.json
+	go tool swagger validate swagger/swagger-dev.json
 
 	# It looks like the order is already correct, so no need to run this
-	# ../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json
+	# go tool swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json
 
 format:               ## Format API definitions
-	../bin/buf format . -w
+	go tool buf format . -w
 
 clean-swagger:
 	find . -name '*.swagger.json' -print -delete
@@ -127,4 +127,4 @@ clean: clean-swagger  ## Remove generated files
 	rm -f swagger/swagger.json swagger/swagger-dev.json
 
 descriptors:          ## Update API compatibility descriptors
-	../bin/buf build -o descriptor.bin --as-file-descriptor-set .
+	go tool buf build -o descriptor.bin --as-file-descriptor-set .

api/README.md

@@ -14,12 +14,7 @@ make serve
 
 1. Edit `.proto` files. Do not edit Swagger, `.pb.go`, `.pb.gw.go`. You can use `make clean` to remove all generated files.
 
-2. Install required tools (once):
-```
-make init
-```
-
-3. Generate files:
+2. Generate files:
 ```
 make gen
 ```