minio support

Author: iQQBot

cmd/experiemental-unmount.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package cmd

cmd/experimental-mount.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package cmd

cmd/root.go

@@ -19,8 +19,11 @@ const (
 	// EnvvarWorkspaceRoot names the environment variable we check for the workspace root path
 	EnvvarWorkspaceRoot = "LEEWAY_WORKSPACE_ROOT"
 
-	// EnvvarRemoteCacheBucket configures a GCP bucket name. This enables the use of GSUtilRemoteStorage
+	// EnvvarRemoteCacheBucket configures a bucket name. This enables the use of RemoteStorage
 	EnvvarRemoteCacheBucket = "LEEWAY_REMOTE_CACHE_BUCKET"
+
+	// EnvvarRemoteCacheStorage configures a Remote Storage Provider. Default is GCP
+	EnvvarRemoteCacheStorage = "LEEWAY_REMOTE_CACHE_STORAGE"
 )
 
 const (
@@ -171,10 +174,23 @@ func getBuildArgs() (leeway.Arguments, error) {
 
 func getRemoteCache() leeway.RemoteCache {
 	remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
+	remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
 	if remoteCacheBucket != "" {
-		return leeway.GSUtilRemoteCache{
-			BucketName: remoteCacheBucket,
+		switch remoteStorage {
+		case "GCP":
+			return leeway.GSUtilRemoteCache{
+				BucketName: remoteCacheBucket,
+			}
+		case "MINIO":
+			return leeway.MinioRemoteCache{
+				BucketName: remoteCacheBucket,
+			}
+		default:
+			return leeway.GSUtilRemoteCache{
+				BucketName: remoteCacheBucket,
+			}
 		}
+
 	}
 
 	return leeway.NoRemoteCache{}

pkg/leeway/cache.go

@@ -154,3 +154,82 @@ func gsutilTransfer(target string, files []string) error {
 	}
 	return nil
 }
+
+// MinioRemoteCache uses the mc command to implement a remote cache
+type MinioRemoteCache struct {
+	BucketName string
+}
+
+// Download makes a best-effort attempt at downloading previously cached build artifacts
+func (rs MinioRemoteCache) Download(dst Cache, pkgs []*Package) error {
+	fmt.Printf("☁️  minio checking remote cache for past build artifacts\n")
+	var (
+		files []string
+		dest  string
+	)
+	for _, pkg := range pkgs {
+		fn, exists := dst.Location(pkg)
+		if exists {
+			continue
+		}
+		if dest == "" {
+			dest = filepath.Dir(fn)
+		} else if dest != filepath.Dir(fn) {
+			return xerrors.Errorf("gsutil only supports one target folder, not %s and %s", dest, filepath.Dir(fn))
+		}
+
+		files = append(files, fmt.Sprintf("minio/%s/%s", rs.BucketName, filepath.Base(fn)))
+	}
+	return minioTransfer(dest, files)
+}
+
+// Upload makes a best effort to upload the build arfitacts to a remote cache
+func (rs MinioRemoteCache) Upload(src Cache, pkgs []*Package) error {
+	fmt.Printf("☁️  minio uploading build artifacts to remote cache\n")
+	var files []string
+	for _, pkg := range pkgs {
+		file, exists := src.Location(pkg)
+		if !exists {
+			continue
+		}
+		files = append(files, file)
+	}
+	return minioTransfer(fmt.Sprintf("minio/%s", rs.BucketName), files)
+}
+
+func minioTransfer(target string, files []string) error {
+	if len(files) == 0 {
+		return nil
+	}
+	log.WithField("target", target).WithField("files", files).Debug("transfering files using gsutil")
+	runCopyCommand := func(source, target string) error {
+		args := make([]string, 0, 3)
+		args = append(args, "cp")
+		args = append(args, source)
+		args = append(args, target)
+		cmd := exec.Command("mc", args...)
+		cmd.Stdout = os.Stdout
+		err := cmd.Start()
+		if err != nil {
+			return err
+		}
+		err = cmd.Wait()
+		if err != nil {
+			if exiterr, ok := err.(*exec.ExitError); ok {
+				if _, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+					// we just swallow non-zero exit codes here as remote caching is best effort
+					return nil
+				}
+			}
+			return err
+		}
+		return nil
+	}
+	for _, source := range files {
+		err := runCopyCommand(source, target)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}