Update gcp and aws existing bucket data sources to align with the azure data source.

Use common.RemoteStorage to initialize the data sources.
Using rclone to verify storage during Read.
Remove aws s3 client mocks and tests that rely on them.

Author: tasdomas

Makefile

@@ -25,5 +25,3 @@ sweep:
 
 testacc:
 	TF_ACC=1 go test ./... -v ${TESTARGS} -timeout 120m
-
-generate:

task/aws/resources/data_source_bucket.go

@@ -3,65 +3,60 @@ package resources
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
 
 	"terraform-provider-iterative/task/common"
+	"terraform-provider-iterative/task/common/machine"
 )
 
 // NewExistingS3Bucket returns a new data source refering to a pre-allocated
 // S3 bucket.
-func NewExistingS3Bucket(client S3Client, credentials aws.Credentials, storageParams common.RemoteStorage) *ExistingS3Bucket {
+func NewExistingS3Bucket(credentials aws.Credentials, storageParams common.RemoteStorage) *ExistingS3Bucket {
 	return &ExistingS3Bucket{
-		client:      client,
 		credentials: credentials,
 		params:      storageParams,
 	}
 }
 
 // ExistingS3Bucket identifies an existing S3 bucket.
 type ExistingS3Bucket struct {
-	client      S3Client
 	credentials aws.Credentials
 
 	params common.RemoteStorage
 }
 
 // Read verifies the specified S3 bucket is accessible.
 func (b *ExistingS3Bucket) Read(ctx context.Context) error {
-	input := s3.HeadBucketInput{
-		Bucket: aws.String(b.params.Container),
-	}
-	if _, err := b.client.HeadBucket(ctx, &input); err != nil {
-		if errorCodeIs(err, errNotFound) {
-			return common.NotFoundError
-		}
-		return err
+	err := machine.CheckStorage(ctx, b.connection())
+	if err != nil {
+		return fmt.Errorf("failed to verify existing s3 bucket: %w", err)
 	}
 	return nil
 }
 
+func (b *ExistingS3Bucket) connection() machine.RcloneConnection {
+	region := b.params.Config["region"]
+	return machine.RcloneConnection{
+		Backend:   machine.RcloneBackendS3,
+		Container: b.params.Container,
+		Path:      b.params.Path,
+		Config: map[string]string{
+			"provider":          "AWS",
+			"region":            region,
+			"access_key_id":     b.credentials.AccessKeyID,
+			"secret_access_key": b.credentials.SecretAccessKey,
+			"session_token":     b.credentials.SessionToken,
+		},
+	}
+}
+
 // ConnectionString implements common.StorageCredentials.
 // The method returns the rclone connection string for the specific bucket.
 func (b *ExistingS3Bucket) ConnectionString(ctx context.Context) (string, error) {
-	region := b.params.Config["region"]
-	connectionString := fmt.Sprintf(
-		":s3,provider=AWS,region=%s,access_key_id=%s,secret_access_key=%s,session_token=%s:%s/%s",
-		region,
-		b.credentials.AccessKeyID,
-		b.credentials.SecretAccessKey,
-		b.credentials.SessionToken,
-		b.params.Container,
-		strings.TrimPrefix(b.params.Path, "/"))
-	return connectionString, nil
+	connection := b.connection()
+	return connection.String(), nil
 }
 
 // build-time check to ensure Bucket implements BucketCredentials.
 var _ common.StorageCredentials = (*ExistingS3Bucket)(nil)
-
-// S3Client defines the functions of the AWS S3 API used.
-type S3Client interface {
-	HeadBucket(context.Context, *s3.HeadBucketInput, ...func(*s3.Options)) (*s3.HeadBucketOutput, error)
-}

task/aws/resources/data_source_bucket_test.go

@@ -5,13 +5,9 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/smithy-go"
-	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 
 	"terraform-provider-iterative/task/aws/resources"
-	"terraform-provider-iterative/task/aws/resources/mocks"
 	"terraform-provider-iterative/task/common"
 )
 
@@ -22,46 +18,11 @@ func TestExistingBucketConnectionString(t *testing.T) {
 		SecretAccessKey: "secret-access-key",
 		SessionToken:    "session-token",
 	}
-	b := resources.NewExistingS3Bucket(nil, creds, common.RemoteStorage{
+	b := resources.NewExistingS3Bucket(creds, common.RemoteStorage{
 		Container: "pre-created-bucket",
 		Config:    map[string]string{"region": "us-east-1"},
 		Path:      "subdirectory"})
 	connStr, err := b.ConnectionString(ctx)
 	require.NoError(t, err)
-	require.Equal(t, connStr, ":s3,provider=AWS,region=us-east-1,access_key_id=access-key-id,secret_access_key=secret-access-key,session_token=session-token:pre-created-bucket/subdirectory")
-}
-
-func TestExistingBucketRead(t *testing.T) {
-	ctx := context.Background()
-	ctl := gomock.NewController(t)
-	defer ctl.Finish()
-
-	s3Cl := mocks.NewMockS3Client(ctl)
-	s3Cl.EXPECT().HeadBucket(gomock.Any(), &s3.HeadBucketInput{Bucket: aws.String("bucket-id")}).Return(nil, nil)
-	b := resources.NewExistingS3Bucket(s3Cl, aws.Credentials{}, common.RemoteStorage{
-		Container: "bucket-id",
-		Config:    map[string]string{"region": "us-east-1"},
-		Path:      "subdirectory"})
-	err := b.Read(ctx)
-	require.NoError(t, err)
-}
-
-// TestExistingBucketReadNotFound tests the case where the s3 client indicates that the bucket could not be
-// found.
-func TestExistingBucketReadNotFound(t *testing.T) {
-	ctx := context.Background()
-	ctl := gomock.NewController(t)
-	defer ctl.Finish()
-
-	s3Cl := mocks.NewMockS3Client(ctl)
-
-	s3Cl.EXPECT().
-		HeadBucket(gomock.Any(), &s3.HeadBucketInput{Bucket: aws.String("bucket-id")}).
-		Return(nil, &smithy.GenericAPIError{Code: "NotFound"})
-	b := resources.NewExistingS3Bucket(s3Cl, aws.Credentials{}, common.RemoteStorage{
-		Container: "bucket-id",
-		Config:    map[string]string{"region": "us-east-1"},
-		Path:      "subdirectory"})
-	err := b.Read(ctx)
-	require.ErrorIs(t, err, common.NotFoundError)
+	require.Equal(t, ":s3,access_key_id='access-key-id',provider='AWS',region='us-east-1',secret_access_key='secret-access-key',session_token='session-token':pre-created-bucket/subdirectory", connStr)
 }

task/aws/resources/mocks/gen.go

@@ -52,18 +52,16 @@ func New(ctx context.Context, cloud common.Cloud, identifier common.Identifier,
 	)
 	var bucketCredentials common.StorageCredentials
 	if task.RemoteStorage != nil {
-		containerPath := task.RemoteStorage.Path
 		// If a subdirectory was not specified, the task id will
 		// be used.
-		if containerPath == "" {
-			containerPath = string(t.Identifier)
+		if task.RemoteStorage.Path == "" {
+			task.RemoteStorage.Path = string(t.Identifier)
 		}
 		// Container config may override the s3 region.
 		if region, ok := task.RemoteStorage.Config[s3_region]; !ok || region == "" {
 			task.RemoteStorage.Config[s3_region] = t.Client.Region
 		}
 		bucket := resources.NewExistingS3Bucket(
-			t.Client.Services.S3,
 			t.Client.Credentials(),
 			*task.RemoteStorage)
 		t.DataSources.Bucket = bucket

task/aws/resources/mocks/s3client_generated.go

@@ -42,6 +42,11 @@ func New(ctx context.Context, cloud common.Cloud, identifier common.Identifier,
 	)
 	var bucketCredentials common.StorageCredentials
 	if task.RemoteStorage != nil {
+		// If a subdirectory was not specified, the task id will
+		// be used.
+		if task.RemoteStorage.Path == "" {
+			task.RemoteStorage.Path = string(t.Identifier)
+		}
 		bucket := resources.NewExistingBlobContainer(t.Client, *task.RemoteStorage)
 		t.DataSources.BlobContainer = bucket
 		bucketCredentials = bucket

task/aws/task.go

@@ -219,7 +219,9 @@ func CheckStorage(ctx context.Context, remoteConn RcloneConnection) error {
 type RcloneBackend string
 
 const (
-	RcloneBackendAzureBlob = "azureblob"
+	RcloneBackendAzureBlob          = "azureblob"
+	RcloneBackendS3                 = "s3"
+	RcloneBackendGoogleCloudStorage = "googlecloudstorage"
 )
 
 // RcloneConnection is used to construct an rclone connection string.

task/az/task.go

@@ -2,66 +2,50 @@ package resources
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"path"
-
-	"google.golang.org/api/googleapi"
-	"google.golang.org/api/storage/v1"
 
 	"terraform-provider-iterative/task/common"
-	"terraform-provider-iterative/task/gcp/client"
+	"terraform-provider-iterative/task/common/machine"
 )
 
 // NewExistingBucket creates a new data source referring to a pre-allocated GCP storage bucket.
-func NewExistingBucket(client *client.Client, id string, path string) *ExistingBucket {
+func NewExistingBucket(clientCredentials string, storageParams common.RemoteStorage) *ExistingBucket {
 	return &ExistingBucket{
-		client: client,
-
-		id:   id,
-		path: path,
+		clientCredentials: clientCredentials,
+		params:            storageParams,
 	}
 }
 
 // ExistingBucket identifies a pre-allocated storage bucket.
 type ExistingBucket struct {
-	client *client.Client
-
-	resource *storage.Bucket
-	id       string
-	path     string
+	clientCredentials string
+	params            common.RemoteStorage
 }
 
 // Read verifies the specified storage bucket exists and is accessible.
 func (b *ExistingBucket) Read(ctx context.Context) error {
-	bucket, err := b.client.Services.Storage.Buckets.Get(b.id).Do()
+	connection := b.connection()
+	err := machine.CheckStorage(ctx, connection)
 	if err != nil {
-		var e *googleapi.Error
-		if errors.As(err, &e) && e.Code == 404 {
-			return common.NotFoundError
-		}
-		return err
+		return fmt.Errorf("failed to verify storage: %w", err)
 	}
-
-	b.resource = bucket
 	return nil
 }
 
+func (b *ExistingBucket) connection() machine.RcloneConnection {
+	return machine.RcloneConnection{
+		Backend:   machine.RcloneBackendGoogleCloudStorage,
+		Container: b.params.Container,
+		Path:      b.params.Path,
+		Config: map[string]string{
+			"service_account_credentials": b.clientCredentials,
+		}}
+}
+
 // ConnectionString implements common.StorageCredentials.
 // The method returns the rclone connection string for the specific bucket.
 func (b *ExistingBucket) ConnectionString(ctx context.Context) (string, error) {
-	if len(b.client.Credentials.JSON) == 0 {
-		return "", errors.New("unable to find credentials JSON string")
-	}
-	credentials := string(b.client.Credentials.JSON)
-	containerPath := path.Join(b.id, b.path)
-	connStr := fmt.Sprintf(
-		":googlecloudstorage,service_account_credentials='%s':%s",
-		credentials,
-		containerPath,
-	)
-
-	return connStr, nil
+	return b.connection().String(), nil
 }
 
 var _ common.StorageCredentials = (*ExistingBucket)(nil)

task/common/machine/storage.go

@@ -0,0 +1,22 @@
+package resources_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"terraform-provider-iterative/task/common"
+	"terraform-provider-iterative/task/gcp/resources"
+)
+
+func TestExistingBucketConnectionString(t *testing.T) {
+	ctx := context.Background()
+	creds := "gcp-credentials-json"
+	b := resources.NewExistingBucket(creds, common.RemoteStorage{
+		Container: "pre-created-bucket",
+		Path:      "subdirectory"})
+	connStr, err := b.ConnectionString(ctx)
+	require.NoError(t, err)
+	require.Equal(t, ":googlecloudstorage,service_account_credentials='gcp-credentials-json':pre-created-bucket/subdirectory", connStr)
+}