Add port, remote backup & CIDRs support (#371)

* Add port, remote backup & CIDRs support

Add support for:
* Specifying the port number on a normal or active/active database
* Configuring backups for normal or active/active databases
* Specifying multiple CIDR ranges when peering a normal or active/active
database

* Run all acceptance tests in parallel

* Stream `go test` logs

Go will only stream test logs if it is testing a single module at a
time, otherwise it will buffer the logs to the end and then display
them. This causes issues for CI as it means there's no idea what is
happening at the moment and the logs will be missing if the run is
cancelled.

See golang/go#27826.

* Run all acceptance tests in parallel

CI is taking far too long and needs to be sped up.

Author: wjam

GNUmakefile

@@ -9,8 +9,8 @@ PROVIDER_VERSION = 99.99.99
 PLUGINS_PATH = ~/.terraform.d/plugins
 PLUGINS_PROVIDER_PATH=$(PROVIDER_HOSTNAME)/$(PROVIDER_NAMESPACE)/$(PROVIDER_TYPE)/$(PROVIDER_VERSION)/$(PROVIDER_TARGET)
 
-# Use a parallelism of 3 by default for tests, overriding whatever GOMAXPROCS is set to.
-TEST_PARALLELISM?=3
+# Use a parallelism of 2 by default for tests, overriding whatever GOMAXPROCS is set to.
+TEST_PARALLELISM?=2
 TESTARGS?=-short
 
 BIN=$(CURDIR)/bin
@@ -30,20 +30,21 @@ clean:
 	@echo "Deleting local provider binary"
 	rm -rf $(BIN)
 
+# `-p=1` added to avoid testing packages in parallel which causes `go test` to not stream logs as they are written
 testacc:
-	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 360m -parallel=$(TEST_PARALLELISM)
+	TF_ACC=1 go test ./... -v $(TESTARGS) -timeout 360m -p=1 -parallel=$(TEST_PARALLELISM)
 
 install_local: build
 	@echo "Installing local provider binary to plugins mirror path $(PLUGINS_PATH)/$(PLUGINS_PROVIDER_PATH)"
 	@mkdir -p $(PLUGINS_PATH)/$(PLUGINS_PROVIDER_PATH)
-	@cp ./$(BIN)/terraform-provider-rediscloud_v$(PROVIDER_VERSION) $(PLUGINS_PATH)/$(PLUGINS_PROVIDER_PATH)
+	@cp $(BIN)/terraform-provider-rediscloud_v$(PROVIDER_VERSION) $(PLUGINS_PATH)/$(PLUGINS_PROVIDER_PATH)
 
 sweep:
 	@echo "WARNING: This will destroy infrastructure. Use only in development accounts."
 	go test ./provider -v -sweep=ALL $(SWEEPARGS) -timeout 30m
 
-tfproviderlintx: $(BIN)/tfproviderlint
+tfproviderlintx: $(BIN)/tfproviderlintx
 	$(BIN)/tfproviderlintx $(TFPROVIDERLINT_ARGS) ./...
 
-tfproviderlint: $(BIN)/tfproviderlintx
+tfproviderlint: $(BIN)/tfproviderlint
 	$(BIN)/tfproviderlint $(TFPROVIDERLINT_ARGS) ./...

docs/resources/rediscloud_active_active_subscription_database.md

@@ -13,7 +13,7 @@ Creates a Database within a specified Active-Active Subscription in your Redis E
 
 ```hcl
 data "rediscloud_payment_method" "card" {
-	card_type = "Visa"
+  card_type = "Visa"
 }
 
 resource "rediscloud_active_active_subscription" "subscription-resource" {
@@ -24,18 +24,18 @@ resource "rediscloud_active_active_subscription" "subscription-resource" {
   creation_plan {
     memory_limit_in_gb = 1
     quantity = 1
-	region {
-		region = "us-east-1"
-		networking_deployment_cidr = "192.168.0.0/24"
-		write_operations_per_second = 1000
-		read_operations_per_second = 1000
-	}
-	region {
-		region = "us-east-2"
-		networking_deployment_cidr = "10.0.1.0/24"
-		write_operations_per_second = 1000
-		read_operations_per_second = 2000
-	}
+    region {
+      region = "us-east-1"
+      networking_deployment_cidr = "192.168.0.0/24"
+      write_operations_per_second = 1000
+      read_operations_per_second = 1000
+    }
+    region {
+      region = "us-east-2"
+      networking_deployment_cidr = "10.0.1.0/24"
+      write_operations_per_second = 1000
+      read_operations_per_second = 2000
+    }
   }
 }
 
@@ -47,23 +47,23 @@ resource "rediscloud_active_active_subscription_database" "database-resource" {
     global_password = "some-random-pass-2" 
     global_source_ips = ["192.168.0.0/16"]
     global_alert {
-	name = "dataset-size"
-	value = 40
+      name = "dataset-size"
+      value = 40
     }
 
     override_region {
-    	name = "us-east-2"
-        override_global_source_ips = ["192.10.0.0/16"]
+      name = "us-east-2"
+      override_global_source_ips = ["192.10.0.0/16"]
     }
 
     override_region {
-    	name = "us-east-1"
-    	override_global_data_persistence = "none"
-    	override_global_password = "region-specific-password"
-    	override_global_alert {
-        	name = "dataset-size"
-        	value = 60
-    	}
+      name = "us-east-1"
+      override_global_data_persistence = "none"
+      override_global_password = "region-specific-password"
+      override_global_alert {
+        name = "dataset-size"
+        value = 60
+      }
    }
 }
 
@@ -89,9 +89,10 @@ The following arguments are supported:
 * `client_ssl_certificate` - (Optional) SSL certificate to authenticate user connections.
 * `data_eviction` - (Optional) The data items eviction policy (either: 'allkeys-lru', 'allkeys-lfu', 'allkeys-random', 'volatile-lru', 'volatile-lfu', 'volatile-random', 'volatile-ttl' or 'noeviction'. Default: 'volatile-lru')
 * `global_data_persistence` - (Optional) Global rate of database data persistence (in persistent storage) of regions that dont override global settings. Default: 'none'
-* `global_password` - (Optional) Password to access the database of regions that dont override global settings. If left empty, the password will be generated automatically
-* `global_alert` - (Optional) A block defining Redis database alert of regions that dont override global settings, documented below, can be specified multiple times
-* `global_source_ips` - (Optional)  List of source IP addresses or subnet masks of regions that dont override global settings. If specified, Redis clients will be able to connect to this database only from within the specified source IP addresses ranges (example: ['192.168.10.0/32', '192.168.12.0/24'])
+* `global_password` - (Optional) Password to access the database of regions that don't override global settings. If left empty, the password will be generated automatically
+* `global_alert` - (Optional) A block defining Redis database alert of regions that don't override global settings, documented below, can be specified multiple times
+* `global_source_ips` - (Optional)  List of source IP addresses or subnet masks of regions that don't override global settings. If specified, Redis clients will be able to connect to this database only from within the specified source IP addresses ranges (example: ['192.168.10.0/32', '192.168.12.0/24'])
+* `port` - (Optional) TCP port on which the database is available - must be between 10000 and 19999.
 * `override_region` - (Optional) Override region specific configuration, documented below
 
 
@@ -102,12 +103,20 @@ The `override_region` block supports:
 * `override_global_password` - (Optional) If specified, this regional instance of an Active-Active database password will be used to access the database
 * `override_global_source_ips` - (Optional)  List of regional instance of an Active-Active database source IP addresses or subnet masks. If specified, Redis clients will be able to connect to this database only from within the specified source IP addresses ranges (example: ['192.168.10.0/32', '192.168.12.0/24'] )
 * `override_global_data_persistence` - (Optional) Regional instance of an Active-Active database data persistence rate (in persistent storage)
+* `remote_backup` - (Optional) Specifies the backup options for the database in this region, documented below
 
 The `override_global_alert` block supports:
 
 * `name` - (Required) Alert name
 * `value` - (Required) Alert value
 
+The `remote_backup` block supports:
+
+* `interval` (Required) - Defines the frequency of the automatic backup
+* `time_utc` (Optional) - Defines the hour automatic backups are made - only applicable when interval is `every-12-hours` or `every-24-hours`
+* `storage_type` (Required) - Defines the provider of the storage location
+* `storage_path` (Required) - Defines a URI representing the backup storage location
+
 ### Timeouts
 
 The `timeouts` block allows you to specify [timeouts](https://www.terraform.io/language/resources/syntax#operation-timeouts) for certain actions:
@@ -129,5 +138,5 @@ The `timeouts` block allows you to specify [timeouts](https://www.terraform.io/l
 $ terraform import rediscloud_active_active_subscription_database.database-resource 123456/12345678
 ```
 
-Note: Due to constraints in the Redis Cloud API, the import process will not import global attributes or override region attributes. If you wish to use these attributes in your Terraform configuraton, you will need to manually add them to your Terraform configuration and run `terraform apply` to update the database.
+Note: Due to constraints in the Redis Cloud API, the import process will not import global attributes or override region attributes. If you wish to use these attributes in your Terraform configuration, you will need to manually add them to your Terraform configuration and run `terraform apply` to update the database.
 

docs/resources/rediscloud_active_active_subscription_peering.md

@@ -77,7 +77,8 @@ The following arguments are supported:
 * `source_region` -	(Required) Name of the region to create the VPC peering from
 * `destination_region` - (Required) Name of the region to create the VPC peering to
 * `vpc_id` - (Required) Identifier of the VPC to be peered
-* `vpc_cidr` - (Required) CIDR range of the VPC to be peered 
+* `vpc_cidr` - (Optional) CIDR range of the VPC to be peered. Either this or `vpc_cidrs` must be specified
+* `vpc_cidrs` - (Optional) CIDR ranges of the VPC to be peered. Either this or `vpc_cidr` must be specified
 
 **GCP ONLY:**
 * `gcp_project_id` - (Required) GCP project ID that the VPC to be peered lives in

docs/resources/rediscloud_subscription_database.md

@@ -100,7 +100,7 @@ The following arguments are supported:
 * `throughput_measurement_by` - (Required) Throughput measurement method, (either ‘number-of-shards’ or ‘operations-per-second’)
 * `throughput_measurement_value` - (Required) Throughput value (as applies to selected measurement method)
 * `memory_limit_in_gb` - (Required) Maximum memory usage for this specific database
-* `protocol` - (Optional) The protocol that will be used to access the database, (either ‘redis’ or 'memcached’) Default: ‘redis’
+* `protocol` - (Optional) The protocol that will be used to access the database, (either ‘redis’ or ‘memcached’) Default: ‘redis’
 * `support_oss_cluster_api` - (Optional) Support Redis open-source (OSS) Cluster API. Default: ‘false’
 * `external_endpoint_for_oss_cluster_api` - (Optional) Should use the external endpoint for open-source (OSS) Cluster API.
   Can only be enabled if OSS Cluster API support is enabled. Default: 'false'
@@ -122,6 +122,8 @@ The following arguments are supported:
   [the documentation on clustering](https://docs.redislabs.com/latest/rc/concepts/clustering/) for more information on the
   hashing policy. This cannot be set when `support_oss_cluster_api` is set to true.
 * `enable_tls` - (Optional) Use TLS for authentication. Default: ‘false’
+* `port` - (Optional) TCP port on which the database is available - must be between 10000 and 19999.
+* `remote_backup` (Optional) Specifies the backup options for the database, documented below
 
 The `alert` block supports:
 
@@ -145,6 +147,13 @@ The `modules` list supports:
     ]
   ```
 
+The `remote_backup` block supports:
+
+* `interval` (Required) - Defines the frequency of the automatic backup
+* `time_utc` (Optional) - Defines the hour automatic backups are made - only applicable when interval is `every-12-hours` or `every-24-hours`
+* `storage_type` (Required) - Defines the provider of the storage location
+* `storage_path` (Required) - Defines a URI representing the backup storage location
+
 ### Timeouts
 
 The `timeouts` block allows you to specify [timeouts](https://www.terraform.io/language/resources/syntax#operation-timeouts) for certain actions:

docs/resources/rediscloud_subscription_peering.md

@@ -75,7 +75,8 @@ The following arguments are supported:
 * `aws_account_id` - (Required AWS) AWS account ID that the VPC to be peered lives in
 * `region` - (Required AWS) AWS Region that the VPC to be peered lives in
 * `vpc_id` - (Required AWS) Identifier of the VPC to be peered
-* `vpc_cidr` - (Required AWS) CIDR range of the VPC to be peered 
+* `vpc_cidr` - (Optional) CIDR range of the VPC to be peered. Either this or `vpc_cidrs` must be specified
+* `vpc_cidrs` - (Optional) CIDR ranges of the VPC to be peered. Either this or `vpc_cidr` must be specified
 
 **GCP ONLY:**
 * `gcp_project_id` - (Required GCP) GCP project ID that the VPC to be peered lives in

go.mod

@@ -3,7 +3,7 @@ module github.com/RedisLabs/terraform-provider-rediscloud
 go 1.17
 
 require (
-	github.com/RedisLabs/rediscloud-go-api v0.2.0
+	github.com/RedisLabs/rediscloud-go-api v0.3.0
 	github.com/bflad/tfproviderlint v0.28.1
 	github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.1

go.sum

@@ -386,8 +386,8 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ=
 github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
-github.com/RedisLabs/rediscloud-go-api v0.2.0 h1:tkS2AIMLLlI3yeW22dT5DP2BvaYO+K7xvytJvFo8z04=
-github.com/RedisLabs/rediscloud-go-api v0.2.0/go.mod h1:EJXWMnC2ZSM5m7k2TCnmxenYv57o6yGlZXo0ZVnMgIs=
+github.com/RedisLabs/rediscloud-go-api v0.3.0 h1:IiQZ8uFoyZVWIRytFMykiD0ohRijmQcAI3YeAT1kEgg=
+github.com/RedisLabs/rediscloud-go-api v0.3.0/go.mod h1:EJXWMnC2ZSM5m7k2TCnmxenYv57o6yGlZXo0ZVnMgIs=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=

provider/datasource_rediscloud_subscription_peerings_test.go

@@ -33,7 +33,7 @@ func TestAccDataSourceRedisCloudSubscriptionPeerings_basic(t *testing.T) {
 	)
 	dataSourceName := "data.rediscloud_subscription_peerings.example"
 
-	resource.Test(t, resource.TestCase{
+	resource.ParallelTest(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t); testAccAwsPeeringPreCheck(t); testAccAwsPreExistingCloudAccountPreCheck(t) },
 		ProviderFactories: providerFactories,
 		CheckDestroy:      testAccCheckSubscriptionDestroy,

provider/datasource_rediscloud_subscription_test.go

@@ -17,7 +17,7 @@ func TestAccDataSourceRedisCloudSubscription_basic(t *testing.T) {
 	dataSourceName := "data.rediscloud_subscription.example"
 	testCloudAccountName := os.Getenv("AWS_TEST_CLOUD_ACCOUNT_NAME")
 
-	resource.Test(t, resource.TestCase{
+	resource.ParallelTest(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t); testAccAwsPreExistingCloudAccountPreCheck(t) },
 		ProviderFactories: providerFactories,
 		CheckDestroy:      testAccCheckSubscriptionDestroy,

provider/provider.go

@@ -69,7 +69,7 @@ func New(version string) func() *schema.Provider {
 }
 
 // Lock that must be acquired when modifying something related to a subscription as only one _thing_ can modify a subscription and all sub-resources at any time
-var subscriptionMutex = NewPerIdLock()
+var subscriptionMutex = newPerIdLock()
 
 type apiClient struct {
 	client *rediscloud_api.Client

provider/resource_rediscloud_active_active_subscription_database.go

@@ -3,6 +3,7 @@ package provider
 import (
 	"context"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/RedisLabs/rediscloud-go-api/redis"
@@ -45,6 +46,23 @@ func resourceRedisCloudActiveActiveSubscriptionDatabase() *schema.Resource {
 			Delete: schema.DefaultTimeout(10 * time.Minute),
 		},
 
+		CustomizeDiff: func(ctx context.Context, diff *schema.ResourceDiff, i interface{}) error {
+			var keys []string
+			for _, key := range diff.GetChangedKeysPrefix("override_region") {
+				if strings.HasSuffix(key, "time_utc") {
+					keys = append(keys, strings.TrimSuffix(key, ".0.time_utc"))
+				}
+			}
+
+			for _, key := range keys {
+				if err := remoteBackupIntervalSetCorrectly(key)(ctx, diff, i); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		},
+
 		Schema: map[string]*schema.Schema{
 			"subscription_id": {
 				Description: "Identifier of the subscription",
@@ -191,6 +209,39 @@ func resourceRedisCloudActiveActiveSubscriptionDatabase() *schema.Resource {
 							Type:        schema.TypeString,
 							Optional:    true,
 						},
+						"remote_backup": {
+							Description: "An object that specifies the backup options for the database in this region",
+							Type:        schema.TypeList,
+							Optional:    true,
+							MaxItems:    1,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"interval": {
+										Description:      "Defines the frequency of the automatic backup",
+										Type:             schema.TypeString,
+										Required:         true,
+										ValidateDiagFunc: validateDiagFunc(validation.StringInSlice(databases.BackupIntervals(), false)),
+									},
+									"time_utc": {
+										Description:      "Defines the hour automatic backups are made - only applicable when interval is `every-12-hours` or `every-24-hours`",
+										Type:             schema.TypeString,
+										Optional:         true,
+										ValidateDiagFunc: isTime(),
+									},
+									"storage_type": {
+										Description:      "Defines the provider of the storage location",
+										Type:             schema.TypeString,
+										Required:         true,
+										ValidateDiagFunc: validateDiagFunc(validation.StringInSlice(databases.BackupStorageTypes(), false)),
+									},
+									"storage_path": {
+										Description: "Defines a URI representing the backup storage location",
+										Type:        schema.TypeString,
+										Required:    true,
+									},
+								},
+							},
+						},
 					},
 				},
 			},
@@ -210,6 +261,13 @@ func resourceRedisCloudActiveActiveSubscriptionDatabase() *schema.Resource {
 					Type: schema.TypeString,
 				},
 			},
+			"port": {
+				Description:      "TCP port on which the database is available",
+				Type:             schema.TypeInt,
+				ValidateDiagFunc: validateDiagFunc(validation.IntBetween(10000, 19999)),
+				Optional:         true,
+				ForceNew:         true,
+			},
 		},
 	}
 }
@@ -286,6 +344,10 @@ func resourceRedisCloudActiveActiveSubscriptionDatabaseCreate(ctx context.Contex
 		createDatabase.GlobalPassword = redis.String(globalPassword)
 	}
 
+	if v, ok := d.GetOk("port"); ok {
+		createDatabase.PortNumber = redis.Int(v.(int))
+	}
+
 	// Confirm Subscription Active status before creating database
 	err = waitForSubscriptionToBeActive(ctx, subId, api)
 	if err != nil {
@@ -416,6 +478,8 @@ func resourceRedisCloudActiveActiveSubscriptionDatabaseRead(ctx context.Context,
 			regionDbConfig["override_global_alert"] = flattenAlerts(regionDb.Alerts)
 		}
 
+		regionDbConfig["remote_backup"] = flattenBackupPlan(regionDb.Backup, getStateRemoteBackup(d, redis.StringValue(regionDb.Region)), "")
+
 		regionDbConfigs = append(regionDbConfigs, regionDbConfig)
 	}
 
@@ -500,8 +564,8 @@ func resourceRedisCloudActiveActiveSubscriptionDatabaseUpdate(ctx context.Contex
 
 		// Make a list of region-specific source IPs for use in the regions list below
 		var overrideSourceIps []*string
-		for _, source_ip := range dbRegion["override_global_source_ips"].(*schema.Set).List() {
-			overrideSourceIps = append(overrideSourceIps, redis.String(source_ip.(string)))
+		for _, sourceIp := range dbRegion["override_global_source_ips"].(*schema.Set).List() {
+			overrideSourceIps = append(overrideSourceIps, redis.String(sourceIp.(string)))
 		}
 
 		regionProps := &databases.LocalRegionProperties{
@@ -533,6 +597,9 @@ func resourceRedisCloudActiveActiveSubscriptionDatabaseUpdate(ctx context.Contex
 				regionProps.Password = redis.String(d.Get("global_password").(string))
 			}
 		}
+
+		regionProps.RemoteBackup = buildBackupPlan(dbRegion["remote_backup"], nil)
+
 		regions = append(regions, regionProps)
 	}
 
@@ -608,6 +675,16 @@ func getStateOverrideRegion(d *schema.ResourceData, regionName string) map[strin
 	return nil
 }
 
+func getStateRemoteBackup(d *schema.ResourceData, regionName string) []interface{} {
+	for _, region := range d.Get("override_region").(*schema.Set).List() {
+		dbRegion := region.(map[string]interface{})
+		if dbRegion["name"].(string) == regionName {
+			return dbRegion["remote_backup"].([]interface{})
+		}
+	}
+	return nil
+}
+
 func getStateAlertsFromDbRegion(dbRegion map[string]interface{}) []*databases.UpdateAlert {
 	// Make a list of region-specific alert configurations for use in the regions list below
 	if dbRegion == nil {