Polish Secrets Manager Integration (#743)

- fixed handling failFast and optional flag
- added logs
- added tests

Author: maciejwalkowiak

docs/src/main/asciidoc/_configprops.adoc

@@ -1,67 +1,67 @@
 |===
 |Name | Default | Description
 
-|aws.paramstore.default-context | application | 
+|aws.paramstore.default-context | application |
 |aws.paramstore.enabled | true | Is AWS Parameter Store support enabled.
 |aws.paramstore.endpoint |  | Overrides the default endpoint.
 |aws.paramstore.fail-fast | true | Throw exceptions during config lookup if true, otherwise, log warnings.
 |aws.paramstore.name |  | Alternative to spring.application.name to use in looking up values in AWS Parameter Store.
 |aws.paramstore.prefix | /config | Prefix indicating first level for every property. Value must start with a forward slash followed by a valid path segment or be empty. Defaults to "/config".
-|aws.paramstore.profile-separator | _ | 
+|aws.paramstore.profile-separator | _ |
 |aws.paramstore.region |  | If region value is not null or empty it will be used in creation of AWSSimpleSystemsManagement.
-|aws.secretsmanager.default-context | application | 
+|aws.secretsmanager.default-context | application |
 |aws.secretsmanager.enabled | true | Is AWS Secrets Manager support enabled.
 |aws.secretsmanager.endpoint |  | Overrides the default endpoint.
 |aws.secretsmanager.fail-fast | true | Throw exceptions during config lookup if true, otherwise, log warnings.
 |aws.secretsmanager.name |  | Alternative to spring.application.name to use in looking up values in AWS Secrets Manager.
 |aws.secretsmanager.prefix | /secret | Prefix indicating first level for every property. Value must start with a forward slash followed by a valid path segment or be empty. Defaults to "/config".
-|aws.secretsmanager.profile-separator | _ | 
+|aws.secretsmanager.profile-separator | _ |
 |aws.secretsmanager.region |  | If region value is not null or empty it will be used in creation of AWSSecretsManager.
 |cloud.aws.credentials.access-key |  | The access key to be used with a static provider.
 |cloud.aws.credentials.instance-profile | false | Configures an instance profile credentials provider with no further configuration.
 |cloud.aws.credentials.profile-name |  | The AWS profile name.
 |cloud.aws.credentials.profile-path |  | The AWS profile path.
 |cloud.aws.credentials.secret-key |  | The secret key to be used with a static provider.
-|cloud.aws.elasticache.cache-names |  | 
+|cloud.aws.elasticache.cache-names |  |
 |cloud.aws.elasticache.clusters |  | Configures the cache clusters for the caching configuration. Support one or multiple caches {@link Cluster} configurations with their physical cache name (as configured in the ElastiCache service) or their logical cache name if the caches are configured inside a stack and {@link org.springframework.cloud.aws.context.config.annotation.EnableStackConfiguration} annotation is used inside the application.
 |cloud.aws.elasticache.default-expiration | 0 | Configures the default expiration time in seconds if there is no custom expiration time configuration with a {@link Cluster} configuration for the cache. The expiration time is implementation specific (e.g. Redis or Memcached) and could therefore differ in the behaviour based on the cache implementation.
 |cloud.aws.elasticache.enabled | true | Enables ElastiCache integration.
-|cloud.aws.elasticache.expiry-time-per-cache |  | 
+|cloud.aws.elasticache.expiry-time-per-cache |  |
 |cloud.aws.instance.data.enabled | false | Enables Instance Data integration.
 |cloud.aws.loader.core-pool-size | 1 | The core pool size of the Task Executor used for parallel S3 interaction. @see org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor#setCorePoolSize(int)
 |cloud.aws.loader.max-pool-size |  | The maximum pool size of the Task Executor used for parallel S3 interaction. @see org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor#setMaxPoolSize(int)
 |cloud.aws.loader.queue-capacity |  | The maximum queue capacity for backed up S3 requests. @see org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor#setQueueCapacity(int)
 |cloud.aws.mail.enabled | true | Enables Mail integration.
-|cloud.aws.mail.endpoint |  | 
-|cloud.aws.mail.region |  | 
+|cloud.aws.mail.endpoint |  |
+|cloud.aws.mail.region |  |
 |cloud.aws.rds.enabled | true | Enables RDS integration.
-|cloud.aws.rds.endpoint |  | 
+|cloud.aws.rds.endpoint |  |
 |cloud.aws.rds.instances |  | List of RdsInstances.
-|cloud.aws.rds.region |  | 
+|cloud.aws.rds.region |  |
 |cloud.aws.s3.endpoint |  | Overrides the default endpoint.
 |cloud.aws.s3.region |  | Overrides the default region.
 |cloud.aws.sns.enabled | true | Enables SNS integration.
-|cloud.aws.sns.endpoint |  | 
-|cloud.aws.sns.region |  | 
+|cloud.aws.sns.endpoint |  |
+|cloud.aws.sns.region |  |
 |cloud.aws.sqs.enabled | true | Enables SQS integration.
-|cloud.aws.sqs.endpoint |  | 
+|cloud.aws.sqs.endpoint |  |
 |cloud.aws.sqs.handler.default-deletion-policy |  | Configures global deletion policy used if deletion policy is not explicitly set on {@link SqsListener}.
 |cloud.aws.sqs.listener.auto-startup | true | Configures if this container should be automatically started.
 |cloud.aws.sqs.listener.back-off-time |  | The number of milliseconds the polling thread must wait before trying to recover when an error occurs (e.g. connection timeout).
 |cloud.aws.sqs.listener.max-number-of-messages | 10 | The maximum number of messages that should be retrieved during one poll to the Amazon SQS system. This number must be a positive, non-zero number that has a maximum number of 10. Values higher then 10 are currently not supported by the queueing system.
 |cloud.aws.sqs.listener.queue-stop-timeout |  | The queue stop timeout that waits for a queue to stop before interrupting the running thread.
 |cloud.aws.sqs.listener.visibility-timeout |  | The duration (in seconds) that the received messages are hidden from subsequent poll requests after being retrieved from the system.
 |cloud.aws.sqs.listener.wait-timeout | 20 | The wait timeout that the poll request will wait for new message to arrive if the are currently no messages on the queue. Higher values will reduce poll request to the system significantly. The value should be between 1 and 20. For more information read the <a href= "https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html">documentation</a>.
-|cloud.aws.sqs.region |  | 
+|cloud.aws.sqs.region |  |
 |cloud.aws.stack.auto | true | Enables the automatic stack name detection for the application.
 |cloud.aws.stack.enabled | true | Enables Stack integration.
 |cloud.aws.stack.name |  | The name of the manually configured stack name that will be used to retrieve the resources.
 |spring.cloud.aws.security.cognito.algorithm | RS256 | Encryption algorithm used to sign the JWK token.
 |spring.cloud.aws.security.cognito.app-client-id |  | Non-dynamic audience string to validate.
 |spring.cloud.aws.security.cognito.enabled | true | Enables Cognito integration.
-|spring.cloud.aws.security.cognito.region |  | 
-|spring.cloud.aws.security.cognito.user-pool-id |  | 
+|spring.cloud.aws.security.cognito.region |  |
+|spring.cloud.aws.security.cognito.user-pool-id |  |
 |spring.cloud.aws.ses.enabled | true | Enables Simple Email Service integration.
 |spring.cloud.aws.ses.region |  | Overrides the default region.
 
-|===
+|===

spring-cloud-aws-secrets-manager-config/src/main/java/org/springframework/cloud/aws/secretsmanager/AwsSecretsManagerPropertySource.java

@@ -16,7 +16,6 @@
 
 package org.springframework.cloud.aws.secretsmanager;
 
-import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
@@ -25,35 +24,40 @@
 import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
 import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
 import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.springframework.core.env.EnumerablePropertySource;
 
 /**
  * Retrieves secret value under the given context / path from the AWS Secrets Manager
- * using the provided SM client.
+ * using the provided Secrets Manager client.
  *
  * @author Fabio Maia
+ * @author Maciej Walkowiak
  * @since 2.0.0
  */
 public class AwsSecretsManagerPropertySource extends EnumerablePropertySource<AWSSecretsManager> {
 
 	private final ObjectMapper jsonMapper = new ObjectMapper();
 
-	private String context;
+	private final String context;
 
-	private Map<String, Object> properties = new LinkedHashMap<>();
+	private final Map<String, Object> properties = new LinkedHashMap<>();
 
 	public AwsSecretsManagerPropertySource(String context, AWSSecretsManager smClient) {
 		super(context, smClient);
 		this.context = context;
 	}
 
+	/**
+	 * Loads properties from the Secrets Manager secret.
+	 * @throws ResourceNotFoundException if specified secret does not exist in the
+	 * database.
+	 */
 	public void init() {
-		GetSecretValueRequest secretValueRequest = new GetSecretValueRequest();
-		secretValueRequest.setSecretId(context);
-		readSecretValue(secretValueRequest);
+		readSecretValue(new GetSecretValueRequest().withSecretId(context));
 	}
 
 	@Override
@@ -78,10 +82,7 @@ private void readSecretValue(GetSecretValueRequest secretValueRequest) {
 				properties.put(secretEntry.getKey(), secretEntry.getValue());
 			}
 		}
-		catch (ResourceNotFoundException e) {
-			logger.debug("AWS secret not found from " + secretValueRequest.getSecretId());
-		}
-		catch (IOException e) {
+		catch (JsonProcessingException e) {
 			throw new RuntimeException(e);
 		}
 	}

spring-cloud-aws-secrets-manager-config/src/main/java/org/springframework/cloud/aws/secretsmanager/AwsSecretsManagerPropertySourceLocator.java

@@ -38,6 +38,9 @@
  * name and default context permutations. Mostly copied from Spring Cloud Consul's config
  * support.
  *
+ * Note: this class is used only by legacy Spring Cloud Bootstrap phase based config
+ * loading.
+ *
  * @author Fabio Maia
  * @author Matej Nedic
  * @author Eddú Meléndez
@@ -86,9 +89,11 @@ public PropertySource<?> locate(Environment environment) {
 		CompositePropertySource composite = new CompositePropertySource(this.propertySourceName);
 
 		for (String propertySourceContext : this.contexts) {
-			PropertySource<AWSSecretsManager> propertySource = sources.createPropertySource(propertySourceContext, true,
-					this.smClient);
-			composite.addPropertySource(propertySource);
+			PropertySource<AWSSecretsManager> propertySource = sources.createPropertySource(propertySourceContext,
+					!this.properties.isFailFast(), this.smClient);
+			if (propertySource != null) {
+				composite.addPropertySource(propertySource);
+			}
 		}
 
 		return composite;

spring-cloud-aws-secrets-manager-config/src/main/java/org/springframework/cloud/aws/secretsmanager/AwsSecretsManagerPropertySources.java

@@ -64,20 +64,30 @@ private void addProfiles(List<String> contexts, String baseContext, List<String>
 		}
 	}
 
+	/**
+	 * Creates property source for given context.
+	 * @param context property source context equivalent to the secret name
+	 * @param optional if creating context should fail with exception if secret cannot be
+	 * loaded
+	 * @param client Secret Manager client
+	 * @return a property source or null if secret could not be loaded and optional is set
+	 * to true
+	 */
 	public AwsSecretsManagerPropertySource createPropertySource(String context, boolean optional,
 			AWSSecretsManager client) {
+		log.info("Loading secrets from AWS Secret Manager secret with name: " + context + ", optional: " + optional);
 		try {
 			AwsSecretsManagerPropertySource propertySource = new AwsSecretsManagerPropertySource(context, client);
 			propertySource.init();
 			return propertySource;
 			// TODO: howto call close when /refresh
 		}
 		catch (Exception e) {
-			if (this.properties.isFailFast() || !optional) {
+			if (!optional) {
 				throw new AwsSecretsManagerPropertySourceNotFoundException(e);
 			}
 			else {
-				log.warn("Unable to load AWS secret from " + context, e);
+				log.warn("Unable to load AWS secret from " + context + ". " + e.getMessage());
 			}
 		}
 		return null;

spring-cloud-aws-secrets-manager-config/src/test/java/org/springframework/cloud/aws/secretsmanager/AwsSecretsManagerPropertySourceLocatorTest.java

@@ -22,12 +22,16 @@
 import com.amazonaws.services.secretsmanager.AWSSecretsManager;
 import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
 import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
+import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
 import org.junit.jupiter.api.Test;
 
+import org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySources.AwsSecretsManagerPropertySourceNotFoundException;
+import org.springframework.core.env.CompositePropertySource;
 import org.springframework.core.env.PropertySource;
 import org.springframework.mock.env.MockEnvironment;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -37,6 +41,7 @@
  *
  * @author Anthony Foulfoin
  * @author Matej Nedic
+ * @author Maciej Walkowiak.
  */
 class AwsSecretsManagerPropertySourceLocatorTest {
 
@@ -74,7 +79,7 @@ void locate_nameNotSpecifiedInConstructor_returnsPropertySourceWithDefaultName()
 	}
 
 	@Test
-	public void contextExpectedToHave2Elements() {
+	void contextExpectedToHave2Elements() {
 		AwsSecretsManagerProperties properties = new AwsSecretsManagerPropertiesBuilder()
 				.withDefaultContext("application").withName("application").build();
 
@@ -91,7 +96,7 @@ public void contextExpectedToHave2Elements() {
 	}
 
 	@Test
-	public void contextExpectedToHave4Elements() {
+	void contextExpectedToHave4Elements() {
 		AwsSecretsManagerProperties properties = new AwsSecretsManagerPropertiesBuilder()
 				.withDefaultContext("application").withName("messaging-service").build();
 
@@ -108,7 +113,7 @@ public void contextExpectedToHave4Elements() {
 	}
 
 	@Test
-	public void contextSpecificOrderExpected() {
+	void contextSpecificOrderExpected() {
 		AwsSecretsManagerProperties properties = new AwsSecretsManagerPropertiesBuilder()
 				.withDefaultContext("application").withName("messaging-service").build();
 
@@ -127,7 +132,34 @@ public void contextSpecificOrderExpected() {
 		assertThat(contextToBeTested.get(1)).isEqualTo("/secret/messaging-service");
 		assertThat(contextToBeTested.get(2)).isEqualTo("/secret/application_test");
 		assertThat(contextToBeTested.get(3)).isEqualTo("/secret/application");
+	}
+
+	@Test
+	void whenFailFastIsTrueAndSecretDoesNotExistThrowsException() {
+		AwsSecretsManagerProperties properties = new AwsSecretsManagerProperties();
+		properties.setFailFast(true);
+
+		when(smClient.getSecretValue(any(GetSecretValueRequest.class))).thenThrow(ResourceNotFoundException.class);
+
+		AwsSecretsManagerPropertySourceLocator locator = new AwsSecretsManagerPropertySourceLocator(smClient,
+				properties);
+		assertThatThrownBy(() -> locator.locate(env))
+				.isInstanceOf(AwsSecretsManagerPropertySourceNotFoundException.class);
+	}
+
+	@Test
+	void whenFailFastIsFalseAndSecretDoesNotExistReturnsEmptyPropertySource() {
+		AwsSecretsManagerProperties properties = new AwsSecretsManagerProperties();
+		properties.setFailFast(false);
+
+		when(smClient.getSecretValue(any(GetSecretValueRequest.class))).thenThrow(ResourceNotFoundException.class);
+
+		AwsSecretsManagerPropertySourceLocator locator = new AwsSecretsManagerPropertySourceLocator(smClient,
+				properties);
+
+		CompositePropertySource result = (CompositePropertySource) locator.locate(env);
 
+		assertThat(result.getPropertySources()).isEmpty();
 	}
 
 	private final static class AwsSecretsManagerPropertiesBuilder {

spring-cloud-aws-secrets-manager-config/src/test/java/org/springframework/cloud/aws/secretsmanager/AwsSecretsManagerPropertySourceTest.java

@@ -19,26 +19,28 @@
 import com.amazonaws.services.secretsmanager.AWSSecretsManager;
 import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
 import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
+import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 class AwsSecretsManagerPropertySourceTest {
 
-	private AWSSecretsManager smClient = mock(AWSSecretsManager.class);
+	private AWSSecretsManager client = mock(AWSSecretsManager.class);
 
 	private AwsSecretsManagerPropertySource propertySource = new AwsSecretsManagerPropertySource("/config/myservice",
-			smClient);
+			client);
 
 	@Test
 	void shouldParseSecretValue() {
 		GetSecretValueResult secretValueResult = new GetSecretValueResult();
 		secretValueResult.setSecretString("{\"key1\": \"value1\", \"key2\": \"value2\"}");
 
-		when(smClient.getSecretValue(any(GetSecretValueRequest.class))).thenReturn(secretValueResult);
+		when(client.getSecretValue(any(GetSecretValueRequest.class))).thenReturn(secretValueResult);
 
 		propertySource.init();
 
@@ -47,4 +49,12 @@ void shouldParseSecretValue() {
 		assertThat(propertySource.getProperty("key2")).isEqualTo("value2");
 	}
 
+	@Test
+	void throwsExceptionWhenSecretNotFound() {
+		when(client.getSecretValue(any(GetSecretValueRequest.class)))
+				.thenThrow(new ResourceNotFoundException("secret not found"));
+
+		assertThatThrownBy(() -> propertySource.init()).isInstanceOf(ResourceNotFoundException.class);
+	}
+
 }

spring-cloud-starter-aws-secrets-manager-config/src/main/java/org/springframework/cloud/aws/autoconfigure/secretsmanager/AwsSecretsManagerConfigDataLoader.java

@@ -27,7 +27,10 @@
 import org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySource;
 
 /**
+ * Loads config data from AWS Secret Manager.
+ *
  * @author Eddú Meléndez
+ * @author Maciej Walkowiak
  * @since 2.3.0
  */
 public class AwsSecretsManagerConfigDataLoader implements ConfigDataLoader<AwsSecretsManagerConfigDataResource> {
@@ -38,7 +41,12 @@ public ConfigData load(ConfigDataLoaderContext context, AwsSecretsManagerConfigD
 			AWSSecretsManager ssm = context.getBootstrapContext().get(AWSSecretsManager.class);
 			AwsSecretsManagerPropertySource propertySource = resource.getPropertySources()
 					.createPropertySource(resource.getContext(), resource.isOptional(), ssm);
-			return new ConfigData(Collections.singletonList(propertySource));
+			if (propertySource != null) {
+				return new ConfigData(Collections.singletonList(propertySource));
+			}
+			else {
+				return null;
+			}
 		}
 		catch (Exception e) {
 			throw new ConfigDataResourceNotFoundException(resource, e);