item) {
+ // data and validation payload may be null
+ String data = item.get(this.dataAttr);
+ String validation = item.get(this.validationAttr);
+ return new DataRecord(item.get(keyAttr),
+ DataRecord.Status.valueOf(item.get(this.statusAttr)),
+ Long.parseLong(item.get(this.expiryAttr)),
+ data,
+ validation,
+ item.get(this.inProgressExpiryAttr) != null ?
+ OptionalLong.of(Long.parseLong(item.get(this.inProgressExpiryAttr))) :
+ OptionalLong.empty());
+ }
+
+ /**
+ * Use this builder to get an instance of {@link RedisPersistenceStore}.
+ * With this builder you can configure the characteristics of the Redis hash attributes.
+ * You can also set a custom {@link JedisPool}.
+ */
+ public static class Builder {
+ private String keyPrefixName = "idempotency";
+ private String keyAttr = "id";
+ private String expiryAttr = "expiration";
+ private String inProgressExpiryAttr = "in-progress-expiration";
+ private String statusAttr = "status";
+ private String dataAttr = "data";
+ private String validationAttr = "validation";
+ private JedisPooled jedisPool;
+
+ /**
+ * Initialize and return a new instance of {@link RedisPersistenceStore}.
+ * Example:
+ *
+ * RedisPersistenceStore.builder().withKeyAttr("uuid").build();
+ *
+ *
+ * @return an instance of the {@link RedisPersistenceStore}
+ */
+ public RedisPersistenceStore build() {
+ return new RedisPersistenceStore(keyPrefixName, keyAttr, expiryAttr,
+ inProgressExpiryAttr, statusAttr, dataAttr, validationAttr, jedisPool);
+ }
+
+ /**
+ * Redis prefix for the hash key (optional), by default "idempotency"
+ *
+ * @param keyPrefixName name of the key prefix
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withKeyPrefixName(String keyPrefixName) {
+ this.keyPrefixName = keyPrefixName;
+ return this;
+ }
+
+ /**
+ * Redis name for hash key (optional), by default "id"
+ *
+ * @param keyAttr name of the key attribute of the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withKeyAttr(String keyAttr) {
+ this.keyAttr = keyAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for expiry timestamp (optional), by default "expiration"
+ *
+ * @param expiryAttr name of the expiry attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withExpiryAttr(String expiryAttr) {
+ this.expiryAttr = expiryAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for in progress expiry timestamp (optional), by default "in-progress-expiration"
+ *
+ * @param inProgressExpiryAttr name of the attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
+ this.inProgressExpiryAttr = inProgressExpiryAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for status (optional), by default "status"
+ *
+ * @param statusAttr name of the status attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withStatusAttr(String statusAttr) {
+ this.statusAttr = statusAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for response data (optional), by default "data"
+ *
+ * @param dataAttr name of the data attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withDataAttr(String dataAttr) {
+ this.dataAttr = dataAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for validation (optional), by default "validation"
+ *
+ * @param validationAttr name of the validation attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withValidationAttr(String validationAttr) {
+ this.validationAttr = validationAttr;
+ return this;
+ }
+
+ /**
+ * Custom {@link JedisPool} used to query DynamoDB (optional).
+ *
+ * @param jedisPool the {@link JedisPool} instance to use
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withJedisPooled(JedisPooled jedisPool) {
+ this.jedisPool = jedisPool;
+ return this;
+ }
+ }
+}
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
new file mode 100644
index 000000000..21fdd2652
--- /dev/null
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.idempotency.redis;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_HOST;
+import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_PORT;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junitpioneer.jupiter.SetEnvironmentVariable;
+import redis.clients.jedis.JedisPooled;
+import redis.embedded.RedisServer;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
+import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
+
+@SetEnvironmentVariable(key = REDIS_HOST, value = "localhost")
+@SetEnvironmentVariable(key = REDIS_PORT, value = "6379")
+public class RedisPersistenceStoreTest {
+ static RedisServer redisServer;
+ private final RedisPersistenceStore redisPersistenceStore = RedisPersistenceStore.builder().build();
+ private final JedisPooled jedisPool = new JedisPooled();
+
+ public RedisPersistenceStoreTest() {
+ }
+
+ @BeforeAll
+ public static void init() {
+ redisServer = new RedisServer(6379);
+ redisServer.start();
+ }
+
+ @AfterAll
+ public static void stop() {
+ redisServer.stop();
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis() {
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
+ Instant now = Instant.now();
+ long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
+ store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.minus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", "Fake Data");
+
+ long ttl = 3600;
+ long expiry2 = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ jedisPool.hset("idempotency:id:key", item);
+ redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ long progressExpiry = now.minus(30, ChronoUnit.SECONDS).toEpochMilli();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ item.put("data", "Fake Data");
+ item.put("in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ }
+
+ @Test
+ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry)); // not expired
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", "Fake Data");
+
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ assertThatThrownBy(() -> redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now)
+ ).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("data")).isEqualTo("Fake Data");
+ }
+
+ @Test
+ public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond(); // not expired
+ long progressExpiry = now.plus(30, ChronoUnit.SECONDS).toEpochMilli(); // not expired
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ item.put("data", "Fake Data");
+ item.put("in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ assertThatThrownBy(() -> redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ "Fake Data 2",
+ null
+ ), now))
+ .isInstanceOf(IdempotencyItemAlreadyExistsException.class);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("data")).isEqualTo("Fake Data");
+ }
+
+ @Test
+ public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", ("Fake Data"));
+ jedisPool.hset("idempotency:id:key", item);
+
+ DataRecord record = redisPersistenceStore.getRecord("key");
+
+ assertThat(record.getIdempotencyKey()).isEqualTo("key");
+ assertThat(record.getStatus()).isEqualTo(DataRecord.Status.COMPLETED);
+ assertThat(record.getResponseData()).isEqualTo("Fake Data");
+ assertThat(record.getExpiryTimestamp()).isEqualTo(expiry);
+ }
+
+ @Test
+ public void getRecord_shouldThrowException_whenRecordIsAbsent() {
+ assertThatThrownBy(() -> redisPersistenceStore.getRecord("key")).isInstanceOf(
+ IdempotencyItemNotFoundException.class);
+ }
+
+ @Test
+ public void updateRecord_shouldUpdateRecord() {
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("idempotency:id:key", item);
+ // enable payload validation
+ redisPersistenceStore.configure(IdempotencyConfig.builder().withPayloadValidationJMESPath("path").build(),
+ null);
+
+ long ttl = 3600;
+ expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
+ redisPersistenceStore.updateRecord(record);
+
+ Map itemInDb = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(itemInDb.get("status")).isEqualTo("COMPLETED");
+ assertThat(itemInDb.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(itemInDb.get("data")).isEqualTo("Fake result");
+ assertThat(itemInDb.get("validation")).isEqualTo("hash");
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void deleteRecord_shouldDeleteRecord() {
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("idempotency:id:key", item);
+
+ redisPersistenceStore.deleteRecord("key");
+
+ Map items = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(items.isEmpty()).isTrue();
+ }
+
+
+ @Test
+ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
+ try {
+ RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
+ .withKeyPrefixName("items-idempotency")
+ .withJedisPooled(jedisPool)
+ .withDataAttr("result")
+ .withExpiryAttr("expiry")
+ .withKeyAttr("key")
+ .withStatusAttr("state")
+ .withValidationAttr("valid")
+ .build();
+
+ Instant now = Instant.now();
+ DataRecord record = new DataRecord(
+ "mykey",
+ DataRecord.Status.INPROGRESS,
+ now.plus(400, ChronoUnit.SECONDS).getEpochSecond(),
+ null,
+ null
+ );
+ // PUT
+ persistenceStore.putRecord(record, now);
+
+ Map itemInDb = jedisPool.hgetAll("items-idempotency:key:mykey");
+
+ // GET
+ DataRecord recordInDb = persistenceStore.getRecord("mykey");
+
+ assertThat(itemInDb).isNotNull();
+ assertThat(itemInDb.get("state")).isEqualTo(recordInDb.getStatus().toString());
+ assertThat(itemInDb.get("expiry")).isEqualTo(String.valueOf(recordInDb.getExpiryTimestamp()));
+
+ // UPDATE
+ DataRecord updatedRecord = new DataRecord(
+ "mykey",
+ DataRecord.Status.COMPLETED,
+ now.plus(500, ChronoUnit.SECONDS).getEpochSecond(),
+ "response",
+ null
+ );
+ persistenceStore.updateRecord(updatedRecord);
+ recordInDb = persistenceStore.getRecord("mykey");
+ assertThat(recordInDb).isEqualTo(updatedRecord);
+
+ // DELETE
+ persistenceStore.deleteRecord("mykey");
+ assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isEqualTo(0);
+
+ } finally {
+ try {
+ jedisPool.del("items-idempotency:key:mykey");
+ } catch (Exception e) {
+ // OK
+ }
+ }
+ }
+
+ @Test
+ @SetEnvironmentVariable(key = software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV, value = "true")
+ public void idempotencyDisabled_noClientShouldBeCreated() {
+ RedisPersistenceStore store = RedisPersistenceStore.builder().build();
+ assertThatThrownBy(() -> store.getRecord("key")).isInstanceOf(NullPointerException.class);
+ }
+
+ @AfterEach
+ public void emptyDB() {
+ jedisPool.del("idempotency:id:key");
+ }
+
+}
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
index ba7da69bf..42e17b5db 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
@@ -27,4 +27,9 @@ public IdempotencyItemAlreadyExistsException() {
public IdempotencyItemAlreadyExistsException(String msg, Throwable e) {
super(msg, e);
}
+
+ public IdempotencyItemAlreadyExistsException(String msg) {
+ super(msg);
+ }
+
}
From d18052954ef7db081fa58f3d89bcd44d2bda4ae4 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 24 Nov 2023 16:42:47 +0200
Subject: [PATCH 02/19] Fix string replacement and add unit test
---
.../redis/RedisPersistenceStore.java | 5 +----
.../redis/RedisPersistenceStoreTest.java | 20 +++++++++++++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index 1a1065e0b..a33e7182c 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -16,18 +16,15 @@
import java.time.Instant;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
-import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
-import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPooled;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
@@ -171,7 +168,7 @@ private Object putItemOnCondition(DataRecord record, Instant now, String inProgr
// only insert in-progress-expiry if it is set
if (inProgressExpiry != null) {
- insertItemExpression.replace(")", ", KEYS[4], ARGV[6])");
+ insertItemExpression = insertItemExpression.replace(")", ", KEYS[4], ARGV[6])");
}
// if redisHashExistsExpression or itemExpiredExpression or itemIsInProgressExpression then insertItemExpression
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index 21fdd2652..b44db768f 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -23,6 +23,7 @@
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.Map;
+import java.util.OptionalLong;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -72,6 +73,25 @@ public void putRecord_shouldCreateItemInRedis() {
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
+ @Test
+ public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ OptionalLong progressExpiry = OptionalLong.of(now.minus(30, ChronoUnit.SECONDS).toEpochMilli());
+ redisPersistenceStore.putRecord(
+ new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("in-progress-expiration")).isEqualTo(String.valueOf(progressExpiry.getAsLong()));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
@Test
public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
From 327fcb35c76ed5495a3ae340017b209048292a4d Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 24 Nov 2023 17:43:31 +0200
Subject: [PATCH 03/19] Address sonar findings
---
.../redis/RedisPersistenceStore.java | 102 +++++++-------
.../redis/RedisPersistenceStoreTest.java | 130 +++++++++---------
2 files changed, 115 insertions(+), 117 deletions(-)
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index a33e7182c..909953832 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -95,11 +95,10 @@ public static Builder builder() {
* @return
*/
private static JedisClientConfig getJedisClientConfig() {
- JedisClientConfig config = DefaultJedisClientConfig.builder()
+ return DefaultJedisClientConfig.builder()
.user(System.getenv().get(Constants.REDIS_USER))
.password(System.getenv().get(Constants.REDIS_SECRET))
.build();
- return config;
}
JedisClientConfig config = getJedisClientConfig();
@@ -117,53 +116,53 @@ public DataRecord getRecord(String idempotencyKey) throws IdempotencyItemNotFoun
}
/**
- * Store's the given idempotency record in the redis store. If there
- * is an existing record that has expired - either due to the
- * cache expiry or due to the in_progress_expiry - the record
+ * Store's the given idempotency dataRecord in the redis store. If there
+ * is an existing dataRecord that has expired - either due to the
+ * cache expiry or due to the in_progress_expiry - the dataRecord
* will be overwritten and the idempotent operation can continue.
*
* Note: This method writes only expiry and status information - not
* the results of the operation itself.
*
- * @param record DataRecord instance to store
+ * @param dataRecord DataRecord instance to store
* @param now
* @throws IdempotencyItemAlreadyExistsException
*/
@Override
- public void putRecord(DataRecord record, Instant now) {
+ public void putRecord(DataRecord dataRecord, Instant now) {
String inProgressExpiry = null;
- if (record.getInProgressExpiryTimestamp().isPresent()) {
- inProgressExpiry = String.valueOf(record.getInProgressExpiryTimestamp().getAsLong());
+ if (dataRecord.getInProgressExpiryTimestamp().isPresent()) {
+ inProgressExpiry = String.valueOf(dataRecord.getInProgressExpiryTimestamp().getAsLong());
}
- LOG.debug("Putting record for idempotency key: {}", record.getIdempotencyKey());
+ LOG.debug("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
- Object execRes = putItemOnCondition(record, now, inProgressExpiry);
+ Object execRes = putItemOnCondition(dataRecord, now, inProgressExpiry);
if (execRes == null) {
- String msg = String.format("Failed to put record for already existing idempotency key: %s",
- getKey(record.getIdempotencyKey()));
+ String msg = String.format("Failed to put dataRecord for already existing idempotency key: %s",
+ getKey(dataRecord.getIdempotencyKey()));
LOG.debug(msg);
throw new IdempotencyItemAlreadyExistsException(msg);
} else {
- LOG.debug("Record for idempotency key is set: {}", record.getIdempotencyKey());
- jedisPool.expireAt(getKey(record.getIdempotencyKey()), record.getExpiryTimestamp());
+ LOG.debug("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
+ jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
}
- private Object putItemOnCondition(DataRecord record, Instant now, String inProgressExpiry) {
+ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inProgressExpiry) {
// if item with key exists
String redisHashExistsExpression = "redis.call('exists', KEYS[1]) == 0";
// if expiry timestamp is exceeded for existing item
String itemExpiredExpression = "redis.call('hget', KEYS[1], KEYS[2]) < ARGV[1]";
- // if item status attribute exists and has value is INPROGRESS
+ // if item status field exists and has value is INPROGRESS
// and the in-progress-expiry timestamp is still valid
String itemIsInProgressExpression = "(redis.call('hexists', KEYS[1], KEYS[4]) ~= 0" +
" and redis.call('hget', KEYS[1], KEYS[4]) < ARGV[2]" +
" and redis.call('hget', KEYS[1], KEYS[3]) == ARGV[3])";
- // insert item and attributes
+ // insert item and fields
String insertItemExpression = "return redis.call('hset', KEYS[1], KEYS[2], ARGV[4], KEYS[3], ARGV[5])";
// only insert in-progress-expiry if it is set
@@ -176,41 +175,40 @@ private Object putItemOnCondition(DataRecord record, Instant now, String inProgr
redisHashExistsExpression, itemExpiredExpression,
itemIsInProgressExpression, insertItemExpression);
- List params = new ArrayList<>();
- params.add(getKey(record.getIdempotencyKey()));
- params.add( this.expiryAttr);
- params.add(this.statusAttr);
- params.add(this.inProgressExpiryAttr);
- params.add(String.valueOf(now.getEpochSecond()));
- params.add(String.valueOf(now.toEpochMilli()));
- params.add(INPROGRESS.toString());
- params.add(String.valueOf(record.getExpiryTimestamp()));
- params.add(record.getStatus().toString());
+ List fields = new ArrayList<>();
+ fields.add(getKey(dataRecord.getIdempotencyKey()));
+ fields.add(this.expiryAttr);
+ fields.add(this.statusAttr);
+ fields.add(this.inProgressExpiryAttr);
+ fields.add(String.valueOf(now.getEpochSecond()));
+ fields.add(String.valueOf(now.toEpochMilli()));
+ fields.add(INPROGRESS.toString());
+ fields.add(String.valueOf(dataRecord.getExpiryTimestamp()));
+ fields.add(dataRecord.getStatus().toString());
if (inProgressExpiry != null) {
- params.add(inProgressExpiry);
+ fields.add(inProgressExpiry);
}
- String []arr = new String[params.size()];
- Object execRes = jedisPool.eval(luaScript, 4, (String[]) params.toArray(arr));
- return execRes;
+ String[] arr = new String[fields.size()];
+ return jedisPool.eval(luaScript, 4, (String[]) fields.toArray(arr));
}
@Override
- public void updateRecord(DataRecord record) {
- LOG.debug("Updating record for idempotency key: {}", record.getIdempotencyKey());
+ public void updateRecord(DataRecord dataRecord) {
+ LOG.debug("Updating dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
Map item = new HashMap<>();
- item.put(this.dataAttr, record.getResponseData());
- item.put(this.expiryAttr, String.valueOf(record.getExpiryTimestamp()));
- item.put(this.statusAttr, String.valueOf(record.getStatus().toString()));
+ item.put(this.dataAttr, dataRecord.getResponseData());
+ item.put(this.expiryAttr, String.valueOf(dataRecord.getExpiryTimestamp()));
+ item.put(this.statusAttr, String.valueOf(dataRecord.getStatus().toString()));
if (payloadValidationEnabled) {
- item.put(this.validationAttr, record.getPayloadHash());
+ item.put(this.validationAttr, dataRecord.getPayloadHash());
}
- jedisPool.hset(getKey(record.getIdempotencyKey()), item);
- jedisPool.expireAt(getKey(record.getIdempotencyKey()), record.getExpiryTimestamp());
+ jedisPool.hset(getKey(dataRecord.getIdempotencyKey()), item);
+ jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
@Override
@@ -252,7 +250,7 @@ private DataRecord itemToRecord(Map item) {
/**
* Use this builder to get an instance of {@link RedisPersistenceStore}.
- * With this builder you can configure the characteristics of the Redis hash attributes.
+ * With this builder you can configure the characteristics of the Redis hash fields.
* You can also set a custom {@link JedisPool}.
*/
public static class Builder {
@@ -293,7 +291,7 @@ public Builder withKeyPrefixName(String keyPrefixName) {
/**
* Redis name for hash key (optional), by default "id"
*
- * @param keyAttr name of the key attribute of the hash
+ * @param keyAttr name of the key field of the hash
* @return the builder instance (to chain operations)
*/
public Builder withKeyAttr(String keyAttr) {
@@ -302,9 +300,9 @@ public Builder withKeyAttr(String keyAttr) {
}
/**
- * Redis attribute name for expiry timestamp (optional), by default "expiration"
+ * Redis field name for expiry timestamp (optional), by default "expiration"
*
- * @param expiryAttr name of the expiry attribute in the hash
+ * @param expiryAttr name of the expiry field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withExpiryAttr(String expiryAttr) {
@@ -313,9 +311,9 @@ public Builder withExpiryAttr(String expiryAttr) {
}
/**
- * Redis attribute name for in progress expiry timestamp (optional), by default "in-progress-expiration"
+ * Redis field name for in progress expiry timestamp (optional), by default "in-progress-expiration"
*
- * @param inProgressExpiryAttr name of the attribute in the hash
+ * @param inProgressExpiryAttr name of the field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
@@ -324,9 +322,9 @@ public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
}
/**
- * Redis attribute name for status (optional), by default "status"
+ * Redis field name for status (optional), by default "status"
*
- * @param statusAttr name of the status attribute in the hash
+ * @param statusAttr name of the status field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withStatusAttr(String statusAttr) {
@@ -335,9 +333,9 @@ public Builder withStatusAttr(String statusAttr) {
}
/**
- * Redis attribute name for response data (optional), by default "data"
+ * Redis field name for response data (optional), by default "data"
*
- * @param dataAttr name of the data attribute in the hash
+ * @param dataAttr name of the data field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withDataAttr(String dataAttr) {
@@ -346,9 +344,9 @@ public Builder withDataAttr(String dataAttr) {
}
/**
- * Redis attribute name for validation (optional), by default "validation"
+ * Redis field name for validation (optional), by default "validation"
*
- * @param validationAttr name of the validation attribute in the hash
+ * @param validationAttr name of the validation field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withValidationAttr(String validationAttr) {
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index b44db768f..d6a438f3a 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -58,7 +58,7 @@ public static void stop() {
}
@Test
- public void putRecord_shouldCreateItemInRedis() {
+ void putRecord_shouldCreateItemInRedis() {
Instant now = Instant.now();
long ttl = 3600;
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
@@ -74,7 +74,7 @@ public void putRecord_shouldCreateItemInRedis() {
}
@Test
- public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
+ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
long ttl = 3600;
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
@@ -82,32 +82,32 @@ public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
redisPersistenceStore.putRecord(
new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("in-progress-expiration")).isEqualTo(String.valueOf(progressExpiry.getAsLong()));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("in-progress-expiration", String.valueOf(progressExpiry.getAsLong()));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
+ void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
}
@Test
- public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
+ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -127,17 +127,17 @@ public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
null
), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
+ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -158,15 +158,15 @@ public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut()
null
), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
}
@Test
- public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
+ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -178,13 +178,16 @@ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordA
jedisPool.hset("idempotency:id:key", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
- assertThatThrownBy(() -> redisPersistenceStore.putRecord(
- new DataRecord("key",
- DataRecord.Status.INPROGRESS,
- expiry2,
- null,
- null
- ), now)
+ DataRecord dataRecord = new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ );
+ assertThatThrownBy(() -> {
+ redisPersistenceStore.putRecord(
+ dataRecord, now);
+ }
).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
Map entry = jedisPool.hgetAll("idempotency:id:key");
@@ -196,7 +199,7 @@ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordA
}
@Test
- public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
+ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -209,25 +212,26 @@ public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpire
jedisPool.hset("idempotency:id:key", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ DataRecord dataRecord = new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ "Fake Data 2",
+ null
+ );
assertThatThrownBy(() -> redisPersistenceStore.putRecord(
- new DataRecord("key",
- DataRecord.Status.INPROGRESS,
- expiry2,
- "Fake Data 2",
- null
- ), now))
+ dataRecord, now))
.isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("data")).isEqualTo("Fake Data");
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("data", "Fake Data");
}
@Test
- public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
+ void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -246,13 +250,13 @@ public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoun
}
@Test
- public void getRecord_shouldThrowException_whenRecordIsAbsent() {
+ void getRecord_shouldThrowException_whenRecordIsAbsent() {
assertThatThrownBy(() -> redisPersistenceStore.getRecord("key")).isInstanceOf(
IdempotencyItemNotFoundException.class);
}
@Test
- public void updateRecord_shouldUpdateRecord() {
+ void updateRecord_shouldUpdateRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
@@ -268,18 +272,18 @@ public void updateRecord_shouldUpdateRecord() {
DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
redisPersistenceStore.updateRecord(record);
- Map itemInDb = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(itemInDb.get("status")).isEqualTo("COMPLETED");
- assertThat(itemInDb.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(itemInDb.get("data")).isEqualTo("Fake result");
- assertThat(itemInDb.get("validation")).isEqualTo("hash");
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("data", "Fake result");
+ assertThat(redisItem).containsEntry("validation", "hash");
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void deleteRecord_shouldDeleteRecord() {
+ void deleteRecord_shouldDeleteRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
@@ -291,12 +295,12 @@ public void deleteRecord_shouldDeleteRecord() {
Map items = jedisPool.hgetAll("idempotency:id:key");
- assertThat(items.isEmpty()).isTrue();
+ assertThat(items).isEmpty();
}
@Test
- public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
+ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
try {
RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
.withKeyPrefixName("items-idempotency")
@@ -319,14 +323,14 @@ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFou
// PUT
persistenceStore.putRecord(record, now);
- Map itemInDb = jedisPool.hgetAll("items-idempotency:key:mykey");
+ Map redisItem = jedisPool.hgetAll("items-idempotency:key:mykey");
// GET
DataRecord recordInDb = persistenceStore.getRecord("mykey");
- assertThat(itemInDb).isNotNull();
- assertThat(itemInDb.get("state")).isEqualTo(recordInDb.getStatus().toString());
- assertThat(itemInDb.get("expiry")).isEqualTo(String.valueOf(recordInDb.getExpiryTimestamp()));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("state", recordInDb.getStatus().toString());
+ assertThat(redisItem).containsEntry("expiry", String.valueOf(recordInDb.getExpiryTimestamp()));
// UPDATE
DataRecord updatedRecord = new DataRecord(
@@ -342,26 +346,22 @@ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFou
// DELETE
persistenceStore.deleteRecord("mykey");
- assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isEqualTo(0);
+ assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isZero();
} finally {
- try {
- jedisPool.del("items-idempotency:key:mykey");
- } catch (Exception e) {
- // OK
- }
+ jedisPool.del("items-idempotency:key:mykey");
}
}
@Test
@SetEnvironmentVariable(key = software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV, value = "true")
- public void idempotencyDisabled_noClientShouldBeCreated() {
+ void idempotencyDisabled_noClientShouldBeCreated() {
RedisPersistenceStore store = RedisPersistenceStore.builder().build();
assertThatThrownBy(() -> store.getRecord("key")).isInstanceOf(NullPointerException.class);
}
@AfterEach
- public void emptyDB() {
+ void emptyDB() {
jedisPool.del("idempotency:id:key");
}
From 1c664bceaeaa5bcd1e496f329811ab7e8e7a0e47 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:52:27 +0200
Subject: [PATCH 04/19] E2E test for idempotency redis implementation
---
.../handlers/idempotency-dynamodb/pom.xml | 72 +++++++++++++++++
.../lambda/powertools/e2e/Function.java | 2 +-
.../amazon/lambda/powertools/e2e/Input.java | 0
.../src/main/resources/log4j2.xml | 0
.../pom.xml | 8 +-
.../lambda/powertools/e2e/Function.java | 56 +++++++++++++
.../amazon/lambda/powertools/e2e/Input.java | 34 ++++++++
.../src/main/resources/log4j2.xml | 16 ++++
.../handlers/largemessage_idempotent/pom.xml | 4 +-
powertools-e2e-tests/handlers/pom.xml | 12 ++-
...E2ET.java => IdempotencyDynamoDBE2ET.java} | 6 +-
.../powertools/IdempotencyRedisE2ET.java | 80 +++++++++++++++++++
.../powertools/testutils/Infrastructure.java | 45 ++++++++++-
powertools-idempotency/pom.xml | 2 +-
.../pom.xml | 2 +-
.../idempotency/redis/Constants.java | 0
.../redis/RedisPersistenceStore.java | 0
.../redis/RedisPersistenceStoreTest.java | 0
18 files changed, 322 insertions(+), 17 deletions(-)
create mode 100644 powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/java/software/amazon/lambda/powertools/e2e/Function.java (98%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/java/software/amazon/lambda/powertools/e2e/Input.java (100%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/resources/log4j2.xml (100%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-redis}/pom.xml (92%)
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
rename powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/{IdempotencyE2ET.java => IdempotencyDynamoDBE2ET.java} (96%)
create mode 100644 powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/pom.xml (98%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java (100%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java (100%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java (100%)
diff --git a/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml b/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
new file mode 100644
index 000000000..b9d9fdb03
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
@@ -0,0 +1,72 @@
+
+ 4.0.0
+
+
+ software.amazon.lambda
+ e2e-test-handlers-parent
+ 1.0.0
+
+
+ e2e-test-handler-idempotency-dynamodb
+ jar
+ A Lambda function using Powertools for AWS Lambda (Java) idempotency
+
+
+
+ software.amazon.lambda
+ powertools-idempotency-dynamodb
+
+
+ software.amazon.lambda
+ powertools-logging
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+
+
+ com.amazonaws
+ aws-lambda-java-events
+
+
+ org.aspectj
+ aspectjrt
+
+
+
+
+
+
+ dev.aspectj
+ aspectj-maven-plugin
+
+ ${maven.compiler.source}
+ ${maven.compiler.target}
+ ${maven.compiler.target}
+
+
+ software.amazon.lambda
+ powertools-idempotency-core
+
+
+ software.amazon.lambda
+ powertools-logging
+
+
+
+
+
+
+ compile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
similarity index 98%
rename from powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index e4c2f2b9a..16109778d 100644
--- a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -27,7 +27,7 @@
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
-import software.amazon.lambda.powertools.idempotency.persistence.DynamoDBPersistenceStore;
+import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
import software.amazon.lambda.powertools.logging.Logging;
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
similarity index 100%
rename from powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/resources/log4j2.xml
similarity index 100%
rename from powertools-e2e-tests/handlers/idempotency/src/main/resources/log4j2.xml
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/resources/log4j2.xml
diff --git a/powertools-e2e-tests/handlers/idempotency/pom.xml b/powertools-e2e-tests/handlers/idempotency-redis/pom.xml
similarity index 92%
rename from powertools-e2e-tests/handlers/idempotency/pom.xml
rename to powertools-e2e-tests/handlers/idempotency-redis/pom.xml
index da2bbfb80..9c0889028 100644
--- a/powertools-e2e-tests/handlers/idempotency/pom.xml
+++ b/powertools-e2e-tests/handlers/idempotency-redis/pom.xml
@@ -8,14 +8,13 @@
2.0.0-SNAPSHOT
- e2e-test-handler-idempotency
+ e2e-test-handler-idempotency-redis
jar
A Lambda function using Powertools for AWS Lambda (Java) idempotency
-
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-redis
software.amazon.lambda
@@ -30,7 +29,6 @@
aspectjrt
-
@@ -43,7 +41,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-core
software.amazon.lambda
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
new file mode 100644
index 000000000..5ca0f316e
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.TimeZone;
+import redis.clients.jedis.JedisPooled;
+import software.amazon.lambda.powertools.idempotency.Idempotency;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.Idempotent;
+import software.amazon.lambda.powertools.idempotency.redis.RedisPersistenceStore;
+import software.amazon.lambda.powertools.logging.Logging;
+
+
+public class Function implements RequestHandler {
+
+ public Function() {
+ this(new JedisPooled(System.getenv().get("REDIS_HOST"), Integer.parseInt(System.getenv().get("REDIS_PORT")), System.getenv().get("REDIS_USER"), System.getenv().get("REDIS_SECRET")));
+ }
+
+ public Function(JedisPooled client) {
+ Idempotency.config().withConfig(
+ IdempotencyConfig.builder()
+ .withExpiration(Duration.of(10, ChronoUnit.SECONDS))
+ .build())
+ .withPersistenceStore(
+ RedisPersistenceStore.builder()
+ .withJedisPooled(client)
+ .build()
+ ).configure();
+ }
+
+ @Logging(logEvent = true)
+ @Idempotent
+ public String handleRequest(Input input, Context context) {
+ DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME.withZone(TimeZone.getTimeZone("UTC").toZoneId());
+ return dtf.format(Instant.now());
+ }
+}
\ No newline at end of file
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
new file mode 100644
index 000000000..e0e4c27c9
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+public class Input {
+ private String message;
+
+ public Input(String message) {
+ this.message = message;
+ }
+
+ public Input() {
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+}
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
new file mode 100644
index 000000000..8925f70b9
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml b/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
index 8cb2cb52c..f1f7ce597 100644
--- a/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
+++ b/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
@@ -15,7 +15,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
software.amazon.lambda
@@ -47,7 +47,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
software.amazon.lambda
diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml
index 412593da9..0a9ac4e6c 100644
--- a/powertools-e2e-tests/handlers/pom.xml
+++ b/powertools-e2e-tests/handlers/pom.xml
@@ -33,9 +33,10 @@
tracing
metrics
batch
- idempotency
largemessage
largemessage_idempotent
+ idempotency-dynamodb
+ idempotency-redis
parameters
validation-alb-event
validation-apigw-event
@@ -72,7 +73,12 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
+ ${lambda.powertools.version}
+
+
+ software.amazon.lambda
+ powertools-idempotency-redis
${lambda.powertools.version}
@@ -160,7 +166,6 @@
aspectj-maven-plugin
${aspectj.plugin.version}
- true
${maven.compiler.source}
${maven.compiler.target}
${maven.compiler.target}
@@ -169,6 +174,7 @@
+ process-sources
compile
test-compile
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
similarity index 96%
rename from powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java
rename to powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
index 242d1a2db..1c9ac30b6 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
@@ -29,7 +29,7 @@
import software.amazon.lambda.powertools.testutils.Infrastructure;
import software.amazon.lambda.powertools.testutils.lambda.InvocationResult;
-public class IdempotencyE2ET {
+public class IdempotencyDynamoDBE2ET {
private static Infrastructure infrastructure;
private static String functionName;
@@ -38,7 +38,7 @@ public class IdempotencyE2ET {
public static void setup() {
String random = UUID.randomUUID().toString().substring(0, 6);
infrastructure = Infrastructure.builder()
- .testName(IdempotencyE2ET.class.getSimpleName())
+ .testName(IdempotencyDynamoDBE2ET.class.getSimpleName())
.pathToFunction("idempotency")
.idempotencyTable("idempo" + random)
.build();
@@ -75,4 +75,4 @@ public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws In
Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult());
Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult());
}
-}
+}
\ No newline at end of file
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
new file mode 100644
index 000000000..e01189207
--- /dev/null
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools;
+
+import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT;
+import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction;
+
+import java.time.Year;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import software.amazon.lambda.powertools.testutils.Infrastructure;
+import software.amazon.lambda.powertools.testutils.lambda.InvocationResult;
+
+public class IdempotencyRedisE2ET {
+ private static Infrastructure infrastructure;
+ private static String functionName;
+
+ @BeforeAll
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ public static void setup() {
+ infrastructure = Infrastructure.builder()
+ .testName(IdempotencyRedisE2ET.class.getSimpleName())
+ .redisHost(System.getenv("REDIS_HOST"))
+ .redisPort(System.getenv("REDIS_PORT"))
+ .redisUser(System.getenv("REDIS_USER"))
+ .redisSecret(System.getenv("REDIS_SECRET"))
+ .pathToFunction("idempotency-redis")
+ .build();
+ Map outputs = infrastructure.deploy();
+ functionName = outputs.get(FUNCTION_NAME_OUTPUT);
+ }
+
+ @AfterAll
+ public static void tearDown() {
+ if (infrastructure != null) {
+ infrastructure.destroy();
+ }
+ }
+
+ @Test
+ public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws InterruptedException {
+ // GIVEN
+ String event = "{\"message\":\"TTL 10sec\"}";
+
+ // WHEN
+ // First invocation
+ InvocationResult result1 = invokeFunction(functionName, event);
+
+ // Second invocation (should get same result)
+ InvocationResult result2 = invokeFunction(functionName, event);
+
+ Thread.sleep(12000);
+
+ // Third invocation (should get different result)
+ InvocationResult result3 = invokeFunction(functionName, event);
+
+ // THEN
+ Assertions.assertThat(result1.getResult()).contains(Year.now().toString());
+ Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult());
+ Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult());
+ }
+}
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index 28a0f2bb4..350dd88cf 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -45,7 +45,11 @@
import software.amazon.awscdk.services.appconfig.CfnDeploymentStrategy;
import software.amazon.awscdk.services.appconfig.CfnEnvironment;
import software.amazon.awscdk.services.appconfig.CfnHostedConfigurationVersion;
-import software.amazon.awscdk.services.dynamodb.*;
+import software.amazon.awscdk.services.dynamodb.Attribute;
+import software.amazon.awscdk.services.dynamodb.AttributeType;
+import software.amazon.awscdk.services.dynamodb.BillingMode;
+import software.amazon.awscdk.services.dynamodb.StreamViewType;
+import software.amazon.awscdk.services.dynamodb.Table;
import software.amazon.awscdk.services.iam.PolicyStatement;
import software.amazon.awscdk.services.kinesis.Stream;
import software.amazon.awscdk.services.kinesis.StreamMode;
@@ -114,6 +118,10 @@ public class Infrastructure {
private final String queue;
private final String kinesisStream;
private final String largeMessagesBucket;
+ private final String redisHost;
+ private final String redisPort;
+ private final String redisUser;
+ private final String redisSecret;
private String ddbStreamsTableName;
private String functionName;
private Object cfnTemplate;
@@ -127,6 +135,10 @@ private Infrastructure(Builder builder) {
this.timeout = builder.timeoutInSeconds;
this.pathToFunction = builder.pathToFunction;
this.idempotencyTable = builder.idemPotencyTable;
+ this.redisHost = builder.redisHost;
+ this.redisPort = builder.redisPort;
+ this.redisUser = builder.redisUser;
+ this.redisSecret = builder.redisSecret;
this.appConfig = builder.appConfig;
this.queue = builder.queue;
this.kinesisStream = builder.kinesisStream;
@@ -273,6 +285,13 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
}
+ if (!StringUtils.isEmpty(redisHost)) {
+ function.addEnvironment("REDIS_HOST", redisHost);
+ function.addEnvironment("REDIS_PORT", redisPort);
+ function.addEnvironment("REDIS_USER", redisUser);
+ function.addEnvironment("REDIS_SECRET", redisSecret);
+ }
+
if (!StringUtils.isEmpty(queue)) {
Queue sqsQueue = Queue.Builder
.create(stack, "SQSQueue")
@@ -504,6 +523,10 @@ public static class Builder {
private String queue;
private String kinesisStream;
private String ddbStreamsTableName;
+ private String redisHost;
+ private String redisPort;
+ private String redisUser;
+ private String redisSecret;
private Builder() {
runtime = mapRuntimeVersion("JAVA_VERSION");
@@ -562,6 +585,26 @@ public Builder idempotencyTable(String tableName) {
return this;
}
+ public Builder redisHost(String redisHost) {
+ this.redisHost = redisHost;
+ return this;
+ }
+
+ public Builder redisPort(String redisPort) {
+ this.redisPort = redisPort;
+ return this;
+ }
+
+ public Builder redisUser(String redisUser) {
+ this.redisUser = redisUser;
+ return this;
+ }
+
+ public Builder redisSecret(String redisSecret) {
+ this.redisSecret = redisSecret;
+ return this;
+ }
+
public Builder appConfig(AppConfig app) {
this.appConfig = app;
return this;
diff --git a/powertools-idempotency/pom.xml b/powertools-idempotency/pom.xml
index b3b0268ce..64b182f86 100644
--- a/powertools-idempotency/pom.xml
+++ b/powertools-idempotency/pom.xml
@@ -36,7 +36,7 @@
powertools-idempotency-core
powertools-idempotency-dynamodb
- powertool-idempotency-redis
+ powertools-idempotency-redis
diff --git a/powertools-idempotency/powertool-idempotency-redis/pom.xml b/powertools-idempotency/powertools-idempotency-redis/pom.xml
similarity index 98%
rename from powertools-idempotency/powertool-idempotency-redis/pom.xml
rename to powertools-idempotency/powertools-idempotency-redis/pom.xml
index 2b9487aa6..1ead9407f 100644
--- a/powertools-idempotency/powertool-idempotency-redis/pom.xml
+++ b/powertools-idempotency/powertools-idempotency-redis/pom.xml
@@ -23,7 +23,7 @@
2.0.0-SNAPSHOT
- powertool-idempotency-redis
+ powertools-idempotency-redis
Powertools for AWS Lambda (Java) library Idempotency - Redis
Redis implementation for the idempotency module
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
rename to powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
From 8a12b0333a685c6d1da49da1d8a8da99b114c41b Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 10:12:52 +0200
Subject: [PATCH 05/19] Adding instructions to bootstrap cdk in e2e README file
---
powertools-e2e-tests/README.md | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/powertools-e2e-tests/README.md b/powertools-e2e-tests/README.md
index 61799e6f7..f41e16cd8 100644
--- a/powertools-e2e-tests/README.md
+++ b/powertools-e2e-tests/README.md
@@ -6,8 +6,16 @@ __Prerequisites__:
([credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html)).
- [Java 11+](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html)
- [Docker](https://docs.docker.com/engine/install/)
+- [CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install)
-To execute the E2E tests, use the following command: `export JAVA_VERSION=11 && mvn clean verify -Pe2e`
+### Execute test
+Before executing the tests in a new AWS account, [bootstrap CDK](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.htmls) using the following command:
+
+`cdk bootstrap aws:///`
+
+To execute the E2E tests, use the following command:
+
+`export JAVA_VERSION=11 && mvn clean verify -Pe2e`
### Under the hood
This module leverages the following components:
From 27ec2a9e96d17d52f6b53ebf3d0d95df5b2c7014 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 17:12:44 +0200
Subject: [PATCH 06/19] Add documentation for redis idempotency
---
docs/utilities/idempotency.md | 90 ++++++++++++++++++++++++++++++++++-
1 file changed, 88 insertions(+), 2 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index f4defbdfd..3bf1a938e 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -159,10 +159,12 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
```
### Required resources
-
Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your Lambda functions will need read and write access to it.
+As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https://redis.io/) are the supported persistnce layers.
+
+#### Using Amazon DynamoDB as persistent storage layer
-As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first.
+If you are using Amazon DynamoDB you'll need to create a table.
**Default table configuration**
@@ -215,6 +217,34 @@ Resources:
see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to
estimate the cost.
+#### Using Redis as persistent storage layer
+
+If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
+In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
+If you don't provide a custom [Redis client](# Customizing Redis client) you can omit the environment variables declaration.
+
+!!! warning "Warning: Avoid including a plain text secret in your template"
+This can infer security implications
+
+!!! warning "Warning: Large responses with Redis persistence layer"
+When using this utility with Redis your function's responses must be [smaller than 512MB].
+Persisting larger items cannot might cause exceptions.
+
+```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+Resources:
+ IdempotencyFunction:
+ Type: AWS::Serverless::Function
+ Properties:
+ CodeUri: Function
+ Handler: helloworld.App::handleRequest
+ Environment:
+ Variables:
+ REDIS_HOST: %redis-host%
+ REDIS_PORT: %redis-port%
+ REDIS_USER: %redis-user%
+ REDIS_SECRET: %redis-secret%
+```
+
### Idempotent annotation
You can quickly start by initializing the `DynamoDBPersistenceStore` and using it with the `@Idempotent` annotation on your Lambda handler.
@@ -635,6 +665,29 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by
| **SortKeyAttr** | | | Sort key of the table (if table is configured with a sort key). |
| **StaticPkValue** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **SortKeyAttr** is set. |
+#### RedisPersistenceStore
+
+The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in Standalone mode.
+
+We are using [Redis hashes](https://redis.io/docs/data-types/hashes/) to store the idempotency fields and values.
+There are some predefined fields that you can see listed in the following table. The predefined fields have some default values.
+
+
+You can alter the field names by passing these parameters when initializing the persistence layer:
+
+| Parameter | Required | Default | Description |
+|--------------------|----------|--------------------------------------|--------------------------------------------------------------------------------------------------------|
+| **KeyPrefixName** | Y | `idempotency` | The redis hash key prefix |
+| **KeyAttr** | Y | `id` | The redis hash key field name |
+| **ExpiryAttr** | | `expiration` | Unix timestamp of when record expires |
+| **StatusAttr** | | `status` | Stores status of the Lambda execution during and after invocation |
+| **DataAttr** | | `data` | Stores results of successfully idempotent methods |
+| **ValidationAttr** | | `validation` | Hashed representation of the parts of the event used for validation |
+
+
+!!! Tip "Tip: You can share the same prefix and key for all functions"
+You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
+
## Advanced
### Customizing the default behavior
@@ -884,6 +937,39 @@ When creating the `DynamoDBPersistenceStore`, you can set a custom [`DynamoDbCli
.build();
```
+### Customizing Redis client
+
+The `RedisPersistenceStore` uses the JedisPooled java client to connect to the Redis Server.
+When creating the `RedisPersistenceStore`, you can set a custom [`JedisPooled`](https://www.javadoc.io/doc/redis.clients/jedis/latest/redis/clients/jedis/JedisPooled.html) client:
+
+=== "Custom JedisPooled with connection timeout"
+
+ ```java hl_lines="2-6 11"
+ public App() {
+ JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), DefaultJedisClientConfig.builder()
+ .user("user")
+ .password("secret")
+ .connectionTimeoutMillis(3000)
+ .build())
+
+ Idempotency.config().withPersistenceStore(
+ RedisPersistenceStore.builder()
+ .withKeyPrefixName("items-idempotency")
+ .withJedisPooled(jedisPooled)
+ .build()
+ ).configure();
+ }
+ ```
+
+!!! info "Default configuration is the following:"
+
+ ```java
+ DefaultJedisClientConfig.builder()
+ .user(System.getenv(Constants.REDIS_USER))
+ .password(System.getenv(Constants.REDIS_SECRET))
+ .build();
+ ```
+
### Using a DynamoDB table with a composite primary key
When using a composite primary key table (hash+range key), use `SortKeyAttr` parameter when initializing your persistence store.
From dcfb45d702537cff14724287bb3db6aa54d41951 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 8 Dec 2023 10:33:21 +0200
Subject: [PATCH 07/19] Add support for Redis Cluster, improve documentation
and e2e tests
---
docs/utilities/idempotency.md | 46 ++++-
.../lambda/powertools/e2e/Function.java | 7 -
.../powertools/IdempotencyRedisE2ET.java | 8 +-
.../powertools/testutils/Infrastructure.java | 178 +++++++++++------
.../powertools-idempotency-redis/pom.xml | 7 +-
.../idempotency/redis/Constants.java | 1 +
.../redis/RedisPersistenceStore.java | 146 ++++++++------
.../redis/RedisPersistenceStoreTest.java | 185 +++++++++++-------
8 files changed, 371 insertions(+), 207 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 3bf1a938e..8d6077d68 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -219,18 +219,26 @@ Resources:
#### Using Redis as persistent storage layer
+##### Redis resources
+
+You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) as persistent storage layer provider.
+!!! tip "Tip:No existing Redis service?"
+If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+
If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
+If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
+environment variable `REDIS_CLUSTER_MODE` to `true`
In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
-If you don't provide a custom [Redis client](# Customizing Redis client) you can omit the environment variables declaration.
+If you provide a [custom Redis client](#Customizing Redis client) you can omit the environment variables declaration.
-!!! warning "Warning: Avoid including a plain text secret in your template"
+!!! warning "Warning: Avoid including a plain text secret directly in your template"
This can infer security implications
!!! warning "Warning: Large responses with Redis persistence layer"
-When using this utility with Redis your function's responses must be [smaller than 512MB].
+When using this utility with Redis your function's responses must be smaller than 512MB.
Persisting larger items cannot might cause exceptions.
-```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+```yaml hl_lines="9-13" title="AWS Serverless Application Model (SAM) example"
Resources:
IdempotencyFunction:
Type: AWS::Serverless::Function
@@ -243,7 +251,35 @@ Resources:
REDIS_PORT: %redis-port%
REDIS_USER: %redis-user%
REDIS_SECRET: %redis-secret%
+ REDIS_CLUSTER_MODE: "true"
+```
+##### VPC Access
+Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
+
+!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
+If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+
+!!! warning "Amazon ElastiCache Serverless not supported"
+[Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
+
+!!! warning "Check network connectivity to Redis server"
+Make sure that your AWS Lambda function can connect to your Redis server.
+
+```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+Resources:
+ IdempotencyFunction:
+ Type: AWS::Serverless::Function
+ Properties:
+ CodeUri: Function
+ Handler: helloworld.App::handleRequest
+ VpcConfig: # (1)!
+ SecurityGroupIds:
+ - sg-{your_sg_id}
+ SubnetIds:
+ - subnet-{your_subnet_id_1}
+ - subnet-{your_subnet_id_2}
```
+1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
### Idempotent annotation
@@ -667,7 +703,7 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by
#### RedisPersistenceStore
-The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in Standalone mode.
+The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in either Standalone or Cluster mode.
We are using [Redis hashes](https://redis.io/docs/data-types/hashes/) to store the idempotency fields and values.
There are some predefined fields that you can see listed in the following table. The predefined fields have some default values.
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index 5ca0f316e..994f14d0c 100644
--- a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -21,7 +21,6 @@
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.TimeZone;
-import redis.clients.jedis.JedisPooled;
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
@@ -30,19 +29,13 @@
public class Function implements RequestHandler {
-
public Function() {
- this(new JedisPooled(System.getenv().get("REDIS_HOST"), Integer.parseInt(System.getenv().get("REDIS_PORT")), System.getenv().get("REDIS_USER"), System.getenv().get("REDIS_SECRET")));
- }
-
- public Function(JedisPooled client) {
Idempotency.config().withConfig(
IdempotencyConfig.builder()
.withExpiration(Duration.of(10, ChronoUnit.SECONDS))
.build())
.withPersistenceStore(
RedisPersistenceStore.builder()
- .withJedisPooled(client)
.build()
).configure();
}
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
index e01189207..412389741 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
@@ -19,7 +19,6 @@
import java.time.Year;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterAll;
@@ -34,14 +33,11 @@ public class IdempotencyRedisE2ET {
private static String functionName;
@BeforeAll
- @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ @Timeout(value = 15, unit = TimeUnit.MINUTES)
public static void setup() {
infrastructure = Infrastructure.builder()
.testName(IdempotencyRedisE2ET.class.getSimpleName())
- .redisHost(System.getenv("REDIS_HOST"))
- .redisPort(System.getenv("REDIS_PORT"))
- .redisUser(System.getenv("REDIS_USER"))
- .redisSecret(System.getenv("REDIS_SECRET"))
+ .redisDeployment(true)
.pathToFunction("idempotency-redis")
.build();
Map outputs = infrastructure.deploy();
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index 350dd88cf..e61fbe873 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -27,6 +27,7 @@
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Collectors;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -36,8 +37,10 @@
import software.amazon.awscdk.CfnOutput;
import software.amazon.awscdk.DockerVolume;
import software.amazon.awscdk.Duration;
+import software.amazon.awscdk.Environment;
import software.amazon.awscdk.RemovalPolicy;
import software.amazon.awscdk.Stack;
+import software.amazon.awscdk.StackProps;
import software.amazon.awscdk.cxapi.CloudAssembly;
import software.amazon.awscdk.services.appconfig.CfnApplication;
import software.amazon.awscdk.services.appconfig.CfnConfigurationProfile;
@@ -50,6 +53,14 @@
import software.amazon.awscdk.services.dynamodb.BillingMode;
import software.amazon.awscdk.services.dynamodb.StreamViewType;
import software.amazon.awscdk.services.dynamodb.Table;
+import software.amazon.awscdk.services.ec2.IVpc;
+import software.amazon.awscdk.services.ec2.Peer;
+import software.amazon.awscdk.services.ec2.Port;
+import software.amazon.awscdk.services.ec2.SecurityGroup;
+import software.amazon.awscdk.services.ec2.SubnetSelection;
+import software.amazon.awscdk.services.ec2.Vpc;
+import software.amazon.awscdk.services.elasticache.CfnCacheCluster;
+import software.amazon.awscdk.services.elasticache.CfnSubnetGroup;
import software.amazon.awscdk.services.iam.PolicyStatement;
import software.amazon.awscdk.services.kinesis.Stream;
import software.amazon.awscdk.services.kinesis.StreamMode;
@@ -118,14 +129,15 @@ public class Infrastructure {
private final String queue;
private final String kinesisStream;
private final String largeMessagesBucket;
- private final String redisHost;
- private final String redisPort;
- private final String redisUser;
- private final String redisSecret;
+ private IVpc vpc;
private String ddbStreamsTableName;
private String functionName;
private Object cfnTemplate;
private String cfnAssetDirectory;
+ private SubnetSelection subnetSelection;
+ private CfnSubnetGroup cfnSubnetGroup;
+ private SecurityGroup securityGroup;
+ private boolean isRedisDeployment = false;
private Infrastructure(Builder builder) {
this.stackName = builder.stackName;
@@ -135,28 +147,52 @@ private Infrastructure(Builder builder) {
this.timeout = builder.timeoutInSeconds;
this.pathToFunction = builder.pathToFunction;
this.idempotencyTable = builder.idemPotencyTable;
- this.redisHost = builder.redisHost;
- this.redisPort = builder.redisPort;
- this.redisUser = builder.redisUser;
- this.redisSecret = builder.redisSecret;
+ this.isRedisDeployment = builder.redisDeployment;
this.appConfig = builder.appConfig;
this.queue = builder.queue;
this.kinesisStream = builder.kinesisStream;
this.largeMessagesBucket = builder.largeMessagesBucket;
this.ddbStreamsTableName = builder.ddbStreamsTableName;
+ this.region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1"));
this.app = new App();
- this.stack = createStackWithLambda();
- this.synthesize();
+ this.stack = createStack();
this.httpClient = UrlConnectionHttpClient.builder().build();
- this.region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1"));
+
this.account = StsClient.builder()
.httpClient(httpClient)
.region(region)
.build().getCallerIdentity().account();
+ if (isRedisDeployment) {
+ this.vpc = Vpc.Builder.create(this.stack, "MyVPC-" + stackName)
+ .availabilityZones(List.of(region.toString() + "a", region.toString() + "b"))
+ .build();
+
+ List subnets = vpc.getPublicSubnets().stream().map(subnet ->
+ subnet.getSubnetId()).collect(Collectors.toList());
+
+ securityGroup = SecurityGroup.Builder.create(stack, "ElastiCache-SG-" + stackName)
+ .vpc(vpc)
+ .allowAllOutbound(true)
+ .description("ElastiCache SecurityGroup")
+ .build();
+
+ cfnSubnetGroup = CfnSubnetGroup.Builder.create(stack, "Redis Subnet-" + stackName)
+ .description("A subnet for the ElastiCache cluster")
+ .subnetIds(subnets).cacheSubnetGroupName("redis-SG-" + stackName).build();
+
+ subnetSelection = SubnetSelection.builder().subnets(vpc.getPublicSubnets()).build();
+ }
+
+
+ createStackWithLambda();
+
+ this.synthesize();
+
+
s3 = S3Client.builder()
.httpClient(httpClient)
.region(region)
@@ -213,10 +249,9 @@ public void destroy() {
*
* @return the CDK stack
*/
- private Stack createStackWithLambda() {
+ private void createStackWithLambda() {
boolean createTableForAsyncTests = false;
Stack stack = new Stack(app, stackName);
-
List packagingInstruction = Arrays.asList(
"/bin/sh",
"-c",
@@ -242,14 +277,23 @@ private Stack createStackWithLambda() {
.outputType(BundlingOutput.ARCHIVED);
functionName = stackName + "-function";
- CfnOutput.Builder.create(stack, FUNCTION_NAME_OUTPUT)
+ CfnOutput.Builder.create(this.stack, FUNCTION_NAME_OUTPUT)
.value(functionName)
.build();
LOG.debug("Building Lambda function with command " +
packagingInstruction.stream().collect(Collectors.joining(" ", "[", "]")));
- Function function = Function.Builder
- .create(stack, functionName)
+
+ final SecurityGroup lambdaSecurityGroup = SecurityGroup.Builder.create(this.stack, "Lambda-SG")
+ .vpc(vpc)
+ .allowAllOutbound(true)
+ .description("Lambda SecurityGroup")
+ .build();
+ securityGroup.addIngressRule(Peer.securityGroupId(lambdaSecurityGroup.getSecurityGroupId()), Port.tcp(6379),
+ "Allow ElastiCache Server");
+
+ Function.Builder functionBuilder = Function.Builder
+ .create(this.stack, functionName)
.code(Code.fromAsset("handlers/", AssetOptions.builder()
.bundling(builderOptions
.command(packagingInstruction)
@@ -259,13 +303,22 @@ private Stack createStackWithLambda() {
.handler("software.amazon.lambda.powertools.e2e.Function::handleRequest")
.memorySize(1024)
.timeout(Duration.seconds(timeout))
+ .allowPublicSubnet(true)
.runtime(runtime.getCdkRuntime())
.environment(envVar)
- .tracing(tracing ? Tracing.ACTIVE : Tracing.DISABLED)
- .build();
+ .tracing(tracing ? Tracing.ACTIVE : Tracing.DISABLED);
+
+ if (isRedisDeployment) {
+ functionBuilder.vpc(vpc)
+ .vpcSubnets(subnetSelection)
+ .securityGroups(List.of(lambdaSecurityGroup));
+ }
+
+ Function function = functionBuilder.build();
+
LogGroup.Builder
- .create(stack, functionName + "-logs")
+ .create(this.stack, functionName + "-logs")
.logGroupName("/aws/lambda/" + functionName)
.retention(RetentionDays.ONE_DAY)
.removalPolicy(RemovalPolicy.DESTROY)
@@ -273,7 +326,7 @@ private Stack createStackWithLambda() {
if (!StringUtils.isEmpty(idempotencyTable)) {
Table table = Table.Builder
- .create(stack, "IdempotencyTable")
+ .create(this.stack, "IdempotencyTable")
.billingMode(BillingMode.PAY_PER_REQUEST)
.removalPolicy(RemovalPolicy.DESTROY)
.partitionKey(Attribute.builder().name("id").type(AttributeType.STRING).build())
@@ -285,16 +338,24 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
}
- if (!StringUtils.isEmpty(redisHost)) {
- function.addEnvironment("REDIS_HOST", redisHost);
- function.addEnvironment("REDIS_PORT", redisPort);
- function.addEnvironment("REDIS_USER", redisUser);
- function.addEnvironment("REDIS_SECRET", redisSecret);
+ if (isRedisDeployment) {
+ CfnCacheCluster redisServer = CfnCacheCluster.Builder.create(this.stack, "ElastiCacheCluster-" + stackName)
+ .clusterName("redis-cluster-" + stackName)
+ .engine("redis")
+ .cacheNodeType("cache.t2.micro")
+ .cacheSubnetGroupName(cfnSubnetGroup.getCacheSubnetGroupName())
+ .vpcSecurityGroupIds(List.of(securityGroup.getSecurityGroupId()))
+ .numCacheNodes(1)
+ .build();
+ redisServer.addDependency(cfnSubnetGroup);
+ function.addEnvironment("REDIS_HOST", redisServer.getAttrRedisEndpointAddress());
+ function.addEnvironment("REDIS_PORT", redisServer.getAttrRedisEndpointPort());
+ function.addEnvironment("REDIS_USER", "default");
}
if (!StringUtils.isEmpty(queue)) {
Queue sqsQueue = Queue.Builder
- .create(stack, "SQSQueue")
+ .create(this.stack, "SQSQueue")
.queueName(queue)
.visibilityTimeout(Duration.seconds(timeout * 6))
.retentionPeriod(Duration.seconds(timeout * 6))
@@ -312,14 +373,14 @@ private Stack createStackWithLambda() {
.build();
function.addEventSource(sqsEventSource);
CfnOutput.Builder
- .create(stack, "QueueURL")
+ .create(this.stack, "QueueURL")
.value(sqsQueue.getQueueUrl())
.build();
createTableForAsyncTests = true;
}
if (!StringUtils.isEmpty(kinesisStream)) {
Stream stream = Stream.Builder
- .create(stack, "KinesisStream")
+ .create(this.stack, "KinesisStream")
.streamMode(StreamMode.ON_DEMAND)
.streamName(kinesisStream)
.build();
@@ -335,13 +396,13 @@ private Stack createStackWithLambda() {
.build();
function.addEventSource(kinesisEventSource);
CfnOutput.Builder
- .create(stack, "KinesisStreamName")
+ .create(this.stack, "KinesisStreamName")
.value(stream.getStreamName())
.build();
}
if (!StringUtils.isEmpty(ddbStreamsTableName)) {
- Table ddbStreamsTable = Table.Builder.create(stack, "DDBStreamsTable")
+ Table ddbStreamsTable = Table.Builder.create(this.stack, "DDBStreamsTable")
.tableName(ddbStreamsTableName)
.stream(StreamViewType.KEYS_ONLY)
.removalPolicy(RemovalPolicy.DESTROY)
@@ -355,12 +416,12 @@ private Stack createStackWithLambda() {
.reportBatchItemFailures(true)
.build();
function.addEventSource(ddbEventSource);
- CfnOutput.Builder.create(stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build();
+ CfnOutput.Builder.create(this.stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build();
}
if (!StringUtils.isEmpty(largeMessagesBucket)) {
Bucket offloadBucket = Bucket.Builder
- .create(stack, "LargeMessagesOffloadBucket")
+ .create(this.stack, "LargeMessagesOffloadBucket")
.removalPolicy(RemovalPolicy.RETAIN) // autodelete does not work without cdk deploy
.bucketName(largeMessagesBucket)
.build();
@@ -371,19 +432,19 @@ private Stack createStackWithLambda() {
if (appConfig != null) {
CfnApplication app = CfnApplication.Builder
- .create(stack, "AppConfigApp")
+ .create(this.stack, "AppConfigApp")
.name(appConfig.getApplication())
.build();
CfnEnvironment environment = CfnEnvironment.Builder
- .create(stack, "AppConfigEnvironment")
+ .create(this.stack, "AppConfigEnvironment")
.applicationId(app.getRef())
.name(appConfig.getEnvironment())
.build();
// Create a fast deployment strategy, so we don't have to wait ages
CfnDeploymentStrategy fastDeployment = CfnDeploymentStrategy.Builder
- .create(stack, "AppConfigDeployment")
+ .create(this.stack, "AppConfigDeployment")
.name("FastDeploymentStrategy")
.deploymentDurationInMinutes(0)
.finalBakeTimeInMinutes(0)
@@ -402,14 +463,14 @@ private Stack createStackWithLambda() {
CfnDeployment previousDeployment = null;
for (Map.Entry entry : appConfig.getConfigurationValues().entrySet()) {
CfnConfigurationProfile configProfile = CfnConfigurationProfile.Builder
- .create(stack, "AppConfigProfileFor" + entry.getKey())
+ .create(this.stack, "AppConfigProfileFor" + entry.getKey())
.applicationId(app.getRef())
.locationUri("hosted")
.name(entry.getKey())
.build();
CfnHostedConfigurationVersion configVersion = CfnHostedConfigurationVersion.Builder
- .create(stack, "AppConfigHostedVersionFor" + entry.getKey())
+ .create(this.stack, "AppConfigHostedVersionFor" + entry.getKey())
.applicationId(app.getRef())
.contentType("text/plain")
.configurationProfileId(configProfile.getRef())
@@ -417,7 +478,7 @@ private Stack createStackWithLambda() {
.build();
CfnDeployment deployment = CfnDeployment.Builder
- .create(stack, "AppConfigDepoymentFor" + entry.getKey())
+ .create(this.stack, "AppConfigDepoymentFor" + entry.getKey())
.applicationId(app.getRef())
.environmentId(environment.getRef())
.deploymentStrategyId(fastDeployment.getRef())
@@ -434,7 +495,7 @@ private Stack createStackWithLambda() {
}
if (createTableForAsyncTests) {
Table table = Table.Builder
- .create(stack, "TableForAsyncTests")
+ .create(this.stack, "TableForAsyncTests")
.billingMode(BillingMode.PAY_PER_REQUEST)
.removalPolicy(RemovalPolicy.DESTROY)
.partitionKey(Attribute.builder().name("functionName").type(AttributeType.STRING).build())
@@ -443,9 +504,17 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
function.addEnvironment("TABLE_FOR_ASYNC_TESTS", table.getTableName());
- CfnOutput.Builder.create(stack, "TableNameForAsyncTests").value(table.getTableName()).build();
+ CfnOutput.Builder.create(this.stack, "TableNameForAsyncTests").value(table.getTableName()).build();
}
+ }
+ @NotNull
+ private Stack createStack() {
+ Stack stack = new Stack(app, stackName, StackProps.builder()
+ .env(Environment.builder()
+ .account(account)
+ .region(region.id())
+ .build()).build());
return stack;
}
@@ -456,6 +525,7 @@ private void synthesize() {
CloudAssembly synth = app.synth();
cfnTemplate = synth.getStackByName(stack.getStackName()).getTemplate();
cfnAssetDirectory = synth.getDirectory();
+
}
/**
@@ -496,9 +566,9 @@ private Map findAssets() {
String assetPath = file.get("source").get("path").asText();
String assetPackaging = file.get("source").get("packaging").asText();
String bucketName =
- file.get("destinations").get("current_account-current_region").get("bucketName").asText();
+ file.get("destinations").get("current_account-" + region.id()).get("bucketName").asText();
String objectKey =
- file.get("destinations").get("current_account-current_region").get("objectKey").asText();
+ file.get("destinations").get("current_account-" + region.id()).get("objectKey").asText();
Asset asset = new Asset(assetPath, assetPackaging, bucketName.replace("${AWS::AccountId}", account)
.replace("${AWS::Region}", region.toString()));
assets.put(objectKey, asset);
@@ -523,10 +593,7 @@ public static class Builder {
private String queue;
private String kinesisStream;
private String ddbStreamsTableName;
- private String redisHost;
- private String redisPort;
- private String redisUser;
- private String redisSecret;
+ private boolean redisDeployment = false;
private Builder() {
runtime = mapRuntimeVersion("JAVA_VERSION");
@@ -585,23 +652,8 @@ public Builder idempotencyTable(String tableName) {
return this;
}
- public Builder redisHost(String redisHost) {
- this.redisHost = redisHost;
- return this;
- }
-
- public Builder redisPort(String redisPort) {
- this.redisPort = redisPort;
- return this;
- }
-
- public Builder redisUser(String redisUser) {
- this.redisUser = redisUser;
- return this;
- }
-
- public Builder redisSecret(String redisSecret) {
- this.redisSecret = redisSecret;
+ public Builder redisDeployment(boolean isRedisDeployment) {
+ this.redisDeployment = isRedisDeployment;
return this;
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/pom.xml b/powertools-idempotency/powertools-idempotency-redis/pom.xml
index 1ead9407f..69b89dd3b 100644
--- a/powertools-idempotency/powertools-idempotency-redis/pom.xml
+++ b/powertools-idempotency/powertools-idempotency-redis/pom.xml
@@ -38,7 +38,7 @@
redis.clients
jedis
- 4.3.1
+ 5.1.0
org.signal
@@ -46,6 +46,11 @@
0.8.3
test
+
+ com.github.fppt
+ jedis-mock
+ 1.0.11
+
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
index ea8bd8695..55b37f999 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
@@ -19,4 +19,5 @@ public class Constants {
public static final String REDIS_PORT = "REDIS_PORT";
public static final String REDIS_USER = "REDIS_USER";
public static final String REDIS_SECRET = "REDIS_SECRET";
+ public static final String REDIS_CLUSTER_MODE = "REDIS_CLUSTER_MODE";
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index 909953832..efd139d82 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -20,13 +20,16 @@
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPooled;
+import redis.clients.jedis.UnifiedJedis;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.BasePersistenceStore;
@@ -34,7 +37,7 @@
import software.amazon.lambda.powertools.idempotency.persistence.PersistenceStore;
/**
- * Redis version of the {@link PersistenceStore}. Will store idempotency data in Redis.
+ * Redis version of the {@link PersistenceStore}. Stores idempotency data in Redis standalone or cluster mode.
* Use the {@link Builder} to create a new instance.
*/
public class RedisPersistenceStore extends BasePersistenceStore implements PersistenceStore {
@@ -47,7 +50,7 @@ public class RedisPersistenceStore extends BasePersistenceStore implements Persi
private final String statusAttr;
private final String dataAttr;
private final String validationAttr;
- private final JedisPooled jedisPool;
+ private final UnifiedJedis jedisClient;
/**
* Private: use the {@link Builder} to instantiate a new {@link RedisPersistenceStore}
@@ -59,7 +62,7 @@ private RedisPersistenceStore(String keyPrefixName,
String statusAttr,
String dataAttr,
String validationAttr,
- JedisPooled jedisPool) {
+ UnifiedJedis jedisClient) {
this.keyPrefixName = keyPrefixName;
this.keyAttr = keyAttr;
this.expiryAttr = expiryAttr;
@@ -68,51 +71,63 @@ private RedisPersistenceStore(String keyPrefixName,
this.dataAttr = dataAttr;
this.validationAttr = validationAttr;
- if (jedisPool != null) {
- this.jedisPool = jedisPool;
+ if (jedisClient != null) {
+ this.jedisClient = jedisClient;
} else {
- String idempotencyDisabledEnv = System.getenv().get(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
+ String idempotencyDisabledEnv =
+ System.getenv(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
if (idempotencyDisabledEnv == null || "false".equalsIgnoreCase(idempotencyDisabledEnv)) {
- HostAndPort address = new HostAndPort(System.getenv().get(Constants.REDIS_HOST),
- Integer.parseInt(System.getenv().get(Constants.REDIS_PORT)));
- JedisClientConfig config = getJedisClientConfig();
- this.jedisPool = new JedisPooled(address, config);
+ this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST), Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
} else {
// we do not want to create a Jedis connection pool if idempotency is disabled
// null is ok as idempotency won't be called
- this.jedisPool = null;
+ this.jedisClient = null;
}
}
}
- public static Builder builder() {
- return new Builder();
- }
-
/**
* Set redis user and secret to connect to the redis server
*
* @return
*/
private static JedisClientConfig getJedisClientConfig() {
+ String redisSecret = "";
+ String redisSecretEnv = System.getenv(Constants.REDIS_SECRET);
+ if (redisSecretEnv != null) {
+ redisSecret = redisSecretEnv;
+ }
return DefaultJedisClientConfig.builder()
- .user(System.getenv().get(Constants.REDIS_USER))
- .password(System.getenv().get(Constants.REDIS_SECRET))
+ .user(System.getenv(Constants.REDIS_USER))
+ .password(System.getenv(redisSecret))
.build();
}
- JedisClientConfig config = getJedisClientConfig();
+ public static Builder builder() {
+ return new Builder();
+ }
+ UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
+ HostAndPort address = new HostAndPort(redisHost, redisPort);
+ JedisClientConfig config = getJedisClientConfig();
+ String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
+ if (isClusterMode != null && "true".equalsIgnoreCase(isClusterMode)) {
+ return new JedisCluster(address, getJedisClientConfig(), 5, new GenericObjectPoolConfig<>());
+ } else {
+ return new JedisPooled(address, config);
+ }
+ }
@Override
public DataRecord getRecord(String idempotencyKey) throws IdempotencyItemNotFoundException {
- Map item = jedisPool.hgetAll(getKey(idempotencyKey));
+ String hashKey = getKey(idempotencyKey);
+ Map item = jedisClient.hgetAll(hashKey);
if (item.isEmpty()) {
throw new IdempotencyItemNotFoundException(idempotencyKey);
}
- item.put(this.keyAttr, idempotencyKey);
- return itemToRecord(item);
+ item.put(hashKey, idempotencyKey);
+ return itemToRecord(item, idempotencyKey);
}
/**
@@ -136,18 +151,18 @@ public void putRecord(DataRecord dataRecord, Instant now) {
inProgressExpiry = String.valueOf(dataRecord.getInProgressExpiryTimestamp().getAsLong());
}
- LOG.debug("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
+ LOG.info("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
Object execRes = putItemOnCondition(dataRecord, now, inProgressExpiry);
if (execRes == null) {
String msg = String.format("Failed to put dataRecord for already existing idempotency key: %s",
getKey(dataRecord.getIdempotencyKey()));
- LOG.debug(msg);
+ LOG.info(msg);
throw new IdempotencyItemAlreadyExistsException(msg);
} else {
- LOG.debug("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
- jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
+ LOG.info("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
+ jedisClient.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
}
@@ -176,10 +191,11 @@ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inP
itemIsInProgressExpression, insertItemExpression);
List fields = new ArrayList<>();
- fields.add(getKey(dataRecord.getIdempotencyKey()));
- fields.add(this.expiryAttr);
- fields.add(this.statusAttr);
- fields.add(this.inProgressExpiryAttr);
+ String hashKey = getKey(dataRecord.getIdempotencyKey());
+ fields.add(hashKey);
+ fields.add(prependField(hashKey, this.expiryAttr));
+ fields.add(prependField(hashKey, this.statusAttr));
+ fields.add(prependField(hashKey, this.inProgressExpiryAttr));
fields.add(String.valueOf(now.getEpochSecond()));
fields.add(String.valueOf(now.toEpochMilli()));
fields.add(INPROGRESS.toString());
@@ -191,41 +207,60 @@ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inP
}
String[] arr = new String[fields.size()];
- return jedisPool.eval(luaScript, 4, (String[]) fields.toArray(arr));
+ return jedisClient.eval(luaScript, 4, fields.toArray(arr));
}
@Override
public void updateRecord(DataRecord dataRecord) {
LOG.debug("Updating dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
+ String hashKey = getKey(dataRecord.getIdempotencyKey());
Map item = new HashMap<>();
- item.put(this.dataAttr, dataRecord.getResponseData());
- item.put(this.expiryAttr, String.valueOf(dataRecord.getExpiryTimestamp()));
- item.put(this.statusAttr, String.valueOf(dataRecord.getStatus().toString()));
+ item.put(prependField(hashKey, this.dataAttr), dataRecord.getResponseData());
+ item.put(prependField(hashKey, this.expiryAttr), String.valueOf(dataRecord.getExpiryTimestamp()));
+ item.put(prependField(hashKey, this.statusAttr), String.valueOf(dataRecord.getStatus().toString()));
if (payloadValidationEnabled) {
- item.put(this.validationAttr, dataRecord.getPayloadHash());
+ item.put(prependField(hashKey, this.validationAttr), dataRecord.getPayloadHash());
}
- jedisPool.hset(getKey(dataRecord.getIdempotencyKey()), item);
- jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
+ jedisClient.hset(hashKey, item);
+ jedisClient.expireAt(hashKey, dataRecord.getExpiryTimestamp());
}
+
@Override
public void deleteRecord(String idempotencyKey) {
LOG.debug("Deleting record for idempotency key: {}", idempotencyKey);
- jedisPool.del(getKey(idempotencyKey));
+ jedisClient.del(getKey(idempotencyKey));
}
/**
* Get the key to use for requests
* Sets a keyPrefixName for hash name and a keyAttr for hash key
+ * The key will be used in multi-key operations, therefore we need to
+ * include it into curly braces to instruct the redis cluster which part
+ * of the key will be used for hash and should be stored and looked-up in the same slot.
*
* @param idempotencyKey
* @return
+ * @see Redis Key distribution model
*/
private String getKey(String idempotencyKey) {
- return this.keyPrefixName + ":" + this.keyAttr + ":" + idempotencyKey;
+ return "{" + this.keyPrefixName + ":" + this.keyAttr + ":" + idempotencyKey + "}";
+ }
+
+ /**
+ * Prepend each field key with the unique prefix that will be used for calculating the hash slot
+ * it will be stored in case of cluster mode Redis deployement
+ *
+ * @param hashKey
+ * @param field
+ * @return
+ * @see Redis Key distribution model
+ */
+ private String prependField(String hashKey, String field) {
+ return hashKey + ":" + field;
}
/**
@@ -234,24 +269,22 @@ private String getKey(String idempotencyKey) {
* @param item Item from redis response
* @return DataRecord instance
*/
- private DataRecord itemToRecord(Map item) {
- // data and validation payload may be null
- String data = item.get(this.dataAttr);
- String validation = item.get(this.validationAttr);
- return new DataRecord(item.get(keyAttr),
- DataRecord.Status.valueOf(item.get(this.statusAttr)),
- Long.parseLong(item.get(this.expiryAttr)),
- data,
- validation,
- item.get(this.inProgressExpiryAttr) != null ?
- OptionalLong.of(Long.parseLong(item.get(this.inProgressExpiryAttr))) :
+ private DataRecord itemToRecord(Map item, String idempotencyKey) {
+ String hashKey = getKey(idempotencyKey);
+ return new DataRecord(item.get(getKey(idempotencyKey)),
+ DataRecord.Status.valueOf(item.get(prependField(hashKey, this.statusAttr))),
+ Long.parseLong(item.get(prependField(hashKey, this.expiryAttr))),
+ item.get(prependField(hashKey, this.dataAttr)),
+ item.get(prependField(hashKey, this.validationAttr)),
+ item.get(prependField(hashKey, this.inProgressExpiryAttr)) != null ?
+ OptionalLong.of(Long.parseLong(item.get(prependField(hashKey, this.inProgressExpiryAttr)))) :
OptionalLong.empty());
}
/**
* Use this builder to get an instance of {@link RedisPersistenceStore}.
* With this builder you can configure the characteristics of the Redis hash fields.
- * You can also set a custom {@link JedisPool}.
+ * You can also set a custom {@link UnifiedJedis} client.
*/
public static class Builder {
private String keyPrefixName = "idempotency";
@@ -261,7 +294,7 @@ public static class Builder {
private String statusAttr = "status";
private String dataAttr = "data";
private String validationAttr = "validation";
- private JedisPooled jedisPool;
+ private UnifiedJedis jedisPool;
/**
* Initialize and return a new instance of {@link RedisPersistenceStore}.
@@ -355,13 +388,16 @@ public Builder withValidationAttr(String validationAttr) {
}
/**
- * Custom {@link JedisPool} used to query DynamoDB (optional).
+ * Custom {@link UnifiedJedis} used to query Redis (optional).
+ * This will be cast to either {@link JedisPool} or {@link JedisCluster}
+ * depending on the mode of the Redis deployment and instructed by
+ * the value of {@link Constants#REDIS_CLUSTER_MODE} environment variable.
*
- * @param jedisPool the {@link JedisPool} instance to use
+ * @param jedisClient the {@link UnifiedJedis} instance to use
* @return the builder instance (to chain operations)
*/
- public Builder withJedisPooled(JedisPooled jedisPool) {
- this.jedisPool = jedisPool;
+ public Builder withJedisClient(UnifiedJedis jedisClient) {
+ this.jedisPool = jedisClient;
return this;
}
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index d6a438f3a..b2f6dd2ee 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -19,6 +19,8 @@
import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_HOST;
import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_PORT;
+import com.github.fppt.jedismock.server.ServiceOptions;
+import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
@@ -29,6 +31,7 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.SetEnvironmentVariable;
+import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPooled;
import redis.embedded.RedisServer;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
@@ -64,15 +67,54 @@ void putRecord_shouldCreateItemInRedis() {
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map entry = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
+ @Test
+ void putRecord_shouldCreateItemInRedisClusterMode() throws IOException {
+ com.github.fppt.jedismock.RedisServer redisCluster = com.github.fppt.jedismock.RedisServer
+ .newRedisServer()
+ .setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
+ .start();
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ JedisPooled jp = new JedisPooled(redisCluster.getHost(), redisCluster.getBindPort());
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisClient(jp).build();
+
+ store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jp.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jp.ttl("{idempotency:id:key}");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "true")
+ @Test
+ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
+ com.github.fppt.jedismock.RedisServer redisCluster = com.github.fppt.jedismock.RedisServer
+ .newRedisServer()
+ .setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
+ .start();
+ assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(), redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
+ redisCluster.stop();
+ }
+
+ @SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
+ @Test
+ void putRecord_JedisClientInstanceOfJedisPooled() {
+ assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST), Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
+ }
@Test
void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
@@ -82,13 +124,14 @@ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
redisPersistenceStore.putRecord(
new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("in-progress-expiration", String.valueOf(progressExpiry.getAsLong()));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:in-progress-expiration",
+ String.valueOf(progressExpiry.getAsLong()));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -96,14 +139,14 @@ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
- RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisClient(jedisPool).build();
store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
}
@Test
@@ -112,13 +155,13 @@ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.minus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", "Fake Data");
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
long ttl = 3600;
long expiry2 = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
- jedisPool.hset("idempotency:id:key", item);
+ jedisPool.hset("{idempotency:id:key}", item);
redisPersistenceStore.putRecord(
new DataRecord("key",
DataRecord.Status.INPROGRESS,
@@ -127,12 +170,12 @@ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
null
), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry2));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -143,11 +186,11 @@ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
long progressExpiry = now.minus(30, ChronoUnit.SECONDS).toEpochMilli();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- item.put("data", "Fake Data");
- item.put("in-progress-expiration", String.valueOf(progressExpiry));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
+ item.put("{idempotency:id:key}:in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
redisPersistenceStore.putRecord(
@@ -158,11 +201,11 @@ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
null
), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry2));
}
@Test
@@ -171,11 +214,11 @@ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyE
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry)); // not expired
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", "Fake Data");
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry)); // not expired
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
- jedisPool.hset("idempotency:id:key", item);
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
DataRecord dataRecord = new DataRecord("key",
@@ -190,12 +233,12 @@ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyE
}
).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map entry = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("data")).isEqualTo("Fake Data");
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("{idempotency:id:key}:data")).isEqualTo("Fake Data");
}
@Test
@@ -205,11 +248,11 @@ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterL
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond(); // not expired
long progressExpiry = now.plus(30, ChronoUnit.SECONDS).toEpochMilli(); // not expired
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- item.put("data", "Fake Data");
- item.put("in-progress-expiration", String.valueOf(progressExpiry));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
+ item.put("{idempotency:id:key}:in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
DataRecord dataRecord = new DataRecord("key",
@@ -222,12 +265,12 @@ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterL
dataRecord, now))
.isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("data", "Fake Data");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:data", "Fake Data");
}
@Test
@@ -236,10 +279,10 @@ void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundExcept
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", ("Fake Data"));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", ("Fake Data"));
+ jedisPool.hset("{idempotency:id:key}", item);
DataRecord record = redisPersistenceStore.getRecord("key");
@@ -260,9 +303,9 @@ void updateRecord_shouldUpdateRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("{idempotency:id:key}", item);
// enable payload validation
redisPersistenceStore.configure(IdempotencyConfig.builder().withPayloadValidationJMESPath("path").build(),
null);
@@ -272,13 +315,13 @@ void updateRecord_shouldUpdateRecord() {
DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
redisPersistenceStore.updateRecord(record);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("data", "Fake result");
- assertThat(redisItem).containsEntry("validation", "hash");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:data", "Fake result");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:validation", "hash");
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -287,13 +330,13 @@ void deleteRecord_shouldDeleteRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("{idempotency:id:key}", item);
redisPersistenceStore.deleteRecord("key");
- Map items = jedisPool.hgetAll("idempotency:id:key");
+ Map items = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(items).isEmpty();
}
@@ -304,7 +347,7 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
try {
RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
.withKeyPrefixName("items-idempotency")
- .withJedisPooled(jedisPool)
+ .withJedisClient(jedisPool)
.withDataAttr("result")
.withExpiryAttr("expiry")
.withKeyAttr("key")
@@ -323,14 +366,16 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
// PUT
persistenceStore.putRecord(record, now);
- Map redisItem = jedisPool.hgetAll("items-idempotency:key:mykey");
+ Map redisItem = jedisPool.hgetAll("{items-idempotency:key:mykey}");
// GET
DataRecord recordInDb = persistenceStore.getRecord("mykey");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("state", recordInDb.getStatus().toString());
- assertThat(redisItem).containsEntry("expiry", String.valueOf(recordInDb.getExpiryTimestamp()));
+ assertThat(redisItem).containsEntry("{items-idempotency:key:mykey}:state",
+ recordInDb.getStatus().toString());
+ assertThat(redisItem).containsEntry("{items-idempotency:key:mykey}:expiry",
+ String.valueOf(recordInDb.getExpiryTimestamp()));
// UPDATE
DataRecord updatedRecord = new DataRecord(
@@ -346,10 +391,10 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
// DELETE
persistenceStore.deleteRecord("mykey");
- assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isZero();
+ assertThat(jedisPool.hgetAll("{items-idempotency:key:mykey}").size()).isZero();
} finally {
- jedisPool.del("items-idempotency:key:mykey");
+ jedisPool.del("{items-idempotency:key:mykey}");
}
}
@@ -362,7 +407,7 @@ void idempotencyDisabled_noClientShouldBeCreated() {
@AfterEach
void emptyDB() {
- jedisPool.del("idempotency:id:key");
+ jedisPool.del("{idempotency:id:key}");
}
}
From db9671dd5aa2dc6b157b45452552000a1b9b2724 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 13 Dec 2023 10:19:06 +0200
Subject: [PATCH 08/19] docs improvements - Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com>
---
docs/utilities/idempotency.md | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 8d6077d68..6810628e8 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -164,7 +164,7 @@ As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https
#### Using Amazon DynamoDB as persistent storage layer
-If you are using Amazon DynamoDB you'll need to create a table.
+If you are using Amazon DynamoDB, you'll need to create a table.
**Default table configuration**
@@ -221,9 +221,9 @@ Resources:
##### Redis resources
-You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) as persistent storage layer provider.
+You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/).
!!! tip "Tip:No existing Redis service?"
-If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+ If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
@@ -257,10 +257,10 @@ Resources:
Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
-If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+ If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
!!! warning "Amazon ElastiCache Serverless not supported"
-[Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
+ [Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
!!! warning "Check network connectivity to Redis server"
Make sure that your AWS Lambda function can connect to your Redis server.
@@ -722,7 +722,7 @@ You can alter the field names by passing these parameters when initializing the
!!! Tip "Tip: You can share the same prefix and key for all functions"
-You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
+ You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
## Advanced
From 2a6747f6be4ec73cb69c4d40242e76f7008da655 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 13 Dec 2023 11:14:26 +0200
Subject: [PATCH 09/19] Apply checkstyle to fix import order
---
.../idempotency/IdempotencyConfig.java | 3 +-
.../powertools/idempotency/Idempotent.java | 1 -
.../internal/IdempotencyHandler.java | 11 +++---
.../internal/IdempotentAspect.java | 9 ++---
.../persistence/BasePersistenceStore.java | 39 +++++++++----------
.../idempotency/persistence/DataRecord.java | 3 +-
.../persistence/PersistenceStore.java | 3 +-
.../internal/IdempotencyAspectTest.java | 29 +++++++-------
.../internal/cache/LRUCacheTest.java | 4 +-
.../persistence/BasePersistenceStoreTest.java | 15 ++++---
.../dynamodb/DynamoDBPersistenceStore.java | 23 ++++++-----
.../handlers/IdempotencyFunction.java | 15 ++++---
.../DynamoDBPersistenceStoreTest.java | 17 ++++----
.../redis/RedisPersistenceStore.java | 11 ++++--
.../redis/RedisPersistenceStoreTest.java | 7 +++-
15 files changed, 92 insertions(+), 98 deletions(-)
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
index 2b22cac51..baf939d11 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
@@ -15,9 +15,8 @@
package software.amazon.lambda.powertools.idempotency;
import com.amazonaws.services.lambda.runtime.Context;
-import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
-
import java.time.Duration;
+import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
/**
* Configuration of the idempotency feature. Use the {@link Builder} to create an instance.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
index d08874492..6ca40a0e1 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
@@ -15,7 +15,6 @@
package software.amazon.lambda.powertools.idempotency;
import com.amazonaws.services.lambda.runtime.Context;
-
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
index 7982d911a..2875ab3d1 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
@@ -14,8 +14,13 @@
package software.amazon.lambda.powertools.idempotency.internal;
+import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.EXPIRED;
+import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
+
import com.amazonaws.services.lambda.runtime.Context;
import com.fasterxml.jackson.databind.JsonNode;
+import java.time.Instant;
+import java.util.OptionalInt;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
@@ -32,12 +37,6 @@
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.time.Instant;
-import java.util.OptionalInt;
-
-import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.EXPIRED;
-import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
-
/**
* Internal class that will handle the Idempotency, and use the {@link software.amazon.lambda.powertools.idempotency.persistence.PersistenceStore}
* to store the result of previous calls.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
index ea6d743f0..0b9d729f4 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
@@ -14,8 +14,12 @@
package software.amazon.lambda.powertools.idempotency.internal;
+import static software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor.placedOnRequestHandler;
+
import com.amazonaws.services.lambda.runtime.Context;
import com.fasterxml.jackson.databind.JsonNode;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
@@ -29,11 +33,6 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyConfigurationException;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Method;
-
-import static software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor.placedOnRequestHandler;
-
/**
* Aspect that handles the {@link Idempotent} annotation.
* It uses the {@link IdempotencyHandler} to actually do the job.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
index bafbcbd42..0a1acdf5c 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
@@ -14,20 +14,12 @@
package software.amazon.lambda.powertools.idempotency.persistence;
+import static software.amazon.lambda.powertools.common.internal.LambdaConstants.LAMBDA_FUNCTION_NAME_ENV;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectWriter;
import io.burt.jmespath.Expression;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyKeyException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyValidationException;
-import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
-import software.amazon.lambda.powertools.utilities.JsonConfig;
-
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -41,8 +33,15 @@
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-
-import static software.amazon.lambda.powertools.common.internal.LambdaConstants.LAMBDA_FUNCTION_NAME_ENV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyKeyException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyValidationException;
+import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
+import software.amazon.lambda.powertools.utilities.JsonConfig;
/**
* Persistence layer that will store the idempotency result.
@@ -64,6 +63,14 @@ public abstract class BasePersistenceStore implements PersistenceStore {
private boolean throwOnNoIdempotencyKey = false;
private String hashFunctionName;
+ private static boolean isEqual(String dataRecordPayload, String dataHash) {
+ if (dataHash != null && dataRecordPayload != null) {
+ return dataHash.length() != dataRecordPayload.length() ? false : dataHash.equals(dataRecordPayload);
+ } else {
+ return false;
+ }
+ }
+
/**
* Initialize the base persistence layer from the configuration settings
*
@@ -402,12 +409,4 @@ void configure(IdempotencyConfig config, String functionName, LRUCache
* Use the {@link Builder} to create a new instance.
diff --git a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
index 1296a75c7..76a012930 100644
--- a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
+++ b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
@@ -18,6 +18,13 @@
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent;
import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
@@ -27,14 +34,6 @@
import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
public class IdempotencyFunction implements RequestHandler {
private final static Logger LOG = LogManager.getLogger(IdempotencyFunction.class);
diff --git a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
index cc682a81f..e67420def 100644
--- a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
@@ -14,6 +14,14 @@
package software.amazon.lambda.powertools.idempotency.persistence.dynamodb;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -37,15 +45,6 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
/**
* These test are using DynamoDBLocal and sqlite, see https://nickolasfisher.com/blog/Configuring-an-In-Memory-DynamoDB-instance-with-Java-for-Integration-Testing
* NOTE: on a Mac with Apple Chipset, you need to use the Oracle JDK x86 64-bit
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index efd139d82..0ca687e6e 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -1,4 +1,4 @@
-package software.amazon.lambda.powertools.idempotency.redis;/*
+/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
@@ -12,6 +12,8 @@
*
*/
+package software.amazon.lambda.powertools.idempotency.redis;
+
import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
import java.time.Instant;
@@ -77,7 +79,8 @@ private RedisPersistenceStore(String keyPrefixName,
String idempotencyDisabledEnv =
System.getenv(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
if (idempotencyDisabledEnv == null || "false".equalsIgnoreCase(idempotencyDisabledEnv)) {
- this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST), Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
+ this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST),
+ Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
} else {
// we do not want to create a Jedis connection pool if idempotency is disabled
// null is ok as idempotency won't be called
@@ -107,11 +110,11 @@ public static Builder builder() {
return new Builder();
}
- UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
+ UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
HostAndPort address = new HostAndPort(redisHost, redisPort);
JedisClientConfig config = getJedisClientConfig();
String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
- if (isClusterMode != null && "true".equalsIgnoreCase(isClusterMode)) {
+ if ("true".equalsIgnoreCase(isClusterMode)) {
return new JedisCluster(address, getJedisClientConfig(), 5, new GenericObjectPoolConfig<>());
} else {
return new JedisPooled(address, config);
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index b2f6dd2ee..adacb1a2b 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -106,15 +106,18 @@ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
.newRedisServer()
.setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
.start();
- assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(), redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
+ assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(),
+ redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
redisCluster.stop();
}
@SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
@Test
void putRecord_JedisClientInstanceOfJedisPooled() {
- assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST), Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
+ assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST),
+ Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
}
+
@Test
void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
From 4191e347f9789c1479416bda8badbc57a6bd9d73 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 19 Dec 2023 12:15:50 +0200
Subject: [PATCH 10/19] Fix build
---
powertools-large-messages/pom.xml | 5 +++++
powertools-logging/powertools-logging-log4j/pom.xml | 4 ++++
powertools-validation/pom.xml | 4 ++++
3 files changed, 13 insertions(+)
diff --git a/powertools-large-messages/pom.xml b/powertools-large-messages/pom.xml
index 4206183de..1bd670054 100644
--- a/powertools-large-messages/pom.xml
+++ b/powertools-large-messages/pom.xml
@@ -117,6 +117,11 @@
log4j-slf4j2-impl
test
+
+ org.apache.logging.log4j
+ log4j-api
+ test
+
diff --git a/powertools-logging/powertools-logging-log4j/pom.xml b/powertools-logging/powertools-logging-log4j/pom.xml
index df6154560..752f8014c 100644
--- a/powertools-logging/powertools-logging-log4j/pom.xml
+++ b/powertools-logging/powertools-logging-log4j/pom.xml
@@ -35,6 +35,10 @@
org.apache.logging.log4j
log4j-core
+
+ org.apache.logging.log4j
+ log4j-api
+
org.apache.logging.log4j
log4j-layout-template-json
diff --git a/powertools-validation/pom.xml b/powertools-validation/pom.xml
index 0de38c1c1..da5daa2f1 100644
--- a/powertools-validation/pom.xml
+++ b/powertools-validation/pom.xml
@@ -71,6 +71,10 @@
com.amazonaws
aws-lambda-java-serialization
+
+ org.aspectj
+ aspectjrt
+
From 4fcf499c47edd35eae33e3cde93ecb1129feaedc Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 19 Dec 2023 13:29:11 +0200
Subject: [PATCH 11/19] Fix spotbugs target and build
---
.gitignore | 3 +-
.../{ => persistence}/redis/Constants.java | 2 +-
.../redis/RedisPersistenceStore.java | 8 ++--
.../redis/RedisPersistenceStoreTest.java | 12 ++---
powertools-idempotency/spotbugs-exclude.xml | 46 -------------------
powertools-parameters/pom.xml | 4 ++
spotbugs-exclude.xml | 36 +++++++++++----
7 files changed, 44 insertions(+), 67 deletions(-)
rename powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/{ => persistence}/redis/Constants.java (92%)
rename powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/{ => persistence}/redis/RedisPersistenceStore.java (98%)
rename powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/{ => persistence}/redis/RedisPersistenceStoreTest.java (97%)
delete mode 100644 powertools-idempotency/spotbugs-exclude.xml
diff --git a/.gitignore b/.gitignore
index 6615ac729..995d4ce8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -110,4 +110,5 @@ example/HelloWorldFunction/build
.gradle
build/
.terraform*
-terraform.tfstate*
\ No newline at end of file
+terraform.tfstate*
+/powertools-idempotency/powertools-idempotency-dynamodb/dynamodb-local-metadata.json
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
similarity index 92%
rename from powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
index 55b37f999..b1c65940e 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
@@ -12,7 +12,7 @@
*
*/
-package software.amazon.lambda.powertools.idempotency.redis;
+package software.amazon.lambda.powertools.idempotency.persistence.redis;
public class Constants {
public static final String REDIS_HOST = "REDIS_HOST";
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
similarity index 98%
rename from powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
index 0ca687e6e..4ec9ea344 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
@@ -12,7 +12,7 @@
*
*/
-package software.amazon.lambda.powertools.idempotency.redis;
+package software.amazon.lambda.powertools.idempotency.persistence.redis;
import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
@@ -297,7 +297,7 @@ public static class Builder {
private String statusAttr = "status";
private String dataAttr = "data";
private String validationAttr = "validation";
- private UnifiedJedis jedisPool;
+ private UnifiedJedis jedisClient;
/**
* Initialize and return a new instance of {@link RedisPersistenceStore}.
@@ -310,7 +310,7 @@ public static class Builder {
*/
public RedisPersistenceStore build() {
return new RedisPersistenceStore(keyPrefixName, keyAttr, expiryAttr,
- inProgressExpiryAttr, statusAttr, dataAttr, validationAttr, jedisPool);
+ inProgressExpiryAttr, statusAttr, dataAttr, validationAttr, jedisClient);
}
/**
@@ -400,7 +400,7 @@ public Builder withValidationAttr(String validationAttr) {
* @return the builder instance (to chain operations)
*/
public Builder withJedisClient(UnifiedJedis jedisClient) {
- this.jedisPool = jedisClient;
+ this.jedisClient = jedisClient;
return this;
}
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
similarity index 97%
rename from powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
rename to powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
index adacb1a2b..09c493d78 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
@@ -12,12 +12,10 @@
*
*/
-package software.amazon.lambda.powertools.idempotency.redis;
+package software.amazon.lambda.powertools.idempotency.persistence.redis;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_HOST;
-import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_PORT;
import com.github.fppt.jedismock.server.ServiceOptions;
import java.io.IOException;
@@ -39,8 +37,8 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
-@SetEnvironmentVariable(key = REDIS_HOST, value = "localhost")
-@SetEnvironmentVariable(key = REDIS_PORT, value = "6379")
+@SetEnvironmentVariable(key = Constants.REDIS_HOST, value = "localhost")
+@SetEnvironmentVariable(key = Constants.REDIS_PORT, value = "6379")
public class RedisPersistenceStoreTest {
static RedisServer redisServer;
private final RedisPersistenceStore redisPersistenceStore = RedisPersistenceStore.builder().build();
@@ -114,8 +112,8 @@ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
@SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
@Test
void putRecord_JedisClientInstanceOfJedisPooled() {
- assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST),
- Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
+ assertThat(redisPersistenceStore.getJedisClient(System.getenv(Constants.REDIS_HOST),
+ Integer.parseInt(System.getenv(Constants.REDIS_PORT))) instanceof JedisCluster).isFalse();
}
@Test
diff --git a/powertools-idempotency/spotbugs-exclude.xml b/powertools-idempotency/spotbugs-exclude.xml
deleted file mode 100644
index 9a2369c75..000000000
--- a/powertools-idempotency/spotbugs-exclude.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/powertools-parameters/pom.xml b/powertools-parameters/pom.xml
index 6c90e30a8..c730f4ca3 100644
--- a/powertools-parameters/pom.xml
+++ b/powertools-parameters/pom.xml
@@ -48,6 +48,10 @@
com.fasterxml.jackson.core
jackson-databind
+
+ org.aspectj
+ aspectjrt
+
org.junit.jupiter
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
index ee39b5d0f..893298d2e 100644
--- a/spotbugs-exclude.xml
+++ b/spotbugs-exclude.xml
@@ -27,10 +27,6 @@
-
-
-
-
@@ -85,6 +81,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -110,10 +122,6 @@
-
-
-
-
@@ -138,6 +146,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
From f5b7c1e967aac2d20c2a21b5076ccf36e76f9374 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 19 Dec 2023 13:37:16 +0200
Subject: [PATCH 12/19] Fix e2e build for Java8
---
.../amazon/lambda/powertools/testutils/Infrastructure.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index e61fbe873..be7b556f9 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -168,7 +168,7 @@ private Infrastructure(Builder builder) {
if (isRedisDeployment) {
this.vpc = Vpc.Builder.create(this.stack, "MyVPC-" + stackName)
- .availabilityZones(List.of(region.toString() + "a", region.toString() + "b"))
+ .availabilityZones(Arrays.asList(region.toString() + "a", region + "b"))
.build();
List subnets = vpc.getPublicSubnets().stream().map(subnet ->
@@ -311,7 +311,7 @@ private void createStackWithLambda() {
if (isRedisDeployment) {
functionBuilder.vpc(vpc)
.vpcSubnets(subnetSelection)
- .securityGroups(List.of(lambdaSecurityGroup));
+ .securityGroups(singletonList(lambdaSecurityGroup));
}
Function function = functionBuilder.build();
@@ -344,7 +344,7 @@ private void createStackWithLambda() {
.engine("redis")
.cacheNodeType("cache.t2.micro")
.cacheSubnetGroupName(cfnSubnetGroup.getCacheSubnetGroupName())
- .vpcSecurityGroupIds(List.of(securityGroup.getSecurityGroupId()))
+ .vpcSecurityGroupIds(singletonList(securityGroup.getSecurityGroupId()))
.numCacheNodes(1)
.build();
redisServer.addDependency(cfnSubnetGroup);
From 408afa38065ce88d3358745aae72c72578fd0bc0 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 19 Dec 2023 13:52:38 +0200
Subject: [PATCH 13/19] Fix e2e tests build
---
.../java/software/amazon/lambda/powertools/e2e/Function.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index 994f14d0c..9796ac8ea 100644
--- a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -24,7 +24,7 @@
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
-import software.amazon.lambda.powertools.idempotency.redis.RedisPersistenceStore;
+import software.amazon.lambda.powertools.idempotency.persistence.redis.RedisPersistenceStore;
import software.amazon.lambda.powertools.logging.Logging;
From 6ac04e502d5ed76e9ab340d0df10f26119058ddd Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:15:58 +0200
Subject: [PATCH 14/19] Exposes JedisClientConfig to allow custom config (ssl,
db, etc..), improves doc, extracts lua script as resource
---
docs/utilities/idempotency.md | 69 +++++----
.../lambda/powertools/e2e/Function.java | 6 +
powertools-e2e-tests/pom.xml | 8 +-
.../powertools/testutils/Infrastructure.java | 20 ++-
.../persistence/redis/Constants.java | 10 +-
.../persistence/redis/JedisConfig.java | 91 ++++++++++++
.../redis/RedisPersistenceStore.java | 136 +++++++++---------
.../main/resources/putRecordOnCondition.lua | 19 +++
.../redis/RedisPersistenceStoreTest.java | 54 +++++--
9 files changed, 282 insertions(+), 131 deletions(-)
create mode 100644 powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/JedisConfig.java
create mode 100644 powertools-idempotency/powertools-idempotency-redis/src/main/resources/putRecordOnCondition.lua
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 6810628e8..be577a464 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -162,7 +162,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your Lambda functions will need read and write access to it.
As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https://redis.io/) are the supported persistnce layers.
-#### Using Amazon DynamoDB as persistent storage layer
+#### Using Amazon DynamoDB
If you are using Amazon DynamoDB, you'll need to create a table.
@@ -217,28 +217,23 @@ Resources:
see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to
estimate the cost.
-#### Using Redis as persistent storage layer
+#### Using Redis
##### Redis resources
-You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/).
+You need an existing Redis service before setting up Redis as the persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/).
!!! tip "Tip:No existing Redis service?"
- If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+ If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider. DynamoDB does not require a VPC deployment and is easier to configure and operate.
-If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
environment variable `REDIS_CLUSTER_MODE` to `true`
-In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
-If you provide a [custom Redis client](#Customizing Redis client) you can omit the environment variables declaration.
-
-!!! warning "Warning: Avoid including a plain text secret directly in your template"
-This can infer security implications
+In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variable.
!!! warning "Warning: Large responses with Redis persistence layer"
When using this utility with Redis your function's responses must be smaller than 512MB.
Persisting larger items cannot might cause exceptions.
-```yaml hl_lines="9-13" title="AWS Serverless Application Model (SAM) example"
+```yaml hl_lines="9" title="AWS Serverless Application Model (SAM) example"
Resources:
IdempotencyFunction:
Type: AWS::Serverless::Function
@@ -247,10 +242,6 @@ Resources:
Handler: helloworld.App::handleRequest
Environment:
Variables:
- REDIS_HOST: %redis-host%
- REDIS_PORT: %redis-port%
- REDIS_USER: %redis-user%
- REDIS_SECRET: %redis-secret%
REDIS_CLUSTER_MODE: "true"
```
##### VPC Access
@@ -259,13 +250,7 @@ Your AWS Lambda Function must be able to reach the Redis endpoint before using i
!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
-!!! warning "Amazon ElastiCache Serverless not supported"
- [Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
-
-!!! warning "Check network connectivity to Redis server"
-Make sure that your AWS Lambda function can connect to your Redis server.
-
-```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+```yaml hl_lines="7-12" title="AWS Serverless Application Model (SAM) example"
Resources:
IdempotencyFunction:
Type: AWS::Serverless::Function
@@ -273,20 +258,22 @@ Resources:
CodeUri: Function
Handler: helloworld.App::handleRequest
VpcConfig: # (1)!
- SecurityGroupIds:
+ SecurityGroupIds: # (2)!
- sg-{your_sg_id}
- SubnetIds:
+ SubnetIds: # (3)!
- subnet-{your_subnet_id_1}
- subnet-{your_subnet_id_2}
```
1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
+2. The security group ID or IDs of the VPC where the Redis deployment is configured.
+3. The subnet IDs of the VPC where the Redis deployment is configured.
### Idempotent annotation
-You can quickly start by initializing the `DynamoDBPersistenceStore` and using it with the `@Idempotent` annotation on your Lambda handler.
+You can quickly start by initializing the persistence store used (e.g. the `DynamoDBPersistenceStore`) and using it with the `@Idempotent` annotation on your Lambda handler.
!!! warning "Important"
- Initialization and configuration of the `DynamoDBPersistenceStore` must be performed outside the handler, preferably in the constructor.
+ Initialization and configuration of the persistence store must be performed outside the handler, preferably in the constructor.
=== "App.java"
@@ -975,23 +962,30 @@ When creating the `DynamoDBPersistenceStore`, you can set a custom [`DynamoDbCli
### Customizing Redis client
-The `RedisPersistenceStore` uses the JedisPooled java client to connect to the Redis Server.
-When creating the `RedisPersistenceStore`, you can set a custom [`JedisPooled`](https://www.javadoc.io/doc/redis.clients/jedis/latest/redis/clients/jedis/JedisPooled.html) client:
+The `RedisPersistenceStore` uses the [`JedisPooled`](https://www.javadoc.io/doc/redis.clients/jedis/latest/redis/clients/jedis/JedisPooled.html) java client to connect to the Redis standalone server or the [`JedisCluster`](https://javadoc.io/doc/redis.clients/jedis/4.0.0/redis/clients/jedis/JedisCluster.html) to connect to the Redis cluster.
+When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
=== "Custom JedisPooled with connection timeout"
- ```java hl_lines="2-6 11"
+ ```java hl_lines="2-11 13 18"
public App() {
- JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), DefaultJedisClientConfig.builder()
- .user("user")
- .password("secret")
- .connectionTimeoutMillis(3000)
- .build())
+ JedisConfig jedisConfig = JedisConfig.Builder.builder()
+ .withHost(redisCluster.getHost())
+ .withPort(redisCluster.getBindPort())
+ .withJedisClientConfig(DefaultJedisClientConfig.builder()
+ .user("user")
+ .password("secret")
+ .ssl(true)
+ .connectionTimeoutMillis(3000)
+ .build())
+ .build();
+
+ JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), jedisConfig)
Idempotency.config().withPersistenceStore(
RedisPersistenceStore.builder()
.withKeyPrefixName("items-idempotency")
- .withJedisPooled(jedisPooled)
+ .withJedisClient(jedisPooled)
.build()
).configure();
}
@@ -1001,8 +995,9 @@ When creating the `RedisPersistenceStore`, you can set a custom [`JedisPooled`](
```java
DefaultJedisClientConfig.builder()
- .user(System.getenv(Constants.REDIS_USER))
- .password(System.getenv(Constants.REDIS_SECRET))
+ .user("default")
+ .password("")
+ .ssl(false)
.build();
```
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index 9796ac8ea..3b2f0d49e 100644
--- a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -21,9 +21,11 @@
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.TimeZone;
+import redis.clients.jedis.DefaultJedisClientConfig;
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
+import software.amazon.lambda.powertools.idempotency.persistence.redis.JedisConfig;
import software.amazon.lambda.powertools.idempotency.persistence.redis.RedisPersistenceStore;
import software.amazon.lambda.powertools.logging.Logging;
@@ -36,6 +38,10 @@ public Function() {
.build())
.withPersistenceStore(
RedisPersistenceStore.builder()
+ .withJedisConfig(JedisConfig.Builder.builder()
+ .withJedisClientConfig(DefaultJedisClientConfig.builder().ssl(true).build())
+ .withHost(System.getenv("REDIS_HOST"))
+ .withPort(6379).build())
.build()
).configure();
}
diff --git a/powertools-e2e-tests/pom.xml b/powertools-e2e-tests/pom.xml
index 9d84ce9f2..09379d381 100644
--- a/powertools-e2e-tests/pom.xml
+++ b/powertools-e2e-tests/pom.xml
@@ -31,7 +31,7 @@
1.8
1.8
10.3.0
- 2.109.0
+ 2.115.0
@@ -41,6 +41,12 @@
test
+
+ org.apache.logging.log4j
+ log4j-api
+ test
+
+
software.amazon.awssdk
lambda
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index be7b556f9..0a0de2a1b 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -59,7 +59,7 @@
import software.amazon.awscdk.services.ec2.SecurityGroup;
import software.amazon.awscdk.services.ec2.SubnetSelection;
import software.amazon.awscdk.services.ec2.Vpc;
-import software.amazon.awscdk.services.elasticache.CfnCacheCluster;
+import software.amazon.awscdk.services.elasticache.CfnServerlessCache;
import software.amazon.awscdk.services.elasticache.CfnSubnetGroup;
import software.amazon.awscdk.services.iam.PolicyStatement;
import software.amazon.awscdk.services.kinesis.Stream;
@@ -339,18 +339,16 @@ private void createStackWithLambda() {
}
if (isRedisDeployment) {
- CfnCacheCluster redisServer = CfnCacheCluster.Builder.create(this.stack, "ElastiCacheCluster-" + stackName)
- .clusterName("redis-cluster-" + stackName)
+ List subnets = vpc.getPublicSubnets().stream().map(subnet ->
+ subnet.getSubnetId()).collect(Collectors.toList());
+ CfnServerlessCache redisServer = CfnServerlessCache.Builder.create(this.stack, "ECC-" + stackName)
+ .serverlessCacheName("rc-" + stackName)
.engine("redis")
- .cacheNodeType("cache.t2.micro")
- .cacheSubnetGroupName(cfnSubnetGroup.getCacheSubnetGroupName())
- .vpcSecurityGroupIds(singletonList(securityGroup.getSecurityGroupId()))
- .numCacheNodes(1)
+ .subnetIds(subnets)
+ .securityGroupIds(singletonList(securityGroup.getSecurityGroupId()))
.build();
- redisServer.addDependency(cfnSubnetGroup);
- function.addEnvironment("REDIS_HOST", redisServer.getAttrRedisEndpointAddress());
- function.addEnvironment("REDIS_PORT", redisServer.getAttrRedisEndpointPort());
- function.addEnvironment("REDIS_USER", "default");
+
+ function.addEnvironment("REDIS_HOST", redisServer.getAtt("Endpoint.Address").toString());
}
if (!StringUtils.isEmpty(queue)) {
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
index b1c65940e..06e14bee2 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/Constants.java
@@ -14,10 +14,10 @@
package software.amazon.lambda.powertools.idempotency.persistence.redis;
-public class Constants {
- public static final String REDIS_HOST = "REDIS_HOST";
- public static final String REDIS_PORT = "REDIS_PORT";
- public static final String REDIS_USER = "REDIS_USER";
- public static final String REDIS_SECRET = "REDIS_SECRET";
+final class Constants {
+ private Constants() {
+ throw new IllegalStateException("Utility class");
+ }
+
public static final String REDIS_CLUSTER_MODE = "REDIS_CLUSTER_MODE";
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/JedisConfig.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/JedisConfig.java
new file mode 100644
index 000000000..3b11078be
--- /dev/null
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/JedisConfig.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.idempotency.persistence.redis;
+
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.JedisClientConfig;
+
+public class JedisConfig {
+
+ private final String host;
+ private final Integer port;
+ private final JedisClientConfig jedisClientConfig;
+
+ public JedisConfig(String host, Integer port, JedisClientConfig jedisClientConfig) {
+ this.host = host;
+ this.port = port;
+ this.jedisClientConfig = jedisClientConfig;
+ }
+
+ String getHost() {
+ return host;
+ }
+
+ Integer getPort() {
+ return port;
+ }
+
+ public JedisClientConfig getJedisClientConfig() {
+ return jedisClientConfig;
+ }
+
+ public static class Builder {
+ private String host = "localhost";
+ private Integer port = 6379;
+
+ private JedisClientConfig jedisClientConfig = DefaultJedisClientConfig.builder().build();
+
+ public static JedisConfig.Builder builder() {
+ return new JedisConfig.Builder();
+ }
+
+ public JedisConfig build() {
+ return new JedisConfig(host, port, jedisClientConfig);
+ }
+
+ /**
+ * Host name of the redis deployment (optional), by default "localhost"
+ *
+ * @param host host name of the Redis deployment
+ * @return the builder instance (to chain operations)
+ */
+ public JedisConfig.Builder withHost(String host) {
+ this.host = host;
+ return this;
+ }
+
+ /**
+ * Port for the redis deployment (optional), by default 6379
+ *
+ * @param port port for the Redis deployment
+ * @return the builder instance (to chain operations)
+ */
+ public JedisConfig.Builder withPort(Integer port) {
+ this.port = port;
+ return this;
+ }
+
+ /**
+ * Custom configuration for the redis client, by default {@link DefaultJedisClientConfig}
+ *
+ * @param jedisClientConfig custom configuration for the redis client
+ * @return the builder instance (to chain operations)
+ */
+ public JedisConfig.Builder withJedisClientConfig(JedisClientConfig jedisClientConfig) {
+ this.jedisClientConfig = jedisClientConfig;
+ return this;
+ }
+ }
+}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
index 4ec9ea344..bb47db142 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
@@ -16,22 +16,26 @@
import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
+import java.util.stream.Collectors;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPooled;
import redis.clients.jedis.UnifiedJedis;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyConfigurationException;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.BasePersistenceStore;
@@ -45,6 +49,7 @@
public class RedisPersistenceStore extends BasePersistenceStore implements PersistenceStore {
private static final Logger LOG = LoggerFactory.getLogger(RedisPersistenceStore.class);
+ public static final String UPDATE_SCRIPT_LUA = "putRecordOnCondition.lua";
private final String keyPrefixName;
private final String keyAttr;
private final String expiryAttr;
@@ -53,11 +58,14 @@ public class RedisPersistenceStore extends BasePersistenceStore implements Persi
private final String dataAttr;
private final String validationAttr;
private final UnifiedJedis jedisClient;
+ private final String luaScript;
+ private final JedisConfig jedisConfig;
/**
* Private: use the {@link Builder} to instantiate a new {@link RedisPersistenceStore}
*/
- private RedisPersistenceStore(String keyPrefixName,
+ private RedisPersistenceStore(JedisConfig jedisConfig,
+ String keyPrefixName,
String keyAttr,
String expiryAttr,
String inProgressExpiryAttr,
@@ -65,6 +73,7 @@ private RedisPersistenceStore(String keyPrefixName,
String dataAttr,
String validationAttr,
UnifiedJedis jedisClient) {
+ this.jedisConfig = jedisConfig;
this.keyPrefixName = keyPrefixName;
this.keyAttr = keyAttr;
this.expiryAttr = expiryAttr;
@@ -79,46 +88,35 @@ private RedisPersistenceStore(String keyPrefixName,
String idempotencyDisabledEnv =
System.getenv(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
if (idempotencyDisabledEnv == null || "false".equalsIgnoreCase(idempotencyDisabledEnv)) {
- this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST),
- Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
+ this.jedisClient = getJedisClient(this.jedisConfig);
} else {
// we do not want to create a Jedis connection pool if idempotency is disabled
// null is ok as idempotency won't be called
this.jedisClient = null;
}
}
- }
+ try (InputStreamReader luaScriptReader = new InputStreamReader(
+ RedisPersistenceStore.class.getClassLoader().getResourceAsStream(UPDATE_SCRIPT_LUA))) {
+ luaScript = new BufferedReader(
+ luaScriptReader).lines().collect(Collectors.joining("\n"));
- /**
- * Set redis user and secret to connect to the redis server
- *
- * @return
- */
- private static JedisClientConfig getJedisClientConfig() {
- String redisSecret = "";
- String redisSecretEnv = System.getenv(Constants.REDIS_SECRET);
- if (redisSecretEnv != null) {
- redisSecret = redisSecretEnv;
+ } catch (IOException e) {
+ throw new IdempotencyConfigurationException("Unable to load lua script with name " + UPDATE_SCRIPT_LUA);
}
- return DefaultJedisClientConfig.builder()
- .user(System.getenv(Constants.REDIS_USER))
- .password(System.getenv(redisSecret))
- .build();
}
public static Builder builder() {
return new Builder();
}
- UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
- HostAndPort address = new HostAndPort(redisHost, redisPort);
- JedisClientConfig config = getJedisClientConfig();
- String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
- if ("true".equalsIgnoreCase(isClusterMode)) {
- return new JedisCluster(address, getJedisClientConfig(), 5, new GenericObjectPoolConfig<>());
- } else {
- return new JedisPooled(address, config);
- }
+ private static List getArgs(DataRecord dataRecord, Instant now) {
+ List args = new ArrayList<>();
+ args.add(String.valueOf(now.getEpochSecond()));
+ args.add(String.valueOf(now.toEpochMilli()));
+ args.add(INPROGRESS.toString());
+ args.add(String.valueOf(dataRecord.getExpiryTimestamp()));
+ args.add(dataRecord.getStatus().toString());
+ return args;
}
@Override
@@ -169,48 +167,39 @@ public void putRecord(DataRecord dataRecord, Instant now) {
}
}
- private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inProgressExpiry) {
- // if item with key exists
- String redisHashExistsExpression = "redis.call('exists', KEYS[1]) == 0";
- // if expiry timestamp is exceeded for existing item
- String itemExpiredExpression = "redis.call('hget', KEYS[1], KEYS[2]) < ARGV[1]";
- // if item status field exists and has value is INPROGRESS
- // and the in-progress-expiry timestamp is still valid
- String itemIsInProgressExpression = "(redis.call('hexists', KEYS[1], KEYS[4]) ~= 0" +
- " and redis.call('hget', KEYS[1], KEYS[4]) < ARGV[2]" +
- " and redis.call('hget', KEYS[1], KEYS[3]) == ARGV[3])";
-
- // insert item and fields
- String insertItemExpression = "return redis.call('hset', KEYS[1], KEYS[2], ARGV[4], KEYS[3], ARGV[5])";
-
- // only insert in-progress-expiry if it is set
- if (inProgressExpiry != null) {
- insertItemExpression = insertItemExpression.replace(")", ", KEYS[4], ARGV[6])");
+ UnifiedJedis getJedisClient(JedisConfig jedisConfig) {
+ HostAndPort address = new HostAndPort(jedisConfig.getHost(), jedisConfig.getPort());
+ JedisClientConfig config = jedisConfig.getJedisClientConfig();
+ String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
+ if ("true".equalsIgnoreCase(isClusterMode)) {
+ return new JedisCluster(address, config, 5, new GenericObjectPoolConfig<>());
+ } else {
+ return new JedisPooled(address, config);
}
+ }
- // if redisHashExistsExpression or itemExpiredExpression or itemIsInProgressExpression then insertItemExpression
- String luaScript = String.format("if %s or %s or %s then %s end;",
- redisHashExistsExpression, itemExpiredExpression,
- itemIsInProgressExpression, insertItemExpression);
+ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inProgressExpiry) {
+
+ List keys = getKeys(dataRecord);
+
+ List args = getArgs(dataRecord, now);
- List fields = new ArrayList<>();
- String hashKey = getKey(dataRecord.getIdempotencyKey());
- fields.add(hashKey);
- fields.add(prependField(hashKey, this.expiryAttr));
- fields.add(prependField(hashKey, this.statusAttr));
- fields.add(prependField(hashKey, this.inProgressExpiryAttr));
- fields.add(String.valueOf(now.getEpochSecond()));
- fields.add(String.valueOf(now.toEpochMilli()));
- fields.add(INPROGRESS.toString());
- fields.add(String.valueOf(dataRecord.getExpiryTimestamp()));
- fields.add(dataRecord.getStatus().toString());
if (inProgressExpiry != null) {
- fields.add(inProgressExpiry);
+ args.add(inProgressExpiry);
}
- String[] arr = new String[fields.size()];
- return jedisClient.eval(luaScript, 4, fields.toArray(arr));
+ return jedisClient.evalsha(jedisClient.scriptLoad(luaScript), keys, args);
+ }
+
+ private List getKeys(DataRecord dataRecord) {
+ List keys = new ArrayList<>();
+ String hashKey = getKey(dataRecord.getIdempotencyKey());
+ keys.add(hashKey);
+ keys.add(prependField(hashKey, this.expiryAttr));
+ keys.add(prependField(hashKey, this.statusAttr));
+ keys.add(prependField(hashKey, this.inProgressExpiryAttr));
+ return keys;
}
@Override
@@ -274,13 +263,14 @@ private String prependField(String hashKey, String field) {
*/
private DataRecord itemToRecord(Map item, String idempotencyKey) {
String hashKey = getKey(idempotencyKey);
+ String prependedInProgressExpiryAttr = item.get(prependField(hashKey, this.inProgressExpiryAttr));
return new DataRecord(item.get(getKey(idempotencyKey)),
DataRecord.Status.valueOf(item.get(prependField(hashKey, this.statusAttr))),
Long.parseLong(item.get(prependField(hashKey, this.expiryAttr))),
item.get(prependField(hashKey, this.dataAttr)),
item.get(prependField(hashKey, this.validationAttr)),
- item.get(prependField(hashKey, this.inProgressExpiryAttr)) != null ?
- OptionalLong.of(Long.parseLong(item.get(prependField(hashKey, this.inProgressExpiryAttr)))) :
+ prependedInProgressExpiryAttr != null && !prependedInProgressExpiryAttr.isEmpty() ?
+ OptionalLong.of(Long.parseLong(prependedInProgressExpiryAttr)) :
OptionalLong.empty());
}
@@ -290,6 +280,8 @@ private DataRecord itemToRecord(Map item, String idempotencyKey)
* You can also set a custom {@link UnifiedJedis} client.
*/
public static class Builder {
+
+ private JedisConfig jedisConfig = JedisConfig.Builder.builder().build();
private String keyPrefixName = "idempotency";
private String keyAttr = "id";
private String expiryAttr = "expiration";
@@ -309,7 +301,7 @@ public static class Builder {
* @return an instance of the {@link RedisPersistenceStore}
*/
public RedisPersistenceStore build() {
- return new RedisPersistenceStore(keyPrefixName, keyAttr, expiryAttr,
+ return new RedisPersistenceStore(jedisConfig, keyPrefixName, keyAttr, expiryAttr,
inProgressExpiryAttr, statusAttr, dataAttr, validationAttr, jedisClient);
}
@@ -403,5 +395,17 @@ public Builder withJedisClient(UnifiedJedis jedisClient) {
this.jedisClient = jedisClient;
return this;
}
+
+
+ /**
+ * Custom {@link JedisConfig} used to configure the Redis client(optional)
+ *
+ * @param jedisConfig
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withJedisConfig(JedisConfig jedisConfig) {
+ this.jedisConfig = jedisConfig;
+ return this;
+ }
}
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/resources/putRecordOnCondition.lua b/powertools-idempotency/powertools-idempotency-redis/src/main/resources/putRecordOnCondition.lua
new file mode 100644
index 000000000..cbfd01ba0
--- /dev/null
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/resources/putRecordOnCondition.lua
@@ -0,0 +1,19 @@
+local hashKey = KEYS[1]
+local expiryKey = KEYS[2]
+local statusKey = KEYS[3]
+local inProgressExpiryKey = KEYS[4]
+local timeNowSeconds = ARGV[1]
+local timeNowMillis = ARGV[2]
+local inProgressValue = ARGV[3]
+local expiryValue = ARGV[4]
+local statusValue = ARGV[5]
+local inProgressExpiryValue = ''
+
+if ARGV[6] ~= nil then inProgressExpiryValue = ARGV[6] end;
+
+if redis.call('exists', hashKey) == 0
+ or redis.call('hget', hashKey, expiryKey) < timeNowSeconds
+ or (redis.call('hexists', hashKey, inProgressExpiryKey) ~= 0
+ and redis.call('hget', hashKey, inProgressExpiryKey) < timeNowMillis
+ and redis.call('hget', hashKey, statusKey) == inProgressValue)
+then return redis.call('hset', hashKey, expiryKey, expiryValue, statusKey, statusValue, inProgressExpiryKey, inProgressExpiryValue) end;
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
index 09c493d78..99c10bfed 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
@@ -29,6 +29,7 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.SetEnvironmentVariable;
+import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPooled;
import redis.embedded.RedisServer;
@@ -37,19 +38,17 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
-@SetEnvironmentVariable(key = Constants.REDIS_HOST, value = "localhost")
-@SetEnvironmentVariable(key = Constants.REDIS_PORT, value = "6379")
+@SetEnvironmentVariable(key = "REDIS_HOST", value = "localhost")
+@SetEnvironmentVariable(key = "REDIS_PORT", value = "6379")
public class RedisPersistenceStoreTest {
static RedisServer redisServer;
- private final RedisPersistenceStore redisPersistenceStore = RedisPersistenceStore.builder().build();
private final JedisPooled jedisPool = new JedisPooled();
-
- public RedisPersistenceStoreTest() {
- }
+ private final RedisPersistenceStore redisPersistenceStore = RedisPersistenceStore.builder().build();
@BeforeAll
public static void init() {
- redisServer = new RedisServer(6379);
+
+ redisServer = RedisServer.builder().build();
redisServer.start();
}
@@ -74,6 +73,27 @@ void putRecord_shouldCreateItemInRedis() {
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
+ @Test
+ void putRecord_shouldCreateItemInRedisWithCustomJedisConfig() {
+
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder()
+ .withJedisClient(jedisPool).withJedisConfig(JedisConfig.Builder.builder().build())
+ .build();
+
+ redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
@Test
void putRecord_shouldCreateItemInRedisClusterMode() throws IOException {
com.github.fppt.jedismock.RedisServer redisCluster = com.github.fppt.jedismock.RedisServer
@@ -104,16 +124,28 @@ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
.newRedisServer()
.setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
.start();
- assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(),
- redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
+ JedisConfig jedisConfig = JedisConfig.Builder.builder()
+ .withHost(redisCluster.getHost())
+ .withPort(redisCluster.getBindPort())
+ .withJedisClientConfig(DefaultJedisClientConfig.builder()
+ .user("default")
+ .password("")
+ .ssl(false)
+ .build())
+ .build();
+ assertThat(redisPersistenceStore.getJedisClient(jedisConfig) instanceof JedisCluster).isTrue();
redisCluster.stop();
}
@SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
@Test
void putRecord_JedisClientInstanceOfJedisPooled() {
- assertThat(redisPersistenceStore.getJedisClient(System.getenv(Constants.REDIS_HOST),
- Integer.parseInt(System.getenv(Constants.REDIS_PORT))) instanceof JedisCluster).isFalse();
+ JedisConfig jedisConfig = JedisConfig.Builder.builder()
+ .withHost(System.getenv("REDIS_HOST"))
+ .withPort(Integer.parseInt(System.getenv("REDIS_PORT")))
+ .withJedisClientConfig(DefaultJedisClientConfig.builder().build())
+ .build();
+ assertThat(redisPersistenceStore.getJedisClient(jedisConfig) instanceof JedisCluster).isFalse();
}
@Test
From fb9106ba260b451275329218af03bf2f817815fb Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:24:08 +0200
Subject: [PATCH 15/19] Add database config hint in the documentation
---
docs/utilities/idempotency.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index be577a464..672718376 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -976,6 +976,7 @@ When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
.user("user")
.password("secret")
.ssl(true)
+ .database(1)
.connectionTimeoutMillis(3000)
.build())
.build();
@@ -998,6 +999,7 @@ When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
.user("default")
.password("")
.ssl(false)
+ .database(0)
.build();
```
From e3f4695eee3f9a3016b55f9cd9b0073a96ecd4a2 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 27 Dec 2023 16:09:11 +0200
Subject: [PATCH 16/19] Address spotbugs
---
.../persistence/redis/RedisPersistenceStore.java | 12 ++++++++++--
spotbugs-exclude.xml | 5 +++++
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
index bb47db142..60b83b212 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStore.java
@@ -19,6 +19,7 @@
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
@@ -95,14 +96,21 @@ private RedisPersistenceStore(JedisConfig jedisConfig,
this.jedisClient = null;
}
}
+
+ luaScript = getLuaScript();
+ }
+
+ private String getLuaScript() {
+ final String luaScript;
try (InputStreamReader luaScriptReader = new InputStreamReader(
- RedisPersistenceStore.class.getClassLoader().getResourceAsStream(UPDATE_SCRIPT_LUA))) {
+ RedisPersistenceStore.class.getClassLoader().getResourceAsStream(UPDATE_SCRIPT_LUA),
+ StandardCharsets.UTF_8)) {
luaScript = new BufferedReader(
luaScriptReader).lines().collect(Collectors.joining("\n"));
-
} catch (IOException e) {
throw new IdempotencyConfigurationException("Unable to load lua script with name " + UPDATE_SCRIPT_LUA);
}
+ return luaScript;
}
public static Builder builder() {
diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml
index 893298d2e..f2e0bdd1c 100644
--- a/spotbugs-exclude.xml
+++ b/spotbugs-exclude.xml
@@ -259,4 +259,9 @@
+
+
+
+
+
From 6659a4adb06bee55ace17ff829008165b37ed7ab Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 27 Dec 2023 17:22:35 +0200
Subject: [PATCH 17/19] Include memoryDB support in the documentation
---
docs/utilities/idempotency.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 672718376..058aceb22 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -221,7 +221,7 @@ Resources:
##### Redis resources
-You need an existing Redis service before setting up Redis as the persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/).
+You need an existing Redis service before setting up Redis as the persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) or [Amazon MemoryDB for Redis](https://aws.amazon.com/memorydb/) as persistent storage layer provider.
!!! tip "Tip:No existing Redis service?"
If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider. DynamoDB does not require a VPC deployment and is easier to configure and operate.
From b4feaa24cb74811e4b9fc020e4dfee335888e970 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Thu, 4 Jan 2024 09:54:22 +0200
Subject: [PATCH 18/19] Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com>
---
docs/utilities/idempotency.md | 12 +++++++-----
.../lambda/powertools/testutils/Infrastructure.java | 4 ++--
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 058aceb22..7271a78e6 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -222,6 +222,7 @@ Resources:
##### Redis resources
You need an existing Redis service before setting up Redis as the persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) or [Amazon MemoryDB for Redis](https://aws.amazon.com/memorydb/) as persistent storage layer provider.
+
!!! tip "Tip:No existing Redis service?"
If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider. DynamoDB does not require a VPC deployment and is easier to configure and operate.
@@ -244,11 +245,12 @@ Resources:
Variables:
REDIS_CLUSTER_MODE: "true"
```
+
##### VPC Access
Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
- If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+ If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also consult [this AWS tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
```yaml hl_lines="7-12" title="AWS Serverless Application Model (SAM) example"
Resources:
@@ -266,11 +268,11 @@ Resources:
```
1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
2. The security group ID or IDs of the VPC where the Redis deployment is configured.
-3. The subnet IDs of the VPC where the Redis deployment is configured.
+3. The subnet IDs of the VPC where the Redis deployment is configured.
### Idempotent annotation
-You can quickly start by initializing the persistence store used (e.g. the `DynamoDBPersistenceStore`) and using it with the `@Idempotent` annotation on your Lambda handler.
+You can quickly start by initializing the persistence store used (e.g. `DynamoDBPersistenceStore` or `RedisPersistenceStore`) and using it with the `@Idempotent` annotation on your Lambda handler.
!!! warning "Important"
Initialization and configuration of the persistence store must be performed outside the handler, preferably in the constructor.
@@ -974,14 +976,14 @@ When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
.withPort(redisCluster.getBindPort())
.withJedisClientConfig(DefaultJedisClientConfig.builder()
.user("user")
- .password("secret")
+ .password("secret") // leverage parameters-secrets module to retrieve this from Secrets Manager
.ssl(true)
.database(1)
.connectionTimeoutMillis(3000)
.build())
.build();
- JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), jedisConfig)
+ JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), jedisConfig);
Idempotency.config().withPersistenceStore(
RedisPersistenceStore.builder()
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index 0a0de2a1b..e4132e246 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -167,7 +167,7 @@ private Infrastructure(Builder builder) {
.build().getCallerIdentity().account();
if (isRedisDeployment) {
- this.vpc = Vpc.Builder.create(this.stack, "MyVPC-" + stackName)
+ this.vpc = Vpc.Builder.create(this.stack, "PowertoolsVPC-" + stackName)
.availabilityZones(Arrays.asList(region.toString() + "a", region + "b"))
.build();
@@ -180,7 +180,7 @@ private Infrastructure(Builder builder) {
.description("ElastiCache SecurityGroup")
.build();
- cfnSubnetGroup = CfnSubnetGroup.Builder.create(stack, "Redis Subnet-" + stackName)
+ cfnSubnetGroup = CfnSubnetGroup.Builder.create(stack, "Redis-Subnet-" + stackName)
.description("A subnet for the ElastiCache cluster")
.subnetIds(subnets).cacheSubnetGroupName("redis-SG-" + stackName).build();
From a63a8114abe65e1581d341fbaa37d0599140643e Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Mon, 22 Jan 2024 16:11:59 +0200
Subject: [PATCH 19/19] Minor enhancements
---
docs/utilities/idempotency.md | 14 +++++------
.../powertools/testutils/Infrastructure.java | 23 +++++++++++--------
.../redis/RedisPersistenceStoreTest.java | 6 +++--
3 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 7271a78e6..85e3015ec 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -232,7 +232,7 @@ In the following example, you can see a SAM template for deploying an AWS Lambda
!!! warning "Warning: Large responses with Redis persistence layer"
When using this utility with Redis your function's responses must be smaller than 512MB.
-Persisting larger items cannot might cause exceptions.
+Persisting larger items might cause exceptions.
```yaml hl_lines="9" title="AWS Serverless Application Model (SAM) example"
Resources:
@@ -267,8 +267,8 @@ Resources:
- subnet-{your_subnet_id_2}
```
1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
-2. The security group ID or IDs of the VPC where the Redis deployment is configured.
-3. The subnet IDs of the VPC where the Redis deployment is configured.
+2. The security group ID or IDs of the VPC where the Redis is deployed.
+3. The subnet IDs of the VPC where Redis is deployed.
### Idempotent annotation
@@ -972,8 +972,8 @@ When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
```java hl_lines="2-11 13 18"
public App() {
JedisConfig jedisConfig = JedisConfig.Builder.builder()
- .withHost(redisCluster.getHost())
- .withPort(redisCluster.getBindPort())
+ .withHost("redisHost")
+ .withPort("redisPort")
.withJedisClientConfig(DefaultJedisClientConfig.builder()
.user("user")
.password("secret") // leverage parameters-secrets module to retrieve this from Secrets Manager
@@ -998,8 +998,8 @@ When creating the `RedisPersistenceStore`, you can set a custom Jedis client:
```java
DefaultJedisClientConfig.builder()
- .user("default")
- .password("")
+ .user(null)
+ .password(null)
.ssl(false)
.database(0)
.build();
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index e4132e246..d086adf92 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -136,7 +136,7 @@ public class Infrastructure {
private String cfnAssetDirectory;
private SubnetSelection subnetSelection;
private CfnSubnetGroup cfnSubnetGroup;
- private SecurityGroup securityGroup;
+ private SecurityGroup redisSecurityGroup;
private boolean isRedisDeployment = false;
private Infrastructure(Builder builder) {
@@ -174,7 +174,7 @@ private Infrastructure(Builder builder) {
List subnets = vpc.getPublicSubnets().stream().map(subnet ->
subnet.getSubnetId()).collect(Collectors.toList());
- securityGroup = SecurityGroup.Builder.create(stack, "ElastiCache-SG-" + stackName)
+ redisSecurityGroup = SecurityGroup.Builder.create(stack, "ElastiCache-SG-" + stackName)
.vpc(vpc)
.allowAllOutbound(true)
.description("ElastiCache SecurityGroup")
@@ -284,13 +284,16 @@ private void createStackWithLambda() {
LOG.debug("Building Lambda function with command " +
packagingInstruction.stream().collect(Collectors.joining(" ", "[", "]")));
- final SecurityGroup lambdaSecurityGroup = SecurityGroup.Builder.create(this.stack, "Lambda-SG")
- .vpc(vpc)
- .allowAllOutbound(true)
- .description("Lambda SecurityGroup")
- .build();
- securityGroup.addIngressRule(Peer.securityGroupId(lambdaSecurityGroup.getSecurityGroupId()), Port.tcp(6379),
- "Allow ElastiCache Server");
+ if (isRedisDeployment) {
+ final SecurityGroup lambdaSecurityGroup = SecurityGroup.Builder.create(this.stack, "Lambda-SG")
+ .vpc(vpc)
+ .allowAllOutbound(true)
+ .description("Lambda SecurityGroup")
+ .build();
+ redisSecurityGroup.addIngressRule(Peer.securityGroupId(lambdaSecurityGroup.getSecurityGroupId()),
+ Port.tcp(6379),
+ "Allow ElastiCache Server");
+ }
Function.Builder functionBuilder = Function.Builder
.create(this.stack, functionName)
@@ -345,7 +348,7 @@ private void createStackWithLambda() {
.serverlessCacheName("rc-" + stackName)
.engine("redis")
.subnetIds(subnets)
- .securityGroupIds(singletonList(securityGroup.getSecurityGroupId()))
+ .securityGroupIds(singletonList(redisSecurityGroup.getSecurityGroupId()))
.build();
function.addEnvironment("REDIS_HOST", redisServer.getAtt("Endpoint.Address").toString());
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
index 99c10bfed..e65a58014 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/redis/RedisPersistenceStoreTest.java
@@ -30,6 +30,7 @@
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.SetEnvironmentVariable;
import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.JedisClientConfig;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPooled;
import redis.embedded.RedisServer;
@@ -80,10 +81,11 @@ void putRecord_shouldCreateItemInRedisWithCustomJedisConfig() {
long ttl = 3600;
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
RedisPersistenceStore store = new RedisPersistenceStore.Builder()
- .withJedisClient(jedisPool).withJedisConfig(JedisConfig.Builder.builder().build())
+ .withJedisConfig(JedisConfig.Builder.builder().withJedisClientConfig(
+ DefaultJedisClientConfig.builder().build()).build())
.build();
- redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+ store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
Map entry = jedisPool.hgetAll("{idempotency:id:key}");
long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");