diff --git a/src/main/java/software/amazon/cloudformation/LambdaWrapper.java b/src/main/java/software/amazon/cloudformation/LambdaWrapper.java index 1d6b4a23..37319609 100644 --- a/src/main/java/software/amazon/cloudformation/LambdaWrapper.java +++ b/src/main/java/software/amazon/cloudformation/LambdaWrapper.java @@ -29,6 +29,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Date; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -231,6 +232,11 @@ public void handleRequest(final InputStream inputStream, final OutputStream outp } finally { // A response will be output on all paths, though CloudFormation will // not block on invoking the handlers, but rather listen for callbacks + + if (handlerResponse != null) { + publishExceptionCodeAndCountMetric(request == null ? null : request.getAction(), handlerResponse.getErrorCode(), + handlerResponse.getStatus() == OperationStatus.FAILED); + } writeResponse(outputStream, handlerResponse); } } @@ -471,6 +477,21 @@ private void publishExceptionMetric(final Action action, final Throwable ex, fin } } + /* + * null-safe exception metrics delivery + */ + private void + publishExceptionCodeAndCountMetric(final Action action, final HandlerErrorCode handlerErrorCode, final boolean thrown) { + if (this.metricsPublisherProxy != null) { + EnumSet.allOf(HandlerErrorCode.class).stream() + // publishing 0 value for all (if not thrown) otherwise filtered + .filter(errorCode -> errorCode.equals(handlerErrorCode) || !thrown) + .forEach(errorCode -> this.metricsPublisherProxy.publishExceptionByErrorCodeMetric(Instant.now(), action, + errorCode, thrown)); + this.metricsPublisherProxy.publishExceptionCountMetric(Instant.now(), action, thrown); + } + } + /** * null-safe logger redirect * diff --git a/src/main/java/software/amazon/cloudformation/metrics/Metric.java b/src/main/java/software/amazon/cloudformation/metrics/Metric.java index 10d2fdcc..80b4b14a 100644 --- a/src/main/java/software/amazon/cloudformation/metrics/Metric.java +++ b/src/main/java/software/amazon/cloudformation/metrics/Metric.java @@ -18,6 +18,8 @@ public class Metric { public static final String METRIC_NAMESPACE_ROOT = "AWS/CloudFormation"; public static final String METRIC_NAME_HANDLER_EXCEPTION = "HandlerException"; + public static final String METRIC_NAME_HANDLER_EXCEPTION_BY_ERROR_CODE = "HandlerExceptionByErrorCode"; + public static final String METRIC_NAME_HANDLER_EXCEPTION_BY_EXCEPTION_COUNT = "HandlerExceptionByExceptionCount"; public static final String METRIC_NAME_HANDLER_DURATION = "HandlerInvocationDuration"; public static final String METRIC_NAME_HANDLER_INVOCATION_COUNT = "HandlerInvocationCount"; diff --git a/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisher.java b/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisher.java index 9043c222..4df77784 100644 --- a/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisher.java +++ b/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisher.java @@ -42,6 +42,15 @@ public void publishExceptionMetric(final Instant timestamp, final HandlerErrorCode handlerErrorCode) { } + public void publishExceptionByErrorCodeMetric(final Instant timestamp, + final Action action, + final HandlerErrorCode handlerErrorCode, + final boolean thrown) { + } + + public void publishExceptionCountMetric(final Instant timestamp, final Action action, final boolean thrown) { + } + public void publishInvocationMetric(final Instant timestamp, final Action action) { } diff --git a/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisherImpl.java b/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisherImpl.java index 78e7c319..6ce52345 100644 --- a/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisherImpl.java +++ b/src/main/java/software/amazon/cloudformation/metrics/MetricsPublisherImpl.java @@ -66,6 +66,27 @@ public void publishExceptionMetric(final Instant timestamp, publishMetric(Metric.METRIC_NAME_HANDLER_EXCEPTION, dimensions, StandardUnit.COUNT, 1.0, timestamp); } + @Override + public void publishExceptionByErrorCodeMetric(final Instant timestamp, + final Action action, + final HandlerErrorCode handlerErrorCode, + final boolean thrown) { + Map dimensions = new HashMap<>(); + dimensions.put(Metric.DIMENSION_KEY_ACTION_TYPE, action == null ? "NO_ACTION" : action.name()); + dimensions.put(Metric.DIMENSION_KEY_HANDLER_ERROR_CODE, handlerErrorCode.name()); + + publishMetric(Metric.METRIC_NAME_HANDLER_EXCEPTION_BY_ERROR_CODE, dimensions, StandardUnit.COUNT, thrown ? 1.0 : 0.0, + timestamp); + } + + public void publishExceptionCountMetric(final Instant timestamp, final Action action, final boolean thrown) { + Map dimensions = new HashMap<>(); + dimensions.put(Metric.DIMENSION_KEY_ACTION_TYPE, action == null ? "NO_ACTION" : action.name()); + + publishMetric(Metric.METRIC_NAME_HANDLER_EXCEPTION_BY_EXCEPTION_COUNT, dimensions, StandardUnit.COUNT, thrown ? 1.0 : 0.0, + timestamp); + } + @Override public void publishProviderLogDeliveryExceptionMetric(final Instant timestamp, final Throwable e) { Map dimensions = new HashMap<>(); diff --git a/src/main/java/software/amazon/cloudformation/proxy/MetricsPublisherProxy.java b/src/main/java/software/amazon/cloudformation/proxy/MetricsPublisherProxy.java index 84e5bfc1..2255e2f0 100644 --- a/src/main/java/software/amazon/cloudformation/proxy/MetricsPublisherProxy.java +++ b/src/main/java/software/amazon/cloudformation/proxy/MetricsPublisherProxy.java @@ -35,6 +35,19 @@ public void publishExceptionMetric(final Instant timestamp, .forEach(metricsPublisher -> metricsPublisher.publishExceptionMetric(timestamp, action, e, handlerErrorCode)); } + public void publishExceptionByErrorCodeMetric(final Instant timestamp, + final Action action, + final HandlerErrorCode handlerErrorCode, + final boolean thrown) { + metricsPublishers.stream().forEach( + metricsPublisher -> metricsPublisher.publishExceptionByErrorCodeMetric(timestamp, action, handlerErrorCode, thrown)); + } + + public void publishExceptionCountMetric(final Instant timestamp, final Action action, final boolean thrown) { + metricsPublishers.stream() + .forEach(metricsPublisher -> metricsPublisher.publishExceptionCountMetric(timestamp, action, thrown)); + } + public void publishInvocationMetric(final Instant timestamp, final Action action) { metricsPublishers.stream().forEach(metricsPublisher -> metricsPublisher.publishInvocationMetric(timestamp, action)); } diff --git a/src/test/java/software/amazon/cloudformation/LambdaWrapperTest.java b/src/test/java/software/amazon/cloudformation/LambdaWrapperTest.java index 344691f3..b7141b9d 100644 --- a/src/test/java/software/amazon/cloudformation/LambdaWrapperTest.java +++ b/src/test/java/software/amazon/cloudformation/LambdaWrapperTest.java @@ -158,12 +158,15 @@ public void invokeHandler_nullResponse_returnsFailure(final String requestDataPa verifyInitialiseRuntime(); // validation failure metric should be published for final error handling - verify(providerMetricsPublisher, times(1)).publishExceptionMetric(any(Instant.class), any(), - any(TerminalException.class), any(HandlerErrorCode.class)); + verify(providerMetricsPublisher).publishExceptionMetric(any(Instant.class), any(), any(TerminalException.class), + any(HandlerErrorCode.class)); + verify(providerMetricsPublisher).publishExceptionByErrorCodeMetric(any(Instant.class), any(), + any(HandlerErrorCode.class), eq(Boolean.TRUE)); + verify(providerMetricsPublisher).publishExceptionCountMetric(any(Instant.class), any(), any(Boolean.class)); // all metrics should be published even on terminal failure - verify(providerMetricsPublisher, times(1)).publishInvocationMetric(any(Instant.class), eq(action)); - verify(providerMetricsPublisher, times(1)).publishDurationMetric(any(Instant.class), eq(action), anyLong()); + verify(providerMetricsPublisher).publishInvocationMetric(any(Instant.class), eq(action)); + verify(providerMetricsPublisher).publishDurationMetric(any(Instant.class), eq(action), anyLong()); // verify that model validation occurred for CREATE/UPDATE/DELETE if (action == Action.CREATE || action == Action.UPDATE || action == Action.DELETE) { @@ -399,14 +402,21 @@ public void invokeHandler_InProgress_returnsInProgress(final String requestDataP // verify output response verifyHandlerResponse(out, ProgressEvent.builder().status(OperationStatus.IN_PROGRESS) .resourceModel(TestModel.builder().property1("abc").property2(123).build()).build()); + verify(providerMetricsPublisher, atLeastOnce()).publishExceptionByErrorCodeMetric(any(Instant.class), eq(action), + any(), eq(Boolean.FALSE)); + verify(providerMetricsPublisher).publishExceptionCountMetric(any(Instant.class), eq(action), eq(Boolean.FALSE)); } else { verifyHandlerResponse(out, ProgressEvent.builder().status(OperationStatus.FAILED) .errorCode(HandlerErrorCode.InternalFailure).message("READ and LIST handlers must return synchronously.") .build()); - verify(providerMetricsPublisher, times(1)).publishExceptionMetric(any(Instant.class), eq(action), + verify(providerMetricsPublisher).publishExceptionMetric(any(Instant.class), eq(action), any(TerminalException.class), eq(HandlerErrorCode.InternalFailure)); + verify(providerMetricsPublisher).publishExceptionByErrorCodeMetric(any(Instant.class), eq(action), + eq(HandlerErrorCode.InternalFailure), eq(Boolean.TRUE)); + verify(providerMetricsPublisher).publishExceptionCountMetric(any(Instant.class), eq(action), eq(Boolean.TRUE)); } + // validation failure metric should not be published verifyNoMoreInteractions(providerMetricsPublisher); @@ -446,8 +456,11 @@ public void reInvokeHandler_InProgress_returnsInProgress(final String requestDat verifyInitialiseRuntime(); // all metrics should be published, once for a single invocation - verify(providerMetricsPublisher, times(1)).publishInvocationMetric(any(Instant.class), eq(action)); - verify(providerMetricsPublisher, times(1)).publishDurationMetric(any(Instant.class), eq(action), anyLong()); + verify(providerMetricsPublisher).publishInvocationMetric(any(Instant.class), eq(action)); + verify(providerMetricsPublisher).publishDurationMetric(any(Instant.class), eq(action), anyLong()); + verify(providerMetricsPublisher, atLeastOnce()).publishExceptionByErrorCodeMetric(any(Instant.class), eq(action), + any(), eq(Boolean.FALSE)); + verify(providerMetricsPublisher).publishExceptionCountMetric(any(Instant.class), eq(action), eq(Boolean.FALSE)); // validation failure metric should not be published verifyNoMoreInteractions(providerMetricsPublisher); @@ -798,6 +811,12 @@ public void invokeHandler_metricPublisherThrowable_returnsFailureResponse() thro // verify initialiseRuntime was called and initialised dependencies verifyInitialiseRuntime(); + verify(providerMetricsPublisher).publishExceptionByErrorCodeMetric(any(Instant.class), any(Action.class), + any(HandlerErrorCode.class), any(Boolean.class)); + + verify(providerMetricsPublisher).publishExceptionCountMetric(any(Instant.class), any(Action.class), + any(Boolean.class)); + // no further calls to metrics publisher should occur verifyNoMoreInteractions(providerMetricsPublisher);