diff --git a/docs/core/logging.md b/docs/core/logging.md index 273b9ecb7..f35c88067 100644 --- a/docs/core/logging.md +++ b/docs/core/logging.md @@ -13,16 +13,23 @@ Logging provides an opinionated logger with output structured as JSON. ## Initialization -Powertools extends the functionality of Log4J. Below is an example `#!xml log4j2.xml` file, with the `#!java LambdaJsonLayout` configured. +Powertools extends the functionality of Log4J. Below is an example `#!xml log4j2.xml` file, with the `JsonTemplateLayout` using `#!json LambdaJsonLayout.json` configured. + +!!! info "LambdaJsonLayout is now deprecated" + + Configuring utiltiy using `` plugin is deprecated now. While utility still supports the old configuration, we strongly recommend upgrading the + `log4j2.xml` configuration to `JsonTemplateLayout` instead. [JsonTemplateLayout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html) is recommended way of doing structured logging. + + Please follow [this guide](#upgrade-to-jsontemplatelayout-from-deprecated-lambdajsonlayout-configuration-in-log4j2xml) for upgrade steps. === "log4j2.xml" ```xml hl_lines="5" - + - + @@ -123,6 +130,44 @@ to customise what is logged. } ``` +### Customising fields in logs + +- Utility by default emits `timestamp` field in the logs in format `yyyy-MM-dd'T'HH:mm:ss.SSSZz` and in system default timezone. +If you need to customize format and timezone, you can do so by configuring `log4j2.component.properties` and configuring properties as shown in example below: + +=== "log4j2.component.properties" + + ```properties hl_lines="1 2" + log4j.layout.jsonTemplate.timestampFormatPattern=yyyy-MM-dd'T'HH:mm:ss.SSSZz + log4j.layout.jsonTemplate.timeZone=Europe/Oslo + ``` + +- Utility also provides sample template for [Elastic Common Schema(ECS)](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) layout. +The field emitted in logs will follow specs from [ECS](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) together with field captured by utility as mentioned [above](#standard-structured-keys). + + Use `LambdaEcsLayout.json` as `eventTemplateUri` when configuring `JsonTemplateLayout`. + +=== "log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + ## Setting a Correlation ID You can set a Correlation ID using `correlationIdPath` attribute by passing a [JSON Pointer expression](https://datatracker.ietf.org/doc/html/draft-ietf-appsawg-json-pointer-03){target="_blank"}. @@ -417,3 +462,54 @@ via `samplingRate` attribute on annotation. Variables: POWERTOOLS_LOGGER_SAMPLE_RATE: 0.5 ``` + + +## Upgrade to JsonTemplateLayout from deprecated LambdaJsonLayout configuration in log4j2.xml + +Prior to version [1.10.0](https://github.com/awslabs/aws-lambda-powertools-java/releases/tag/v1.10.0), only supported way of configuring `log4j2.xml` was via ``. This plugin is +deprecated now and will be removed in future version. Switching to `JsonTemplateLayout` is straight forward. + +Below examples shows deprecated and new configuration of `log4j2.xml`. + +=== "Deprecated configuration of log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + +=== "New configuration of log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + diff --git a/pom.xml b/pom.xml index 16cc90f4f..99214bd43 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,11 @@ log4j-api ${log4j.version} + + org.apache.logging.log4j + log4j-layout-template-json + ${log4j.version} + org.apache.logging.log4j log4j-jcl diff --git a/powertools-logging/pom.xml b/powertools-logging/pom.xml index bc9214ca5..eaf0b1cfc 100644 --- a/powertools-logging/pom.xml +++ b/powertools-logging/pom.xml @@ -53,7 +53,10 @@ com.fasterxml.jackson.core jackson-databind - + + org.apache.logging.log4j + log4j-layout-template-json + org.apache.logging.log4j log4j-core diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java index b1fafe3b7..3ceda4b79 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java @@ -33,6 +33,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectWriter; +@Deprecated abstract class AbstractJacksonLayoutCopy extends AbstractStringLayout { protected static final String DEFAULT_EOL = "\r\n"; diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java index 67ec01058..41247cfdb 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java @@ -14,6 +14,7 @@ import java.util.HashSet; import java.util.Set; +@Deprecated abstract class JacksonFactoryCopy { static class JSON extends JacksonFactoryCopy { diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java index ad7ca3cf7..578937231 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java @@ -43,6 +43,10 @@ import static java.time.Instant.ofEpochMilli; import static java.time.format.DateTimeFormatter.ISO_ZONED_DATE_TIME; +/*** + * Note: The LambdaJsonLayout should be considered to be deprecated. Please use JsonTemplateLayout instead. + */ +@Deprecated @Plugin(name = "LambdaJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) public final class LambdaJsonLayout extends AbstractJacksonLayoutCopy { private static final String DEFAULT_FOOTER = "]"; diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java new file mode 100644 index 000000000..c392e2ed9 --- /dev/null +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java @@ -0,0 +1,51 @@ +package software.amazon.lambda.powertools.logging.internal; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolver; +import org.apache.logging.log4j.layout.template.json.util.JsonWriter; +import org.apache.logging.log4j.util.ReadOnlyStringMap; + +final class PowertoolsResolver implements EventResolver { + + private final EventResolver internalResolver; + + PowertoolsResolver() { + internalResolver = new EventResolver() { + @Override + public boolean isResolvable(LogEvent value) { + ReadOnlyStringMap contextData = value.getContextData(); + return null != contextData && !contextData.isEmpty(); + } + + @Override + public void resolve(LogEvent logEvent, JsonWriter jsonWriter) { + StringBuilder stringBuilder = jsonWriter.getStringBuilder(); + // remove dummy field to kick inn powertools resolver + stringBuilder.setLength(stringBuilder.length() - 4); + + // Inject all the context information. + ReadOnlyStringMap contextData = logEvent.getContextData(); + contextData.forEach((key, value) -> { + jsonWriter.writeSeparator(); + jsonWriter.writeString(key); + stringBuilder.append(':'); + jsonWriter.writeValue(value); + }); + } + }; + } + + static String getName() { + return "powertools"; + } + + @Override + public void resolve(LogEvent value, JsonWriter jsonWriter) { + internalResolver.resolve(value, jsonWriter); + } + + @Override + public boolean isResolvable(LogEvent value) { + return internalResolver.isResolvable(value); + } +} diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java new file mode 100644 index 000000000..5683c9688 --- /dev/null +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java @@ -0,0 +1,34 @@ +package software.amazon.lambda.powertools.logging.internal; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolverContext; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolverFactory; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolver; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolverConfig; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolverFactory; + +@Plugin(name = "PowertoolsResolverFactory", category = TemplateResolverFactory.CATEGORY) +public final class PowertoolsResolverFactory implements EventResolverFactory { + + private static final PowertoolsResolverFactory INSTANCE = new PowertoolsResolverFactory(); + + private PowertoolsResolverFactory() {} + + @PluginFactory + public static PowertoolsResolverFactory getInstance() { + return INSTANCE; + } + + @Override + public String getName() { + return PowertoolsResolver.getName(); + } + + @Override + public TemplateResolver create(EventResolverContext context, + TemplateResolverConfig config) { + return new PowertoolsResolver(); + } +} diff --git a/powertools-logging/src/main/resources/LambdaEcsLayout.json b/powertools-logging/src/main/resources/LambdaEcsLayout.json new file mode 100644 index 000000000..4ab9c7ce2 --- /dev/null +++ b/powertools-logging/src/main/resources/LambdaEcsLayout.json @@ -0,0 +1,52 @@ +{ + "@timestamp": { + "$resolver": "timestamp", + "pattern": { + "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "timeZone": "UTC" + } + }, + "ecs.version": "1.2.0", + "log.level": { + "$resolver": "level", + "field": "name" + }, + "message": { + "$resolver": "message", + "stringified": true + }, + "process.thread.name": { + "$resolver": "thread", + "field": "name" + }, + "log.logger": { + "$resolver": "logger", + "field": "name" + }, + "labels": { + "$resolver": "mdc", + "flatten": true, + "stringified": true + }, + "tags": { + "$resolver": "ndc" + }, + "error.type": { + "$resolver": "exception", + "field": "className" + }, + "error.message": { + "$resolver": "exception", + "field": "message" + }, + "error.stack_trace": { + "$resolver": "exception", + "field": "stackTrace", + "stackTrace": { + "stringified": true + } + }, + "": { + "$resolver": "powertools" + } +} \ No newline at end of file diff --git a/powertools-logging/src/main/resources/LambdaJsonLayout.json b/powertools-logging/src/main/resources/LambdaJsonLayout.json new file mode 100644 index 000000000..dfc1fc78f --- /dev/null +++ b/powertools-logging/src/main/resources/LambdaJsonLayout.json @@ -0,0 +1,89 @@ +{ + "timestamp": { + "$resolver": "timestamp" + }, + "instant": { + "epochSecond": { + "$resolver": "timestamp", + "epoch": { + "unit": "secs", + "rounded": true + } + }, + "nanoOfSecond": { + "$resolver": "timestamp", + "epoch": { + "unit": "secs.nanos" + } + } + }, + "thread": { + "$resolver": "thread", + "field": "name" + }, + "level": { + "$resolver": "level", + "field": "name" + }, + "loggerName": { + "$resolver": "logger", + "field": "name" + }, + "message": { + "$resolver": "message", + "stringified": true + }, + "thrown": { + "message": { + "$resolver": "exception", + "field": "message" + }, + "name": { + "$resolver": "exception", + "field": "className" + }, + "extendedStackTrace": { + "$resolver": "exception", + "field": "stackTrace" + } + }, + "contextStack": { + "$resolver": "ndc" + }, + "endOfBatch": { + "$resolver": "endOfBatch" + }, + "loggerFqcn": { + "$resolver": "logger", + "field": "fqcn" + }, + "threadId": { + "$resolver": "thread", + "field": "id" + }, + "threadPriority": { + "$resolver": "thread", + "field": "priority" + }, + "source": { + "class": { + "$resolver": "source", + "field": "className" + }, + "method": { + "$resolver": "source", + "field": "methodName" + }, + "file": { + "$resolver": "source", + "field": "fileName" + }, + "line": { + "$resolver": "source", + "field": "lineNumber" + } + }, + "": { + "$resolver": "powertools" + } +} diff --git a/powertools-logging/src/main/resources/log4j2.component.properties b/powertools-logging/src/main/resources/log4j2.component.properties new file mode 100644 index 000000000..3c392dd13 --- /dev/null +++ b/powertools-logging/src/main/resources/log4j2.component.properties @@ -0,0 +1,2 @@ +log4j.layout.jsonTemplate.timestampFormatPattern=yyyy-MM-dd'T'HH:mm:ss.SSSZz +#log4j.layout.jsonTemplate.timeZone= \ No newline at end of file diff --git a/powertools-logging/src/test/resources/log4j2.xml b/powertools-logging/src/test/resources/log4j2.xml index 108e32b75..22a44ee8b 100644 --- a/powertools-logging/src/test/resources/log4j2.xml +++ b/powertools-logging/src/test/resources/log4j2.xml @@ -1,8 +1,8 @@ - + - +