Skip to content

Commit be47355

Browse files
mhalbrittermhalbritter
committed
Merge pull request #42158 from slissner
* pr/42158: Polish "Add Graylog Extended Log Format (GELF) for structured logging" Add Graylog Extended Log Format (GELF) for structured logging Closes gh-42158
2 parents 915fc2e + b5e7302 commit be47355

File tree

13 files changed

+799
-3
lines changed

13 files changed

+799
-3
lines changed

‎spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ Structured logging is a technique where the log output is written in a well-defi
445445
Spring Boot supports structured logging and has support for the following JSON formats out of the box:
446446

447447
* xref:#features.logging.structured.ecs[Elastic Common Schema (ECS)]
448+
* xref:#features.logging.structured.gelf[Graylog Extended Log Format (GELF)]
448449
* xref:#features.logging.structured.logstash[Logstash]
449450

450451
To enable structured logging, set the property configprop:logging.structured.format.console[] (for console output) or configprop:logging.structured.format.file[] (for file output) to the id of the format you want to use.
@@ -492,6 +493,51 @@ logging:
492493

493494
NOTE: configprop:logging.structured.ecs.service.name[] will default to configprop:spring.application.name[] if not specified.
494495

496+
NOTE: configprop:logging.structured.ecs.service.version[] will default to configprop:spring.application.version[] if not specified.
497+
498+
499+
500+
[[features.logging.structured.gelf]]
501+
=== Graylog Extended Log Format (GELF)
502+
https://go2docs.graylog.org/current/getting_in_log_data/gelf.html[Graylog Extended Log Format] is a JSON based logging format for the Graylog log analytics platform.
503+
504+
To enable the Graylog Extended Log Format, set the appropriate `format` property to `gelf`:
505+
506+
[configprops,yaml]
507+
----
508+
logging:
509+
structured:
510+
format:
511+
console: gelf
512+
file: gelf
513+
----
514+
515+
A log line looks like this:
516+
517+
[source,json]
518+
----
519+
{"version":"1.1","short_message":"No active profile set, falling back to 1 default profile: \"default\"","timestamp":1725958035.857,"level":6,"_level_name":"INFO","_process_pid":47649,"_process_thread_name":"main","_log_logger":"org.example.Application"}
520+
----
521+
522+
This format also adds every key value pair contained in the MDC to the JSON object.
523+
You can also use the https://www.slf4j.org/manual.html#fluent[SLF4J fluent logging API] to add key value pairs to the logged JSON object with the https://www.slf4j.org/apidocs/org/slf4j/spi/LoggingEventBuilder.html#addKeyValue(java.lang.String,java.lang.Object)[addKeyValue] method.
524+
525+
The `service` values can be customized using `logging.structured.gelf.service` properties:
526+
527+
[configprops,yaml]
528+
----
529+
logging:
530+
structured:
531+
gelf:
532+
service:
533+
name: MyService
534+
version: 1.0
535+
----
536+
537+
NOTE: configprop:logging.structured.gelf.service.name[] will default to configprop:spring.application.name[] if not specified.
538+
539+
NOTE: configprop:logging.structured.gelf.service.version[] will default to configprop:spring.application.version[] if not specified.
540+
495541

496542

497543
[[features.logging.structured.logstash]]

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/GraylogExtendedLogFormatStructuredLogFormatter.java

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* Copyright 2012-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.boot.logging.log4j2;
18+
19+
import java.math.BigDecimal;
20+
import java.util.Objects;
21+
import java.util.Set;
22+
import java.util.function.BiConsumer;
23+
import java.util.regex.Pattern;
24+
25+
import org.apache.commons.logging.Log;
26+
import org.apache.commons.logging.LogFactory;
27+
import org.apache.logging.log4j.Level;
28+
import org.apache.logging.log4j.core.LogEvent;
29+
import org.apache.logging.log4j.core.net.Severity;
30+
import org.apache.logging.log4j.core.time.Instant;
31+
import org.apache.logging.log4j.message.Message;
32+
import org.apache.logging.log4j.util.ReadOnlyStringMap;
33+
34+
import org.springframework.boot.json.JsonWriter;
35+
import org.springframework.boot.json.JsonWriter.WritableJson;
36+
import org.springframework.boot.logging.structured.CommonStructuredLogFormat;
37+
import org.springframework.boot.logging.structured.GraylogExtendedLogFormatService;
38+
import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter;
39+
import org.springframework.boot.logging.structured.StructuredLogFormatter;
40+
import org.springframework.core.env.Environment;
41+
import org.springframework.core.log.LogMessage;
42+
import org.springframework.util.Assert;
43+
import org.springframework.util.ObjectUtils;
44+
45+
/**
46+
* Log4j2 {@link StructuredLogFormatter} for
47+
* {@link CommonStructuredLogFormat#GRAYLOG_EXTENDED_LOG_FORMAT}. Supports GELF version
48+
* 1.1.
49+
*
50+
* @author Samuel Lissner
51+
* @author Moritz Halbritter
52+
*/
53+
class GraylogExtendedLogFormatStructuredLogFormatter extends JsonWriterStructuredLogFormatter<LogEvent> {
54+
55+
private static final Log logger = LogFactory.getLog(GraylogExtendedLogFormatStructuredLogFormatter.class);
56+
57+
/**
58+
* Allowed characters in field names are any word character (letter, number,
59+
* underscore), dashes and dots.
60+
*/
61+
private static final Pattern FIELD_NAME_VALID_PATTERN = Pattern.compile("^[\\w.\\-]*$");
62+
63+
/**
64+
* Every field been sent and prefixed with an underscore "_" will be treated as an
65+
* additional field.
66+
*/
67+
private static final String ADDITIONAL_FIELD_PREFIX = "_";
68+
69+
/**
70+
* Libraries SHOULD not allow to send id as additional field ("_id"). Graylog server
71+
* nodes omit this field automatically.
72+
*/
73+
private static final Set<String> ADDITIONAL_FIELD_ILLEGAL_KEYS = Set.of("id", "_id");
74+
75+
GraylogExtendedLogFormatStructuredLogFormatter(Environment environment) {
76+
super((members) -> jsonMembers(environment, members));
77+
}
78+
79+
private static void jsonMembers(Environment environment, JsonWriter.Members<LogEvent> members) {
80+
members.add("version", "1.1");
81+
// note: a blank message will lead to a Graylog error as of Graylog v6.0.x. We are
82+
// ignoring this here.
83+
members.add("short_message", LogEvent::getMessage).as(Message::getFormattedMessage);
84+
members.add("timestamp", LogEvent::getInstant)
85+
.as(GraylogExtendedLogFormatStructuredLogFormatter::formatTimeStamp);
86+
members.add("level", GraylogExtendedLogFormatStructuredLogFormatter::convertLevel);
87+
members.add("_level_name", LogEvent::getLevel).as(Level::name);
88+
members.add("_process_pid", environment.getProperty("spring.application.pid", Long.class))
89+
.when(Objects::nonNull);
90+
members.add("_process_thread_name", LogEvent::getThreadName);
91+
GraylogExtendedLogFormatService.get(environment).jsonMembers(members);
92+
members.add("_log_logger", LogEvent::getLoggerName);
93+
members.from(LogEvent::getContextData)
94+
.whenNot(ReadOnlyStringMap::isEmpty)
95+
.usingPairs((contextData, pairs) -> contextData
96+
.forEach((key, value) -> createAdditionalField(key, value, pairs)));
97+
members.add().whenNotNull(LogEvent::getThrownProxy).usingMembers((eventMembers) -> {
98+
eventMembers.add("full_message",
99+
GraylogExtendedLogFormatStructuredLogFormatter::formatFullMessageWithThrowable);
100+
eventMembers.add("_error_type", (event) -> event.getThrownProxy().getThrowable())
101+
.whenNotNull()
102+
.as(ObjectUtils::nullSafeClassName);
103+
eventMembers.add("_error_stack_trace", (event) -> event.getThrownProxy().getExtendedStackTraceAsString());
104+
eventMembers.add("_error_message", (event) -> event.getThrownProxy().getMessage());
105+
});
106+
}
107+
108+
/**
109+
* GELF requires "seconds since UNIX epoch with optional <b>decimal places for
110+
* milliseconds</b>". To comply with this requirement, we format a POSIX timestamp
111+
* with millisecond precision as e.g. "1725459730385" -> "1725459730.385"
112+
* @param timeStamp the timestamp of the log message. Note it is not the standard Java
113+
* `Instant` type but {@link org.apache.logging.log4j.core.time}
114+
* @return the timestamp formatted as string with millisecond precision
115+
*/
116+
private static WritableJson formatTimeStamp(Instant timeStamp) {
117+
return (out) -> out.append(new BigDecimal(timeStamp.getEpochMillisecond()).movePointLeft(3).toPlainString());
118+
}
119+
120+
/**
121+
* Converts the log4j2 event level to the Syslog event level code.
122+
* @param event the log event
123+
* @return an integer representing the syslog log level code
124+
* @see Severity class from Log4j2 which contains the conversion logic
125+
*/
126+
private static int convertLevel(LogEvent event) {
127+
return Severity.getSeverity(event.getLevel()).getCode();
128+
}
129+
130+
private static String formatFullMessageWithThrowable(LogEvent event) {
131+
return event.getMessage().getFormattedMessage() + "\n\n"
132+
+ event.getThrownProxy().getExtendedStackTraceAsString();
133+
}
134+
135+
private static void createAdditionalField(String fieldName, Object value, BiConsumer<Object, Object> pairs) {
136+
Assert.notNull(fieldName, "fieldName must not be null");
137+
if (!FIELD_NAME_VALID_PATTERN.matcher(fieldName).matches()) {
138+
logger.warn(LogMessage.format("'%s' is not a valid field name according to GELF standard", fieldName));
139+
return;
140+
}
141+
if (ADDITIONAL_FIELD_ILLEGAL_KEYS.contains(fieldName)) {
142+
logger.warn(LogMessage.format("'%s' is an illegal field name according to GELF standard", fieldName));
143+
return;
144+
}
145+
String key = (fieldName.startsWith(ADDITIONAL_FIELD_PREFIX)) ? fieldName : ADDITIONAL_FIELD_PREFIX + fieldName;
146+
pairs.accept(key, value);
147+
}
148+
149+
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ private void addCommonFormatters(CommonFormatters<LogEvent> commonFormatters) {
105105
commonFormatters.add(CommonStructuredLogFormat.ELASTIC_COMMON_SCHEMA,
106106
(instantiator) -> new ElasticCommonSchemaStructuredLogFormatter(
107107
instantiator.getArg(Environment.class)));
108+
commonFormatters.add(CommonStructuredLogFormat.GRAYLOG_EXTENDED_LOG_FORMAT,
109+
(instantiator) -> new GraylogExtendedLogFormatStructuredLogFormatter(
110+
instantiator.getArg(Environment.class)));
108111
commonFormatters.add(CommonStructuredLogFormat.LOGSTASH,
109112
(instantiator) -> new LogstashStructuredLogFormatter());
110113
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/GraylogExtendedLogFormatStructuredLogFormatter.java

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Copyright 2012-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.boot.logging.logback;
18+
19+
import java.math.BigDecimal;
20+
import java.util.Objects;
21+
import java.util.Set;
22+
import java.util.function.BiConsumer;
23+
import java.util.regex.Pattern;
24+
25+
import ch.qos.logback.classic.pattern.ThrowableProxyConverter;
26+
import ch.qos.logback.classic.spi.ILoggingEvent;
27+
import ch.qos.logback.classic.spi.IThrowableProxy;
28+
import ch.qos.logback.classic.util.LevelToSyslogSeverity;
29+
import org.apache.commons.logging.Log;
30+
import org.apache.commons.logging.LogFactory;
31+
32+
import org.springframework.boot.json.JsonWriter;
33+
import org.springframework.boot.json.JsonWriter.WritableJson;
34+
import org.springframework.boot.logging.structured.CommonStructuredLogFormat;
35+
import org.springframework.boot.logging.structured.GraylogExtendedLogFormatService;
36+
import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter;
37+
import org.springframework.boot.logging.structured.StructuredLogFormatter;
38+
import org.springframework.core.env.Environment;
39+
import org.springframework.core.log.LogMessage;
40+
import org.springframework.util.Assert;
41+
import org.springframework.util.CollectionUtils;
42+
43+
/**
44+
* Logback {@link StructuredLogFormatter} for
45+
* {@link CommonStructuredLogFormat#GRAYLOG_EXTENDED_LOG_FORMAT}. Supports GELF version
46+
* 1.1.
47+
*
48+
* @author Samuel Lissner
49+
* @author Moritz Halbritter
50+
*/
51+
class GraylogExtendedLogFormatStructuredLogFormatter extends JsonWriterStructuredLogFormatter<ILoggingEvent> {
52+
53+
private static final Log logger = LogFactory.getLog(GraylogExtendedLogFormatStructuredLogFormatter.class);
54+
55+
/**
56+
* Allowed characters in field names are any word character (letter, number,
57+
* underscore), dashes and dots.
58+
*/
59+
private static final Pattern FIELD_NAME_VALID_PATTERN = Pattern.compile("^[\\w.\\-]*$");
60+
61+
/**
62+
* Every field been sent and prefixed with an underscore "_" will be treated as an
63+
* additional field.
64+
*/
65+
private static final String ADDITIONAL_FIELD_PREFIX = "_";
66+
67+
/**
68+
* Libraries SHOULD not allow to send id as additional field ("_id"). Graylog server
69+
* nodes omit this field automatically.
70+
*/
71+
private static final Set<String> ADDITIONAL_FIELD_ILLEGAL_KEYS = Set.of("id", "_id");
72+
73+
GraylogExtendedLogFormatStructuredLogFormatter(Environment environment,
74+
ThrowableProxyConverter throwableProxyConverter) {
75+
super((members) -> jsonMembers(environment, throwableProxyConverter, members));
76+
}
77+
78+
private static void jsonMembers(Environment environment, ThrowableProxyConverter throwableProxyConverter,
79+
JsonWriter.Members<ILoggingEvent> members) {
80+
members.add("version", "1.1");
81+
// note: a blank message will lead to a Graylog error as of Graylog v6.0.x. We are
82+
// ignoring this here.
83+
members.add("short_message", ILoggingEvent::getFormattedMessage);
84+
members.add("timestamp", ILoggingEvent::getTimeStamp)
85+
.as(GraylogExtendedLogFormatStructuredLogFormatter::formatTimeStamp);
86+
members.add("level", LevelToSyslogSeverity::convert);
87+
members.add("_level_name", ILoggingEvent::getLevel);
88+
members.add("_process_pid", environment.getProperty("spring.application.pid", Long.class))
89+
.when(Objects::nonNull);
90+
members.add("_process_thread_name", ILoggingEvent::getThreadName);
91+
GraylogExtendedLogFormatService.get(environment).jsonMembers(members);
92+
members.add("_log_logger", ILoggingEvent::getLoggerName);
93+
members.from(ILoggingEvent::getMDCPropertyMap)
94+
.when((mdc) -> !CollectionUtils.isEmpty(mdc))
95+
.usingPairs((mdc, pairs) -> mdc.forEach((key, value) -> createAdditionalField(key, value, pairs)));
96+
members.from(ILoggingEvent::getKeyValuePairs)
97+
.when((keyValuePairs) -> !CollectionUtils.isEmpty(keyValuePairs))
98+
.usingPairs((keyValuePairs, pairs) -> keyValuePairs
99+
.forEach((keyValuePair) -> createAdditionalField(keyValuePair.key, keyValuePair.value, pairs)));
100+
members.add().whenNotNull(ILoggingEvent::getThrowableProxy).usingMembers((throwableMembers) -> {
101+
throwableMembers.add("full_message",
102+
(event) -> formatFullMessageWithThrowable(throwableProxyConverter, event));
103+
throwableMembers.add("_error_type", ILoggingEvent::getThrowableProxy).as(IThrowableProxy::getClassName);
104+
throwableMembers.add("_error_stack_trace", throwableProxyConverter::convert);
105+
throwableMembers.add("_error_message", ILoggingEvent::getThrowableProxy).as(IThrowableProxy::getMessage);
106+
});
107+
}
108+
109+
/**
110+
* GELF requires "seconds since UNIX epoch with optional <b>decimal places for
111+
* milliseconds</b>". To comply with this requirement, we format a POSIX timestamp
112+
* with millisecond precision as e.g. "1725459730385" -> "1725459730.385"
113+
* @param timeStamp the timestamp of the log message
114+
* @return the timestamp formatted as string with millisecond precision
115+
*/
116+
private static WritableJson formatTimeStamp(long timeStamp) {
117+
return (out) -> out.append(new BigDecimal(timeStamp).movePointLeft(3).toPlainString());
118+
}
119+
120+
private static String formatFullMessageWithThrowable(ThrowableProxyConverter throwableProxyConverter,
121+
ILoggingEvent event) {
122+
return event.getFormattedMessage() + "\n\n" + throwableProxyConverter.convert(event);
123+
}
124+
125+
private static void createAdditionalField(String key, Object value, BiConsumer<Object, Object> pairs) {
126+
Assert.notNull(key, "fieldName must not be null");
127+
if (!FIELD_NAME_VALID_PATTERN.matcher(key).matches()) {
128+
logger.warn(LogMessage.format("'%s' is not a valid field name according to GELF standard", key));
129+
return;
130+
}
131+
if (ADDITIONAL_FIELD_ILLEGAL_KEYS.contains(key)) {
132+
logger.warn(LogMessage.format("'%s' is an illegal field name according to GELF standard", key));
133+
return;
134+
}
135+
String keyWithPrefix = (key.startsWith(ADDITIONAL_FIELD_PREFIX)) ? key : ADDITIONAL_FIELD_PREFIX + key;
136+
pairs.accept(keyWithPrefix, value);
137+
}
138+
139+
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ private void addCommonFormatters(CommonFormatters<ILoggingEvent> commonFormatter
8282
commonFormatters.add(CommonStructuredLogFormat.ELASTIC_COMMON_SCHEMA,
8383
(instantiator) -> new ElasticCommonSchemaStructuredLogFormatter(instantiator.getArg(Environment.class),
8484
instantiator.getArg(ThrowableProxyConverter.class)));
85+
commonFormatters
86+
.add(CommonStructuredLogFormat.GRAYLOG_EXTENDED_LOG_FORMAT,
87+
(instantiator) -> new GraylogExtendedLogFormatStructuredLogFormatter(
88+
instantiator.getArg(Environment.class),
89+
instantiator.getArg(ThrowableProxyConverter.class)));
8590
commonFormatters.add(CommonStructuredLogFormat.LOGSTASH, (instantiator) -> new LogstashStructuredLogFormatter(
8691
instantiator.getArg(ThrowableProxyConverter.class)));
8792
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/CommonStructuredLogFormat.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ public enum CommonStructuredLogFormat {
3131
*/
3232
ELASTIC_COMMON_SCHEMA("ecs"),
3333

34+
/**
35+
* <a href="https://go2docs.graylog.org/current/getting_in_log_data/gelf.html">Graylog
36+
* Extended Log Format</a> (GELF) log format.
37+
*/
38+
GRAYLOG_EXTENDED_LOG_FORMAT("gelf"),
39+
3440
/**
3541
* The <a href=
3642
* "https://github.com/logfellow/logstash-logback-encoder?tab=readme-ov-file#standard-fields">Logstash</a>

0 commit comments

Comments
 (0)