diff --git a/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java b/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java index ffd89430c87..9fe8b8326cf 100644 --- a/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java +++ b/log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java @@ -16,29 +16,23 @@ */ package org.apache.logging.log4j.core.config.plugins.processor; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; -import java.util.List; import java.util.Map; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import java.util.TreeMap; +import org.junit.jupiter.api.Test; -@RunWith(JUnit4.class) -public class PluginCacheTest { +class PluginCacheTest { @Test - public void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException { - final PluginCache cacheA = new PluginCache(); - createCategory(cacheA, "one", Arrays.asList("bravo", "alpha", "charlie")); - createCategory(cacheA, "two", Arrays.asList("alpha", "charlie", "bravo")); - assertEquals(cacheA.getAllCategories().size(), 2); - assertEquals(cacheA.getAllCategories().get("one").size(), 3); - assertEquals(cacheA.getAllCategories().get("two").size(), 3); + void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException { + final PluginCache cacheA = createSampleCache(); final PluginCache cacheB = new PluginCache(); createCategory(cacheB, "two", Arrays.asList("bravo", "alpha", "charlie")); createCategory(cacheB, "one", Arrays.asList("alpha", "charlie", "bravo")); @@ -48,9 +42,51 @@ public void testOutputIsReproducibleWhenInputOrderingChanges() throws IOExceptio assertArrayEquals(cacheData(cacheA), cacheData(cacheB)); } - private void createCategory(final PluginCache cache, final String categoryName, final List entryNames) { + @Test + void testDeserialize() throws IOException { + final PluginCache expected = createSampleCache(); + + final Map> actual = new TreeMap<>(); + PluginCache.loadInputStream(new ByteArrayInputStream(cacheData(expected)), actual); + + assertThat(actual).as("Deserialized plugin cache").isEqualTo(expected.getAllCategories()); + } + + @Test + void testConcatenationOfCaches() throws IOException { + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + boolean first = true; + for (final String categoryName : Arrays.asList("one", "two")) { + final PluginCache cache = new PluginCache(); + createCategory(cache, categoryName, Arrays.asList("alpha", "bravo", "charlie")); + cache.writeCache(output); + if (first) { + output.write('\n'); + first = false; + } + } + + final Map> actual = new TreeMap<>(); + PluginCache.loadInputStream(new ByteArrayInputStream(output.toByteArray()), actual); + + assertThat(actual) + .as("Deserialized plugin cache") + .isEqualTo(createSampleCache().getAllCategories()); + } + + private PluginCache createSampleCache() { + final PluginCache cacheA = new PluginCache(); + createCategory(cacheA, "one", Arrays.asList("alpha", "bravo", "charlie")); + createCategory(cacheA, "two", Arrays.asList("alpha", "bravo", "charlie")); + assertEquals(cacheA.getAllCategories().size(), 2); + assertEquals(cacheA.getAllCategories().get("one").size(), 3); + assertEquals(cacheA.getAllCategories().get("two").size(), 3); + return cacheA; + } + + private void createCategory(final PluginCache cache, final String categoryName, final Iterable entryNames) { final Map category = cache.getCategory(categoryName); - for (String entryName : entryNames) { + for (final String entryName : entryNames) { final PluginEntry entry = new PluginEntry(); entry.setKey(entryName); entry.setClassName("com.example.Plugin"); diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java index ab3f84a4913..e1e72ceb86c 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java @@ -23,6 +23,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.URL; import java.util.Enumeration; @@ -52,8 +53,12 @@ public Map> getAllCategories() { * @return plugin mapping of names to plugin entries. */ public Map getCategory(final String category) { - final String key = toRootLowerCase(category); - return categories.computeIfAbsent(key, ignored -> new TreeMap<>()); + return getCategory(category, categories); + } + + private static Map getCategory( + final String category, final Map> categories) { + return categories.computeIfAbsent(toRootLowerCase(category), ignored -> new TreeMap<>()); } /** @@ -94,11 +99,26 @@ public void loadCacheFiles(final Enumeration resources) throws IOException categories.clear(); while (resources.hasMoreElements()) { final URL url = resources.nextElement(); - try (final DataInputStream in = new DataInputStream(new BufferedInputStream(url.openStream()))) { + loadInputStream(url.openStream(), categories); + } + } + + static void loadInputStream(final InputStream input, final Map> categories) + throws IOException { + try (final DataInputStream in = new DataInputStream(new BufferedInputStream(input))) { + // Allow concatenated plugin files separated by `\n` + boolean first = true; + while (in.available() > 0) { + if (first) { + first = false; + } else { + // Read the `\n` character + in.readByte(); + } final int count = in.readInt(); for (int i = 0; i < count; i++) { final String category = in.readUTF(); - final Map m = getCategory(category); + final Map m = getCategory(category, categories); final int entries = in.readInt(); for (int j = 0; j < entries; j++) { // Must always read all parts of the entry, even if not adding, so that the stream progresses diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java index 7f2b89d1a3c..3d148945aa9 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java @@ -17,6 +17,7 @@ package org.apache.logging.log4j.core.config.plugins.processor; import java.io.Serializable; +import java.util.Objects; /** * Memento object for storing a plugin entry to a cache file. @@ -79,6 +80,23 @@ public void setCategory(final String category) { this.category = category; } + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final PluginEntry that = (PluginEntry) o; + return printable == that.printable + && defer == that.defer + && Objects.equals(key, that.key) + && Objects.equals(className, that.className) + && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(key, className, name, printable, defer); + } + @Override public String toString() { return "PluginEntry [key=" + key + ", className=" + className + ", name=" + name + ", printable=" + printable diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/package-info.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/package-info.java index 1322b47dcdf..42617177fec 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/package-info.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/package-info.java @@ -20,7 +20,7 @@ * executable {@link org.apache.logging.log4j.core.config.plugins.util.PluginManager} class in your build process. */ @Export -@Version("2.20.1") +@Version("2.20.2") package org.apache.logging.log4j.core.config.plugins.processor; import org.osgi.annotation.bundle.Export; diff --git a/log4j-flume-ng/pom.xml b/log4j-flume-ng/pom.xml index 505f5969a68..d59e05a9006 100644 --- a/log4j-flume-ng/pom.xml +++ b/log4j-flume-ng/pom.xml @@ -54,11 +54,6 @@ org.apache.logging.log4j log4j-core - - org.apache.flume - flume-ng-core - true - org.apache.flume flume-ng-embedded-agent diff --git a/src/changelog/.2.x.x/plugin-cache-additivity.xml b/src/changelog/.2.x.x/plugin-cache-additivity.xml new file mode 100644 index 00000000000..f2daa46e94b --- /dev/null +++ b/src/changelog/.2.x.x/plugin-cache-additivity.xml @@ -0,0 +1,7 @@ + + + Add support for concatenated `Log4j2Plugins.dat` files. + diff --git a/src/site/antora/modules/ROOT/pages/faq.adoc b/src/site/antora/modules/ROOT/pages/faq.adoc index cb4fbd97e9f..917bc58020a 100644 --- a/src/site/antora/modules/ROOT/pages/faq.adoc +++ b/src/site/antora/modules/ROOT/pages/faq.adoc @@ -313,17 +313,8 @@ https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html[`ServiceL You need to properly merge them by concatenating conflicting files. `META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat`:: -These files contain Log4j plugin descriptors and need to be properly merged using the appropriate resource transformer for your shading plugin: - -https://maven.apache.org/plugins/maven-assembly-plugin/[Maven Assembly Plugin]::: -https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]::: -We are not aware of any resource transformers capable of merging Log4j plugin descriptors. - -https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]::: -You need to use the -https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer]. - -https://gradleup.com/shadow/[Gradle Shadow Plugin]::: -You need to use the -https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`]. +These files contain +xref:manual/plugins.adoc[Log4j Plugin] +descriptors. +See xref:manual/plugins.adoc#plugin-merge[Merging plugin descriptors] for more details. ==== diff --git a/src/site/antora/modules/ROOT/pages/manual/plugins.adoc b/src/site/antora/modules/ROOT/pages/manual/plugins.adoc index 1c33e97f32c..bdf17d8bf45 100644 --- a/src/site/antora/modules/ROOT/pages/manual/plugins.adoc +++ b/src/site/antora/modules/ROOT/pages/manual/plugins.adoc @@ -281,3 +281,48 @@ See `@PluginElement("EventTemplateAdditionalField")` usage in {project-github-ur link:../javadoc/log4j-core/org/apache/logging/log4j/core/config/plugins/util/PluginUtil.html[`PluginUtil`], which is a convenient wrapper around <<#plugin-discovery,`PluginManager`>>, to discover and load plugins. See {project-github-url}/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/TemplateResolverFactories.java[`TemplateResolverFactories.java`] for example usages. + +[#plugin-merge] +== Merging plugin descriptors + +The plugin descriptor is located in a JAR file at the following fixed location: +---- +META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat +---- + +Some deployment techniques like +https://softwareengineering.stackexchange.com/questions/297276/what-is-a-shaded-java-dependency[shading] +require multiple plugin descriptors to be merged into one. +This can be done in multiple ways: + +* By using a resource transformer specialized in `Log4j2Plugins.dat` files. +We are aware of the existence of the following resource transformers for different Java build tools: + +https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]::: +You can use the +https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer]. + +https://gradleup.com/shadow/[Gradle Shadow Plugin]::: +You can use the +https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`]. + +* Since version `2.24.0` you can any resource transformer capable of concatenating files using `\n` as separator. +The following resource transformers are known to exist: + +https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]::: +You can use `MergeStrategy.concat`. +See +https://github.com/sbt/sbt-assembly?tab=readme-ov-file#merge-strategy[SBT Assembly Merge Strategy] +for details. + +https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]::: +You can use the standard `AppendingTransformer`. +See +https://maven.apache.org/plugins/maven-shade-plugin/examples/resource-transformers.html#AppendingTransformer[Maven Shade Plugin Resource Transformers] +for details. + +https://gradleup.com/shadow/[Gradle Shadow Plugin]::: +You can use the standard `AppendingTransformer`. +See +https://gradleup.com/shadow/configuration/merging/#appending-text-files[Appending Text Files] +in the Gradle Shadow Plugin documentation for details. \ No newline at end of file