Add support for concatenated plugin caches

log4j-core-test/src/test/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCacheTest.java

@@ -16,29 +16,23 @@
  */
 package org.apache.logging.log4j.core.config.plugins.processor;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import java.util.TreeMap;
+import org.junit.jupiter.api.Test;
 
-@RunWith(JUnit4.class)
-public class PluginCacheTest {
+class PluginCacheTest {
 
     @Test
-    public void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException {
-        final PluginCache cacheA = new PluginCache();
-        createCategory(cacheA, "one", Arrays.asList("bravo", "alpha", "charlie"));
-        createCategory(cacheA, "two", Arrays.asList("alpha", "charlie", "bravo"));
-        assertEquals(cacheA.getAllCategories().size(), 2);
-        assertEquals(cacheA.getAllCategories().get("one").size(), 3);
-        assertEquals(cacheA.getAllCategories().get("two").size(), 3);
+    void testOutputIsReproducibleWhenInputOrderingChanges() throws IOException {
+        final PluginCache cacheA = createSampleCache();
         final PluginCache cacheB = new PluginCache();
         createCategory(cacheB, "two", Arrays.asList("bravo", "alpha", "charlie"));
         createCategory(cacheB, "one", Arrays.asList("alpha", "charlie", "bravo"));
@@ -48,9 +42,51 @@ public void testOutputIsReproducibleWhenInputOrderingChanges() throws IOExceptio
         assertArrayEquals(cacheData(cacheA), cacheData(cacheB));
     }
 
-    private void createCategory(final PluginCache cache, final String categoryName, final List<String> entryNames) {
+    @Test
+    void testDeserialize() throws IOException {
+        final PluginCache expected = createSampleCache();
+
+        final Map<String, Map<String, PluginEntry>> actual = new TreeMap<>();
+        PluginCache.loadInputStream(new ByteArrayInputStream(cacheData(expected)), actual);
+
+        assertThat(actual).as("Deserialized plugin cache").isEqualTo(expected.getAllCategories());
+    }
+
+    @Test
+    void testConcatenationOfCaches() throws IOException {
+        final ByteArrayOutputStream output = new ByteArrayOutputStream();
+        boolean first = true;
+        for (final String categoryName : Arrays.asList("one", "two")) {
+            final PluginCache cache = new PluginCache();
+            createCategory(cache, categoryName, Arrays.asList("alpha", "bravo", "charlie"));
+            cache.writeCache(output);
+            if (first) {
+                output.write('\n');
+                first = false;
+            }
+        }
+
+        final Map<String, Map<String, PluginEntry>> actual = new TreeMap<>();
+        PluginCache.loadInputStream(new ByteArrayInputStream(output.toByteArray()), actual);
+
+        assertThat(actual)
+                .as("Deserialized plugin cache")
+                .isEqualTo(createSampleCache().getAllCategories());
+    }
+
+    private PluginCache createSampleCache() {
+        final PluginCache cacheA = new PluginCache();
+        createCategory(cacheA, "one", Arrays.asList("alpha", "bravo", "charlie"));
+        createCategory(cacheA, "two", Arrays.asList("alpha", "bravo", "charlie"));
+        assertEquals(cacheA.getAllCategories().size(), 2);
+        assertEquals(cacheA.getAllCategories().get("one").size(), 3);
+        assertEquals(cacheA.getAllCategories().get("two").size(), 3);
+        return cacheA;
+    }
+
+    private void createCategory(final PluginCache cache, final String categoryName, final Iterable<String> entryNames) {
         final Map<String, PluginEntry> category = cache.getCategory(categoryName);
-        for (String entryName : entryNames) {
+        for (final String entryName : entryNames) {
             final PluginEntry entry = new PluginEntry();
             entry.setKey(entryName);
             entry.setClassName("com.example.Plugin");

log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java

@@ -23,6 +23,7 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
 import java.util.Enumeration;
@@ -52,8 +53,12 @@ public Map<String, Map<String, PluginEntry>> getAllCategories() {
      * @return plugin mapping of names to plugin entries.
      */
     public Map<String, PluginEntry> getCategory(final String category) {
-        final String key = toRootLowerCase(category);
-        return categories.computeIfAbsent(key, ignored -> new TreeMap<>());
+        return getCategory(category, categories);
+    }
+
+    private static Map<String, PluginEntry> getCategory(
+            final String category, final Map<String, Map<String, PluginEntry>> categories) {
+        return categories.computeIfAbsent(toRootLowerCase(category), ignored -> new TreeMap<>());
     }
 
     /**
@@ -94,11 +99,26 @@ public void loadCacheFiles(final Enumeration<URL> resources) throws IOException
         categories.clear();
         while (resources.hasMoreElements()) {
             final URL url = resources.nextElement();
-            try (final DataInputStream in = new DataInputStream(new BufferedInputStream(url.openStream()))) {
+            loadInputStream(url.openStream(), categories);
+        }
+    }
+
+    static void loadInputStream(final InputStream input, final Map<String, Map<String, PluginEntry>> categories)
+            throws IOException {
+        try (final DataInputStream in = new DataInputStream(new BufferedInputStream(input))) {
+            // Allow concatenated plugin files separated by `\n`
+            boolean first = true;
+            while (in.available() > 0) {
+                if (first) {
+                    first = false;
+                } else {
+                    // Read the `\n` character
+                    in.readByte();
+                }
                 final int count = in.readInt();
                 for (int i = 0; i < count; i++) {
                     final String category = in.readUTF();
-                    final Map<String, PluginEntry> m = getCategory(category);
+                    final Map<String, PluginEntry> m = getCategory(category, categories);
                     final int entries = in.readInt();
                     for (int j = 0; j < entries; j++) {
                         // Must always read all parts of the entry, even if not adding, so that the stream progresses

log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginEntry.java

@@ -17,6 +17,7 @@
 package org.apache.logging.log4j.core.config.plugins.processor;
 
 import java.io.Serializable;
+import java.util.Objects;
 
 /**
  * Memento object for storing a plugin entry to a cache file.
@@ -79,6 +80,23 @@ public void setCategory(final String category) {
         this.category = category;
     }
 
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final PluginEntry that = (PluginEntry) o;
+        return printable == that.printable
+                && defer == that.defer
+                && Objects.equals(key, that.key)
+                && Objects.equals(className, that.className)
+                && Objects.equals(name, that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, className, name, printable, defer);
+    }
+
     @Override
     public String toString() {
         return "PluginEntry [key=" + key + ", className=" + className + ", name=" + name + ", printable=" + printable

log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/package-info.java

@@ -20,7 +20,7 @@
  * executable {@link org.apache.logging.log4j.core.config.plugins.util.PluginManager} class in your build process.
  */
 @Export
-@Version("2.20.1")
+@Version("2.20.2")
 package org.apache.logging.log4j.core.config.plugins.processor;
 
 import org.osgi.annotation.bundle.Export;

log4j-flume-ng/pom.xml

@@ -54,11 +54,6 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.flume</groupId>
-      <artifactId>flume-ng-core</artifactId>
-      <optional>true</optional>
-    </dependency>
     <dependency>
       <groupId>org.apache.flume</groupId>
       <artifactId>flume-ng-embedded-agent</artifactId>

src/changelog/.2.x.x/plugin-cache-additivity.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<entry xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns="https://logging.apache.org/xml/ns"
+       xsi:schemaLocation="https://logging.apache.org/xml/ns https://logging.apache.org/xml/ns/log4j-changelog-0.xsd"
+       type="added">
+  <description format="asciidoc">Add support for concatenated `Log4j2Plugins.dat` files.</description>
+</entry>

src/site/antora/modules/ROOT/pages/faq.adoc

@@ -313,17 +313,8 @@ https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html[`ServiceL
 You need to properly merge them by concatenating conflicting files.
 
 `META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat`::
-These files contain Log4j plugin descriptors and need to be properly merged using the appropriate resource transformer for your shading plugin:
-
-https://maven.apache.org/plugins/maven-assembly-plugin/[Maven Assembly Plugin]:::
-https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]:::
-We are not aware of any resource transformers capable of merging Log4j plugin descriptors.
-
-https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
-You need to use the
-https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer].
-
-https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
-You need to use the
-https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`].
+These files contain
+xref:manual/plugins.adoc[Log4j Plugin]
+descriptors.
+See xref:manual/plugins.adoc#plugin-merge[Merging plugin descriptors] for more details.
 ====

src/site/antora/modules/ROOT/pages/manual/plugins.adoc

@@ -281,3 +281,48 @@ See `@PluginElement("EventTemplateAdditionalField")` usage in {project-github-ur
 link:../javadoc/log4j-core/org/apache/logging/log4j/core/config/plugins/util/PluginUtil.html[`PluginUtil`],
 which is a convenient wrapper around <<#plugin-discovery,`PluginManager`>>, to discover and load plugins.
 See {project-github-url}/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/TemplateResolverFactories.java[`TemplateResolverFactories.java`] for example usages.
+
+[#plugin-merge]
+== Merging plugin descriptors
+
+The plugin descriptor is located in a JAR file at the following fixed location:
+----
+META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat
+----
+
+Some deployment techniques like
+https://softwareengineering.stackexchange.com/questions/297276/what-is-a-shaded-java-dependency[shading]
+require multiple plugin descriptors to be merged into one.
+This can be done in multiple ways:
+
+* By using a resource transformer specialized in `Log4j2Plugins.dat` files.
+We are aware of the existence of the following resource transformers for different Java build tools:
+
+https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
+You can use the
+https://logging.staged.apache.org/log4j/transform/log4j-transform-maven-shade-plugin-extensions.html#log4j-plugin-cache-transformer[Log4j Plugin Descriptor Transformer].
+
+https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
+You can use the
+https://github.com/GradleUp/shadow/blob/main/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.groovy[`Log4j2PluginsCacheFileTransformer`].
+
+* Since version `2.24.0` you can any resource transformer capable of concatenating files using `\n` as separator.
+The following resource transformers are known to exist:
+
+https://github.com/sbt/sbt-assembly[SBT Assembly Plugin]:::
+You can use `MergeStrategy.concat`.
+See
+https://github.com/sbt/sbt-assembly?tab=readme-ov-file#merge-strategy[SBT Assembly Merge Strategy]
+for details.
+
+https://maven.apache.org/plugins/maven-shade-plugin/[Maven Shade Plugin]:::
+You can use the standard `AppendingTransformer`.
+See
+https://maven.apache.org/plugins/maven-shade-plugin/examples/resource-transformers.html#AppendingTransformer[Maven Shade Plugin Resource Transformers]
+for details.
+
+https://gradleup.com/shadow/[Gradle Shadow Plugin]:::
+You can use the standard `AppendingTransformer`.
+See
+https://gradleup.com/shadow/configuration/merging/#appending-text-files[Appending Text Files]
+in the Gradle Shadow Plugin documentation for details.