Add benchmark for performance regression testing

.github/workflows/benchmark.yml

@@ -0,0 +1,46 @@
+name: Benchmarks
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+      - name: Setup JDK
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: 'maven'
+      - name: Build with Maven
+        run: |
+          mvn --batch-mode --update-snapshots verify -Dstyle.color=always -Dmaven.javadoc.skip=true -Pbenchmark
+      - name: Run benchmark
+        run: |
+          java -jar target/benchmarks.jar -wi 3 -i 3 -f 1 -rf json
+      - name: Store raw benchmark result as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-result
+          path: jmh-result.json
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: JSON Schema Validator Benchmark
+          tool: 'jmh'
+          output-file-path: jmh-result.json
+          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          comment-on-alert: false
+          max-items-in-chart: 50
+          summary-always: true

pom.xml

@@ -80,7 +80,6 @@
     <version.graaljs>21.3.10</version.graaljs> <!-- 22.x and above is not Java 8 compatible -->
     <version.hamcrest>3.0</version.hamcrest>
     <version.junit>5.11.3</version.junit>
-    <version.undertow>2.2.37.Final</version.undertow> <!-- 2.3.x and above is not Java 8 compatible -->
     <version.jacoco-maven-plugin>0.8.12</version.jacoco-maven-plugin>
     <version.maven-compiler-plugin>3.13.0</version.maven-compiler-plugin>
     <version.maven-bundle-plugin>5.1.9</version.maven-bundle-plugin>
@@ -92,6 +91,7 @@
     <version.nexus-staging-maven>1.7.0</version.nexus-staging-maven>
     <version.maven-gpg>3.2.7</version.maven-gpg>
 
+    <version.jmh>1.37</version.jmh>
     <argLine>-Duser.language=en -Duser.region=GB</argLine> <!-- For surefire -->
   </properties>
 
@@ -123,13 +123,6 @@
       <version>${version.jackson}</version>
     </dependency>
 
-    <dependency>
-      <groupId>io.undertow</groupId>
-      <artifactId>undertow-core</artifactId>
-      <version>${version.undertow}</version>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest</artifactId>
@@ -184,10 +177,21 @@
       <version>${version.slf4j}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+      <version>${version.jmh}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <version>${version.jmh}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
-
     <resources>
       <resource>
         <filtering>false</filtering>
@@ -475,7 +479,43 @@
         </plugins>
       </build>
     </profile>
-
+    <profile>
+      <id>benchmark</id>
+      <properties>
+        <skipTests>true</skipTests>
+      </properties>
+      <build>
+        <finalName>benchmarks</finalName>
+        <plugins>
+          <plugin>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <version>3.7.1</version>
+            <configuration>
+              <descriptors>
+                <descriptor>src/test/assembly/assembly.xml</descriptor>
+              </descriptors>
+            </configuration>
+            <executions>
+              <execution>
+                <id>make-assembly</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+                <configuration>
+                  <archive>
+                    <manifest>
+                      <mainClass>org.openjdk.jmh.Main</mainClass>
+                    </manifest>
+                  </archive>
+                  <appendAssemblyId>false</appendAssemblyId>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
 </project>

src/main/resources/draft/2020-12/meta/format-assertion

@@ -0,0 +1,11 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://json-schema.org/draft/2020-12/meta/format-assertion",
+    "$dynamicAnchor": "meta",
+
+    "title": "Format vocabulary meta-schema for assertion results",
+    "type": ["object", "boolean"],
+    "properties": {
+        "format": { "type": "string" }
+    }
+}

src/test/assembly/assembly.xml

@@ -0,0 +1,32 @@
+<assembly
+    xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+    <id>benchmark</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>test</scope>
+        </dependencySet>
+    </dependencySets>
+    <fileSets>
+        <fileSet>
+            <directory>${project.build.directory}/test-classes</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>**/*.class</include>
+                <include>**/*.json</include>
+                <include>**/*.properties</include>
+                <include>**/*.xml</include>
+                <include>**/META-INF/**</include>
+            </includes>
+            <useDefaultExcludes>true</useDefaultExcludes>
+        </fileSet>
+    </fileSets>
+</assembly>

src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java

@@ -19,6 +19,8 @@
 import com.networknt.schema.SpecVersion.VersionFlag;
 import com.networknt.schema.regex.JDKRegularExpressionFactory;
 import com.networknt.schema.regex.JoniRegularExpressionFactory;
+import com.networknt.schema.resource.InputStreamSource;
+import com.networknt.schema.resource.SchemaLoader;
 import com.networknt.schema.suite.TestCase;
 import com.networknt.schema.suite.TestSource;
 import com.networknt.schema.suite.TestSpec;
@@ -27,6 +29,8 @@
 import org.junit.jupiter.api.DynamicNode;
 import org.opentest4j.AssertionFailedError;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -44,17 +48,15 @@
 import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
 import static org.junit.jupiter.api.DynamicTest.dynamicTest;
 
-abstract class AbstractJsonSchemaTestSuite extends HTTPServiceSupport {
+abstract class AbstractJsonSchemaTestSuite {
 
     private static String toForwardSlashPath(Path file) {
         return file.toString().replace('\\', '/');
     }
 
     private static void executeTest(JsonSchema schema, TestSpec testSpec) {
         Set<ValidationMessage> errors = schema.validate(testSpec.getData(), OutputFormat.DEFAULT, (executionContext, validationContext) -> {
-            if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")
-                    || "ecmascript-regex.json"
-                            .equals(testSpec.getTestCase().getSource().getPath().getFileName().toString())) {
+            if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")) {
                 executionContext.getExecutionConfig().setFormatAssertionsEnabled(true);
             }
         });
@@ -180,14 +182,31 @@ private DynamicNode buildContainer(VersionFlag defaultVersion, TestCase testCase
 
     private JsonSchemaFactory buildValidatorFactory(VersionFlag defaultVersion, TestCase testCase) {
         if (testCase.isDisabled()) return null;
-
+        SchemaLoader schemaLoader = new SchemaLoader() {
+            @Override
+            public InputStreamSource getSchema(AbsoluteIri absoluteIri) {
+                String iri = absoluteIri.toString();
+                if (iri.startsWith("http://localhost:1234")) {
+                    return () -> {
+                        String path = iri.substring("http://localhost:1234".length());
+                        File file = new File("src/test/resources/remotes" + path);
+                        if (file.exists()) {
+                            return new FileInputStream("src/test/resources/remotes" + path);
+                        }
+                        return new FileInputStream("src/test/suite/remotes" + path);
+                    };
+                }
+                return null;
+            }
+        };
         VersionFlag specVersion = detectVersion(testCase.getSchema(), testCase.getSpecification(), defaultVersion, false);
         JsonSchemaFactory base = JsonSchemaFactory.getInstance(specVersion);
         return JsonSchemaFactory
                 .builder(base)
                 .schemaMappers(schemaMappers -> schemaMappers
                         .mapPrefix("https://", "http://")
                         .mapPrefix("http://json-schema.org", "resource:"))
+                .schemaLoaders(schemaLoaders -> schemaLoaders.add(schemaLoader))               
                 .build();
     }
 
@@ -241,6 +260,9 @@ private void executeAndReset(JsonSchema schema, TestSpec testSpec) {
         }
     }
 
+    protected void cleanup() {
+    }
+
     private List<Path> findTestCases(String basePath) {
         try (Stream<Path> paths = Files.walk(Paths.get(basePath))) {
             return paths

src/test/java/com/networknt/schema/Issue428Test.java

@@ -12,7 +12,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
-class Issue428Test extends HTTPServiceSupport {
+class Issue428Test {
     protected ObjectMapper mapper = new ObjectMapper();
     protected JsonSchemaFactory validatorFactory = JsonSchemaFactory
             .builder(JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4)).build();