From 880b13a6f14b1dba21602d757dbb2fa13042a141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 23 Feb 2023 17:13:56 +0100 Subject: [PATCH 01/16] Fix addImageClasspathEntry bundle substitution to substituteClassPath --- .../src/com/oracle/svm/driver/NativeImage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 2758516f392e..c76b91c4c7a9 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1791,7 +1791,7 @@ private void addImageClasspathEntry(LinkedHashSet destination, Path classp return; } - Path classpathEntryFinal = bundleSupport != null ? bundleSupport.substituteModulePath(classpathEntry) : classpathEntry; + Path classpathEntryFinal = bundleSupport != null ? bundleSupport.substituteClassPath(classpathEntry) : classpathEntry; if (!imageClasspath.contains(classpathEntryFinal) && !customImageClasspath.contains(classpathEntryFinal)) { destination.add(classpathEntryFinal); if (ClasspathUtils.isJar(classpathEntryFinal)) { From 88cd2cf3bfbb814f0e28bee23f05cfc5a603e1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 27 Feb 2023 20:14:49 +0100 Subject: [PATCH 02/16] Add support for passing environment vars to builder --- .../svm/driver/DefaultOptionHandler.java | 8 +++++ .../com/oracle/svm/driver/NativeImage.java | 36 +++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index a42d45764971..6d252fcf703c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -166,6 +166,14 @@ public boolean consume(ArgumentQueue args) { nativeImage.addOptionKeyValue(keyValue[0], keyValue[1]); return true; } + String envVarArgPrefix = "-E"; + if (headArg.startsWith(envVarArgPrefix)) { + args.poll(); + String envVarSetting = headArg.substring(envVarArgPrefix.length()); + String[] keyValue = envVarSetting.split("=", 2); + nativeImage.addImageBuilderEnvVar(keyValue[0], keyValue.length > 1 ? keyValue[1] : null); + return true; + } if (headArg.startsWith("-J")) { args.poll(); if (headArg.equals("-J")) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c76b91c4c7a9..f98dbc463e02 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -263,6 +263,7 @@ private static String oR(OptionKey option) { private static final String pKeyNativeImageArgs = "NativeImageArgs"; + private final Map imageBuilderEnvironment = new HashMap<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); private final LinkedHashSet imageBuilderClasspath = new LinkedHashSet<>(); @@ -473,7 +474,7 @@ public List getBuilderJavaArgs() { if (useJVMCINativeLibrary == null) { useJVMCINativeLibrary = false; ProcessBuilder pb = new ProcessBuilder(); - sanitizeJVMEnvironment(pb.environment()); + sanitizeJVMEnvironment(pb.environment(), Map.of()); List command = pb.command(); command.add(getJavaExecutable().toString()); command.add("-XX:+PrintFlagsFinal"); @@ -1469,11 +1470,11 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa try { ProcessBuilder pb = new ProcessBuilder(); pb.command(command); + sanitizeJVMEnvironment(pb.environment(), imageBuilderEnvironment); pb.environment().put(ModuleSupport.ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(config.modulePathBuild)); if (OS.getCurrent() == OS.WINDOWS) { WindowsBuildEnvironmentUtil.propagateEnv(pb.environment()); } - sanitizeJVMEnvironment(pb.environment()); p = pb.inheritIO().start(); imageBuilderPid = p.pid(); return p.waitFor(); @@ -1490,11 +1491,30 @@ boolean useBundle() { return bundleSupport != null; } - private static void sanitizeJVMEnvironment(Map environment) { - String[] jvmAffectingEnvironmentVariables = {"JAVA_COMPILER", "_JAVA_OPTIONS", "JAVA_TOOL_OPTIONS", "JDK_JAVA_OPTIONS", "CLASSPATH"}; - for (String affectingEnvironmentVariable : jvmAffectingEnvironmentVariables) { - environment.remove(affectingEnvironmentVariable); + private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { + Map restrictedEnvironment = new HashMap<>(); + String[] jvmRequiredEnvironmentVariables = {"PATH", "HOME", "PWD"}; + for (String requiredEnvironmentVariable : jvmRequiredEnvironmentVariables) { + String val = environment.get(requiredEnvironmentVariable); + if (val != null) { + restrictedEnvironment.put(requiredEnvironmentVariable, val); + } } + imageBuilderEnvironment.forEach((requiredKey, requiredValue) -> { + String prevValue = environment.get(requiredKey); + if (prevValue == null) { + if (requiredValue == null) { + NativeImage.showWarning("Environment variable '" + requiredKey + "' undefined. It will not be available at image build-time."); + } else { + restrictedEnvironment.put(requiredKey, requiredValue); + } + } else { + restrictedEnvironment.put(requiredKey, requiredValue == null ? prevValue : requiredValue); + } + }); + + environment.clear(); + environment.putAll(restrictedEnvironment); } private static final Function defaultNativeImageProvider = NativeImage::new; @@ -1629,6 +1649,10 @@ void addImageBuilderClasspath(Path classpath) { imageBuilderClasspath.add(canonicalize(classpath)); } + public void addImageBuilderEnvVar(String key, String value) { + imageBuilderEnvironment.put(key, value); + } + void addImageBuilderJavaArgs(String... javaArgs) { addImageBuilderJavaArgs(Arrays.asList(javaArgs)); } From 3bdbcf55c66f492504c526743cd5e1440d9bbab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 28 Feb 2023 18:15:25 +0100 Subject: [PATCH 03/16] Remove whitespace from json output --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index b07e65a8ed2d..88526db77f23 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -639,7 +639,7 @@ private static Manifest createManifest() { private static final String substitutionMapDstField = "dst"; private static void printPathMapping(Map.Entry entry, JsonWriter w) throws IOException { - w.append('{').quote(substitutionMapSrcField).append(" : ").quote(entry.getKey()); + w.append('{').quote(substitutionMapSrcField).append(':').quote(entry.getKey()); w.append(',').quote(substitutionMapDstField).append(':').quote(entry.getValue()); w.append('}'); } From 8cf6a3202483a04abb54caa2c6e5014cce0f87dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 28 Feb 2023 11:59:30 +0100 Subject: [PATCH 04/16] Support native-image environment variable capturing in bundles --- .../com/oracle/svm/driver/BundleSupport.java | 57 ++++++++++++++++- .../svm/driver/DefaultOptionHandler.java | 2 +- .../com/oracle/svm/driver/NativeImage.java | 62 +++++++++++-------- 3 files changed, 91 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 88526db77f23..10d3d00b74ae 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -279,13 +279,20 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } + Path environmentFile = stageDir.resolve("environment.json"); + try (Reader reader = Files.newBufferedReader(environmentFile)) { + new EnvironmentParser(nativeImage.imageBuilderEnvironment).parseAndRegister(reader); + } catch (IOException e) { + throw NativeImage.showError("Failed to read bundle-file " + environmentFile, e); + } + Path buildArgsFile = stageDir.resolve("build.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { List buildArgsFromFile = new ArrayList<>(); new BuildArgsParser(buildArgsFromFile).parseAndRegister(reader); buildArgs = Collections.unmodifiableList(buildArgsFromFile); } catch (IOException e) { - throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); + throw NativeImage.showError("Failed to read bundle-file " + buildArgsFile, e); } } @@ -567,6 +574,13 @@ private Path writeBundle() { } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } + Path environmentFile = stageDir.resolve("environment.json"); + try (JsonWriter writer = new JsonWriter(environmentFile)) { + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, nativeImage.imageBuilderEnvironment.entrySet(), Map.Entry.comparingByKey(), BundleSupport::printEnvironmentVariable); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); + } Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { @@ -599,7 +613,7 @@ private Path writeBundle() { /* Printing as list with defined sort-order ensures useful diffs are possible */ JsonPrinter.printCollection(writer, cleanBuildArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); + throw NativeImage.showError("Failed to write bundle-file " + buildArgsFile, e); } bundleProperties.write(); @@ -648,6 +662,18 @@ private static void printBuildArg(String entry, JsonWriter w) throws IOException w.quote(entry); } + private static final String environmentKeyField = "key"; + private static final String environmentValueField = "val"; + + private static void printEnvironmentVariable(Map.Entry entry, JsonWriter w) throws IOException { + if (entry.getValue() == null) { + throw NativeImage.showError("Storing environment variable '" + entry.getKey() + "' in bundle requires to have its value defined."); + } + w.append('{').quote(environmentKeyField).append(':').quote(entry.getKey()); + w.append(',').quote(environmentValueField).append(':').quote(entry.getValue()); + w.append('}'); + } + private static final class PathMapParser extends ConfigurationParser { private final Map pathMap; @@ -674,6 +700,33 @@ public void parseAndRegister(Object json, URI origin) { } } + private static final class EnvironmentParser extends ConfigurationParser { + + private final Map environment; + + private EnvironmentParser(Map environment) { + super(true); + environment.clear(); + this.environment = environment; + } + + @Override + public void parseAndRegister(Object json, URI origin) { + for (var rawEntry : asList(json, "Expected a list of environment variable objects")) { + var entry = asMap(rawEntry, "Expected a environment variable object"); + Object envVarKeyString = entry.get(environmentKeyField); + if (envVarKeyString == null) { + throw new JSONParserException("Expected " + environmentKeyField + "-field in environment variable object"); + } + Object envVarValueString = entry.get(environmentValueField); + if (envVarValueString == null) { + throw new JSONParserException("Expected " + environmentValueField + "-field in environment variable object"); + } + environment.put(envVarKeyString.toString(), envVarValueString.toString()); + } + } + } + private static final class BuildArgsParser extends ConfigurationParser { private final List args; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 6d252fcf703c..bf7464788301 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -171,7 +171,7 @@ public boolean consume(ArgumentQueue args) { args.poll(); String envVarSetting = headArg.substring(envVarArgPrefix.length()); String[] keyValue = envVarSetting.split("=", 2); - nativeImage.addImageBuilderEnvVar(keyValue[0], keyValue.length > 1 ? keyValue[1] : null); + nativeImage.imageBuilderEnvironment.put(keyValue[0], keyValue.length > 1 ? keyValue[1] : null); return true; } if (headArg.startsWith("-J")) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index f98dbc463e02..189dea2a9676 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.ListIterator; @@ -263,7 +264,7 @@ private static String oR(OptionKey option) { private static final String pKeyNativeImageArgs = "NativeImageArgs"; - private final Map imageBuilderEnvironment = new HashMap<>(); + final Map imageBuilderEnvironment = new HashMap<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); private final LinkedHashSet imageBuilderClasspath = new LinkedHashSet<>(); @@ -1446,12 +1447,24 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa /* Construct ProcessBuilder command from final arguments */ List command = new ArrayList<>(); - command.add(canonicalize(config.getJavaExecutable()).toString()); - List completeCommandList = new ArrayList<>(command); + String javaExecutable = canonicalize(config.getJavaExecutable()).toString(); + command.add(javaExecutable); command.add(createVMInvocationArgumentFile(arguments)); command.add(createImageBuilderArgumentFile(finalImageBuilderArgs)); - - completeCommandList.addAll(Stream.concat(arguments.stream(), finalImageBuilderArgs.stream()).collect(Collectors.toList())); + ProcessBuilder pb = new ProcessBuilder(); + pb.command(command); + Map environment = pb.environment(); + sanitizeJVMEnvironment(environment, imageBuilderEnvironment); + if (OS.WINDOWS.isCurrent()) { + WindowsBuildEnvironmentUtil.propagateEnv(environment); + } + environment.put(ModuleSupport.ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(config.modulePathBuild)); + + List completeCommandList = new ArrayList<>(); + completeCommandList.addAll(environment.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).sorted().toList()); + completeCommandList.add(javaExecutable); + completeCommandList.addAll(arguments); + completeCommandList.addAll(finalImageBuilderArgs); final String commandLine = SubstrateUtil.getShellCommandString(completeCommandList, true); if (isDiagnostics()) { // write to the diagnostics dir @@ -1468,13 +1481,6 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Process p = null; try { - ProcessBuilder pb = new ProcessBuilder(); - pb.command(command); - sanitizeJVMEnvironment(pb.environment(), imageBuilderEnvironment); - pb.environment().put(ModuleSupport.ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(config.modulePathBuild)); - if (OS.getCurrent() == OS.WINDOWS) { - WindowsBuildEnvironmentUtil.propagateEnv(pb.environment()); - } p = pb.inheritIO().start(); imageBuilderPid = p.pid(); return p.waitFor(); @@ -1493,26 +1499,32 @@ boolean useBundle() { private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { Map restrictedEnvironment = new HashMap<>(); - String[] jvmRequiredEnvironmentVariables = {"PATH", "HOME", "PWD"}; + List jvmRequiredEnvironmentVariables = new ArrayList<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); for (String requiredEnvironmentVariable : jvmRequiredEnvironmentVariables) { String val = environment.get(requiredEnvironmentVariable); if (val != null) { restrictedEnvironment.put(requiredEnvironmentVariable, val); } } - imageBuilderEnvironment.forEach((requiredKey, requiredValue) -> { - String prevValue = environment.get(requiredKey); - if (prevValue == null) { - if (requiredValue == null) { - NativeImage.showWarning("Environment variable '" + requiredKey + "' undefined. It will not be available at image build-time."); + for (Iterator> iterator = imageBuilderEnvironment.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String requiredKey = entry.getKey(); + String requiredValue = entry.getValue(); + if (requiredValue != null) { + restrictedEnvironment.put(requiredKey, requiredValue); + } else { + String existingValue = environment.get(requiredKey); + if (existingValue != null) { + restrictedEnvironment.put(requiredKey, existingValue); + /* Capture found existingValue for storing vars in bundle */ + entry.setValue(existingValue); } else { - restrictedEnvironment.put(requiredKey, requiredValue); + NativeImage.showWarning("Environment variable '" + requiredKey + "' is undefined and therefore not available during image build-time."); + /* Remove undefined environment for storing vars in bundle */ + iterator.remove(); } - } else { - restrictedEnvironment.put(requiredKey, requiredValue == null ? prevValue : requiredValue); } - }); - + } environment.clear(); environment.putAll(restrictedEnvironment); } @@ -1649,10 +1661,6 @@ void addImageBuilderClasspath(Path classpath) { imageBuilderClasspath.add(canonicalize(classpath)); } - public void addImageBuilderEnvVar(String key, String value) { - imageBuilderEnvironment.put(key, value); - } - void addImageBuilderJavaArgs(String... javaArgs) { addImageBuilderJavaArgs(Arrays.asList(javaArgs)); } From f847a1c76948c5514c138e01627ad215d67f72d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 11:09:02 +0100 Subject: [PATCH 05/16] INCLUDE and LIB needed in the env-var allowlist on Windows --- .../src/com/oracle/svm/driver/NativeImage.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 189dea2a9676..9e8ab3c0b647 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1500,6 +1500,9 @@ boolean useBundle() { private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { Map restrictedEnvironment = new HashMap<>(); List jvmRequiredEnvironmentVariables = new ArrayList<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); + if (OS.WINDOWS.isCurrent()) { + jvmRequiredEnvironmentVariables.addAll(List.of("INCLUDE", "LIB")); + } for (String requiredEnvironmentVariable : jvmRequiredEnvironmentVariables) { String val = environment.get(requiredEnvironmentVariable); if (val != null) { From 4517fdbf7a27303f2e5287a0b4ea803975e3edc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 11:41:26 +0100 Subject: [PATCH 06/16] Provide help text for -E[=] --- .../src/com.oracle.svm.driver/resources/HelpExtra.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index b460584fcac8..cef6eccb5565 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -34,5 +34,11 @@ Non-standard options help: creates a new bundle app_dbg.nib based on the given app.nib bundle. Both bundles are the same except the new one also uses the -g option. + -E[=] + allow native-image to access the given environment variable during + image build. If the optional is not given, the value + of the environment variable will be taken from the environment + native-image was invoked from. + -V= provide values for placeholders in native-image.properties files From 80db87f556a59f2bf64d84bf17e1548e3cedb3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 14:21:12 +0100 Subject: [PATCH 07/16] TEMP needed in the env-var allowlist on Windows --- .../src/com/oracle/svm/driver/NativeImage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 9e8ab3c0b647..115864d46dc8 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1501,7 +1501,7 @@ private static void sanitizeJVMEnvironment(Map environment, Map< Map restrictedEnvironment = new HashMap<>(); List jvmRequiredEnvironmentVariables = new ArrayList<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); if (OS.WINDOWS.isCurrent()) { - jvmRequiredEnvironmentVariables.addAll(List.of("INCLUDE", "LIB")); + jvmRequiredEnvironmentVariables.addAll(List.of("TEMP", "INCLUDE", "LIB")); } for (String requiredEnvironmentVariable : jvmRequiredEnvironmentVariables) { String val = environment.get(requiredEnvironmentVariable); From d1cfb357a8a5bf326838fa0085617552b6ed7a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 16:01:01 +0100 Subject: [PATCH 08/16] Ensure -E options are not added to build.json --- .../src/com/oracle/svm/driver/BundleSupport.java | 3 +++ .../src/com/oracle/svm/driver/DefaultOptionHandler.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 10d3d00b74ae..5af94b553d09 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -595,6 +595,9 @@ private Path writeBundle() { if (buildArg.startsWith(nativeImage.oHPath)) { continue; } + if (buildArg.startsWith(DefaultOptionHandler.envVarArgPrefix)) { + continue; + } if (buildArg.equals(CmdLineOptionHandler.VERBOSE_OPTION)) { continue; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index bf7464788301..2209d6a9094f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -45,6 +45,8 @@ class DefaultOptionHandler extends NativeImage.OptionHandler { static final String addModulesOption = "--add-modules"; private static final String addModulesErrorMessage = " requires modules to be specified"; + static final String envVarArgPrefix = "-E"; + /* Defunct legacy options that we have to accept to maintain backward compatibility */ private static final String noServerOption = "--no-server"; @@ -166,7 +168,6 @@ public boolean consume(ArgumentQueue args) { nativeImage.addOptionKeyValue(keyValue[0], keyValue[1]); return true; } - String envVarArgPrefix = "-E"; if (headArg.startsWith(envVarArgPrefix)) { args.poll(); String envVarSetting = headArg.substring(envVarArgPrefix.length()); From 061c41255924102b88b5ca8efe10a328119e56c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 16:19:45 +0100 Subject: [PATCH 09/16] Allow bundles to work with NATIVE_IMAGE_CONFIG_FILE use --- .../com/oracle/svm/driver/BundleSupport.java | 40 +++++++++---------- .../com/oracle/svm/driver/NativeImage.java | 35 ++++++++-------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 5af94b553d09..574f989b608b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -86,8 +86,8 @@ final class BundleSupport { Map pathCanonicalizations = new HashMap<>(); Map pathSubstitutions = new HashMap<>(); - private final List buildArgs; - private Collection updatedBuildArgs; + private final List nativeImageArgs; + private Collection updatedNativeImageArgs; boolean loadBundle; boolean writeBundle; @@ -110,9 +110,13 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; - private enum BundleOptionVariants { + enum BundleOptionVariants { create(), - apply() + apply(); + + String optionName() { + return BUNDLE_OPTION + "-" + this; + } } static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { @@ -121,10 +125,6 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma "Bundle support is still experimental and needs to be unlocked with '" + UNLOCK_BUNDLE_SUPPORT_OPTION + "'. The unlock option must precede '" + bundleArg + "'."); } - if (!nativeImage.userConfigProperties.isEmpty()) { - throw NativeImage.showError("Bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); - } - try { String variant = bundleArg.substring(BUNDLE_OPTION.length() + 1); String bundleFilename = null; @@ -133,31 +133,31 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma variant = variantParts[0]; bundleFilename = variantParts[1]; } - String applyOptionStr = BUNDLE_OPTION + "-" + BundleOptionVariants.apply; - String createOptionStr = BUNDLE_OPTION + "-" + BundleOptionVariants.create; + String applyOptionName = BundleOptionVariants.apply.optionName(); + String createOptionName = BundleOptionVariants.create.optionName(); BundleSupport bundleSupport; switch (BundleOptionVariants.valueOf(variant)) { case apply: if (nativeImage.useBundle()) { if (nativeImage.bundleSupport.loadBundle) { - throw NativeImage.showError(String.format("native-image allows option %s to be specified only once.", applyOptionStr)); + throw NativeImage.showError(String.format("native-image allows option %s to be specified only once.", applyOptionName)); } if (nativeImage.bundleSupport.writeBundle) { - throw NativeImage.showError(String.format("native-image option %s is not allowed to be used after option %s.", applyOptionStr, createOptionStr)); + throw NativeImage.showError(String.format("native-image option %s is not allowed to be used after option %s.", applyOptionName, createOptionName)); } } if (bundleFilename == null) { - throw NativeImage.showError(String.format("native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionStr, applyOptionStr)); + throw NativeImage.showError(String.format("native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionName, applyOptionName)); } bundleSupport = new BundleSupport(nativeImage, bundleFilename); /* Inject the command line args from the loaded bundle in-place */ - List buildArgs = bundleSupport.getBuildArgs(); + List buildArgs = bundleSupport.getNativeImageArgs(); for (int i = buildArgs.size() - 1; i >= 0; i--) { args.push(buildArgs.get(i)); } nativeImage.showVerboseMessage(nativeImage.isVerbose(), BUNDLE_INFO_MESSAGE_PREFIX + "Inject args: '" + String.join(" ", buildArgs) + "'"); /* Snapshot args after in-place expansion (includes also args after this one) */ - bundleSupport.updatedBuildArgs = args.snapshot(); + bundleSupport.updatedNativeImageArgs = args.snapshot(); break; case create: if (nativeImage.useBundle()) { @@ -209,7 +209,7 @@ private BundleSupport(NativeImage nativeImage) { } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } - this.buildArgs = Collections.unmodifiableList(nativeImage.config.getBuildArgs()); + this.nativeImageArgs = nativeImage.getNativeImageArgs(); } private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { @@ -290,14 +290,14 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { try (Reader reader = Files.newBufferedReader(buildArgsFile)) { List buildArgsFromFile = new ArrayList<>(); new BuildArgsParser(buildArgsFromFile).parseAndRegister(reader); - buildArgs = Collections.unmodifiableList(buildArgsFromFile); + nativeImageArgs = Collections.unmodifiableList(buildArgsFromFile); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + buildArgsFile, e); } } - public List getBuildArgs() { - return buildArgs; + public List getNativeImageArgs() { + return nativeImageArgs; } Path recordCanonicalization(Path before, Path after) { @@ -585,7 +585,7 @@ private Path writeBundle() { Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { ArrayList cleanBuildArgs = new ArrayList<>(); - for (String buildArg : updatedBuildArgs != null ? updatedBuildArgs : buildArgs) { + for (String buildArg : updatedNativeImageArgs != null ? updatedNativeImageArgs : nativeImageArgs) { if (buildArg.equals(UNLOCK_BUNDLE_SUPPORT_OPTION)) { continue; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 115864d46dc8..06c53f88a30f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -262,8 +262,6 @@ private static String oR(OptionKey option) { static final String oXmx = "-Xmx"; static final String oXms = "-Xms"; - private static final String pKeyNativeImageArgs = "NativeImageArgs"; - final Map imageBuilderEnvironment = new HashMap<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); @@ -785,17 +783,18 @@ protected void registerOptionHandler(OptionHandler handle optionHandlers.add(handler); } - protected Map getUserConfigProperties() { - return userConfigProperties; - } - - protected Path getUserConfigDir() { - String envVarKey = "NATIVE_IMAGE_USER_HOME"; - String userHomeStr = System.getenv(envVarKey); - if (userHomeStr == null || userHomeStr.isEmpty()) { - return Paths.get(System.getProperty("user.home"), ".native-image"); + private List getDefaultNativeImageArgs() { + String defaultNativeImageArgs = userConfigProperties.get("NativeImageArgs"); + if (defaultNativeImageArgs != null && !defaultNativeImageArgs.isEmpty()) { + String optionName = BundleSupport.BundleOptionVariants.apply.optionName(); + if (config.getBuildArgs().stream().noneMatch(arg -> arg.startsWith(optionName + "="))) { + return List.of(defaultNativeImageArgs.split(" ")); + } else { + showWarning(String.format("Option %s in use. Ignoring args from file specified with environment variable %s.", + optionName, NativeImage.CONFIG_FILE_ENV_VAR_KEY)); + } } - return Paths.get(userHomeStr); + return List.of(); } static void ensureDirectoryExists(Path dir) { @@ -2006,18 +2005,16 @@ protected static List getJars(Path dir, String... jarBaseNames) { private List processNativeImageArgs() { NativeImageArgsProcessor argsProcessor = new NativeImageArgsProcessor(null); - String defaultNativeImageArgs = getUserConfigProperties().get(pKeyNativeImageArgs); - if (defaultNativeImageArgs != null && !defaultNativeImageArgs.isEmpty()) { - for (String defaultArg : defaultNativeImageArgs.split(" ")) { - argsProcessor.accept(defaultArg); - } - } - for (String arg : config.getBuildArgs()) { + for (String arg : getNativeImageArgs()) { argsProcessor.accept(arg); } return argsProcessor.apply(false); } + List getNativeImageArgs() { + return Stream.concat(getDefaultNativeImageArgs().stream(), config.getBuildArgs().stream()).toList(); + } + protected String getXmsValue() { return "1g"; } From 15356fe9492e505e0116256765c501bbee49855f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Mar 2023 16:26:06 +0100 Subject: [PATCH 10/16] Provide workaround for SRCHOME env-var use. --- .../src/com/oracle/svm/driver/NativeImage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 06c53f88a30f..884d5601a136 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1499,6 +1499,7 @@ boolean useBundle() { private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { Map restrictedEnvironment = new HashMap<>(); List jvmRequiredEnvironmentVariables = new ArrayList<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); + jvmRequiredEnvironmentVariables.add("SRCHOME"); // FIXME if (OS.WINDOWS.isCurrent()) { jvmRequiredEnvironmentVariables.addAll(List.of("TEMP", "INCLUDE", "LIB")); } From b1fd08074567edaeef503106540143a689f93093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Mar 2023 11:53:14 +0100 Subject: [PATCH 11/16] Refactor bundleArgs sanitation --- .../com/oracle/svm/driver/BundleSupport.java | 45 +++++++------------ .../svm/driver/DefaultOptionHandler.java | 10 ++--- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 574f989b608b..6160b40ff6a0 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -40,11 +40,11 @@ import java.time.format.FormatStyle; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -87,7 +87,7 @@ final class BundleSupport { Map pathSubstitutions = new HashMap<>(); private final List nativeImageArgs; - private Collection updatedNativeImageArgs; + private List updatedNativeImageArgs; boolean loadBundle; boolean writeBundle; @@ -584,37 +584,24 @@ private Path writeBundle() { Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { - ArrayList cleanBuildArgs = new ArrayList<>(); - for (String buildArg : updatedNativeImageArgs != null ? updatedNativeImageArgs : nativeImageArgs) { - if (buildArg.equals(UNLOCK_BUNDLE_SUPPORT_OPTION)) { - continue; - } - if (buildArg.startsWith(BUNDLE_OPTION)) { - continue; - } - if (buildArg.startsWith(nativeImage.oHPath)) { - continue; - } - if (buildArg.startsWith(DefaultOptionHandler.envVarArgPrefix)) { - continue; - } - if (buildArg.equals(CmdLineOptionHandler.VERBOSE_OPTION)) { - continue; - } - if (buildArg.equals(CmdLineOptionHandler.DRY_RUN_OPTION)) { - continue; - } - if (buildArg.startsWith("-Dllvm.bin.dir=")) { - Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); - if (existing.isPresent() && !existing.get().equals(buildArg)) { - throw NativeImage.showError("Bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); + List equalsNonBundleOptions = List.of(UNLOCK_BUNDLE_SUPPORT_OPTION, CmdLineOptionHandler.VERBOSE_OPTION, CmdLineOptionHandler.DRY_RUN_OPTION); + List startsWithNonBundleOptions = List.of(BUNDLE_OPTION, DefaultOptionHandler.ADD_ENV_VAR_OPTION, nativeImage.oHPath); + ArrayList bundleArgs = new ArrayList<>(updatedNativeImageArgs != null ? updatedNativeImageArgs : nativeImageArgs); + ListIterator bundleArgsIterator = bundleArgs.listIterator(); + while (bundleArgsIterator.hasNext()) { + String arg = bundleArgsIterator.next(); + if (equalsNonBundleOptions.contains(arg) || startsWithNonBundleOptions.stream().anyMatch(arg::startsWith)) { + bundleArgsIterator.remove(); + } else if (arg.startsWith("-Dllvm.bin.dir=")) { + Optional existing = nativeImage.config.getBuildArgs().stream().filter(a -> a.startsWith("-Dllvm.bin.dir=")).findFirst(); + if (existing.isPresent() && !existing.get().equals(arg)) { + throw NativeImage.showError("Bundle native-image argument '" + arg + "' conflicts with existing '" + existing.get() + "'."); } - continue; + bundleArgsIterator.remove(); } - cleanBuildArgs.add(buildArg); } /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, cleanBuildArgs, null, BundleSupport::printBuildArg); + JsonPrinter.printCollection(writer, bundleArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + buildArgsFile, e); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 2209d6a9094f..55f28b38867b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -45,7 +45,7 @@ class DefaultOptionHandler extends NativeImage.OptionHandler { static final String addModulesOption = "--add-modules"; private static final String addModulesErrorMessage = " requires modules to be specified"; - static final String envVarArgPrefix = "-E"; + static final String ADD_ENV_VAR_OPTION = "-E"; /* Defunct legacy options that we have to accept to maintain backward compatibility */ private static final String noServerOption = "--no-server"; @@ -168,9 +168,9 @@ public boolean consume(ArgumentQueue args) { nativeImage.addOptionKeyValue(keyValue[0], keyValue[1]); return true; } - if (headArg.startsWith(envVarArgPrefix)) { + if (headArg.startsWith(ADD_ENV_VAR_OPTION)) { args.poll(); - String envVarSetting = headArg.substring(envVarArgPrefix.length()); + String envVarSetting = headArg.substring(ADD_ENV_VAR_OPTION.length()); String[] keyValue = envVarSetting.split("=", 2); nativeImage.imageBuilderEnvironment.put(keyValue[0], keyValue.length > 1 ? keyValue[1] : null); return true; @@ -200,7 +200,7 @@ public boolean consume(ArgumentQueue args) { Path origArgFile = Paths.get(headArg); Path argFile = nativeImage.bundleSupport != null ? nativeImage.bundleSupport.substituteAuxiliaryPath(origArgFile, BundleMember.Role.Input) : origArgFile; NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(OptionOrigin.argFilePrefix + argFile); - readArgFile(argFile).forEach(processor::accept); + readArgFile(argFile).forEach(processor); List leftoverArgs = processor.apply(false); if (leftoverArgs.size() > 0) { NativeImage.showError(String.format("Found unrecognized options while parsing argument file '%s':%n%s", argFile, String.join(System.lineSeparator(), leftoverArgs))); @@ -220,7 +220,7 @@ enum PARSER_STATE { IN_TOKEN } - class CTX_ARGS { + static class CTX_ARGS { PARSER_STATE state; int cptr; int eob; From e7bd51a917ac5c91d4bc76bac987e173a8d79db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Mar 2023 13:15:24 +0100 Subject: [PATCH 12/16] Add NATIVE_IMAGE_SLOPPY_BUILDER_SANITATION fallback --- .../svm/driver/DefaultOptionHandler.java | 2 +- .../com/oracle/svm/driver/NativeImage.java | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 55f28b38867b..b4f701b5fd62 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -230,7 +230,7 @@ static class CTX_ARGS { } // Ported from JDK11's java.base/share/native/libjli/args.c - private List readArgFile(Path file) { + private static List readArgFile(Path file) { List arguments = new ArrayList<>(); // Use of the at sign (@) to recursively interpret files isn't supported. arguments.add("--disable-@files"); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 884d5601a136..03538d7e5e20 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1453,7 +1453,20 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa ProcessBuilder pb = new ProcessBuilder(); pb.command(command); Map environment = pb.environment(); - sanitizeJVMEnvironment(environment, imageBuilderEnvironment); + String sloppySanitationKey = "NATIVE_IMAGE_SLOPPY_BUILDER_SANITATION"; + String sloppySanitationValue = System.getenv().getOrDefault(sloppySanitationKey, "false"); + if (Boolean.parseBoolean(sloppySanitationValue)) { + if (useBundle()) { + bundleSupport = null; + throw showError("Bundle support is not compatible with environment variable %s=%s.".formatted(sloppySanitationKey, sloppySanitationValue)); + } + if (!imageBuilderEnvironment.isEmpty()) { + throw showError("Option -E[=] is not compatible with environment variable %s=%s.".formatted(sloppySanitationKey, sloppySanitationValue)); + } + sloppySanitizeJVMEnvironment(environment); + } else { + sanitizeJVMEnvironment(environment, imageBuilderEnvironment); + } if (OS.WINDOWS.isCurrent()) { WindowsBuildEnvironmentUtil.propagateEnv(environment); } @@ -1496,6 +1509,14 @@ boolean useBundle() { return bundleSupport != null; } + @Deprecated + private static void sloppySanitizeJVMEnvironment(Map environment) { + String[] jvmAffectingEnvironmentVariables = {"JAVA_COMPILER", "_JAVA_OPTIONS", "JAVA_TOOL_OPTIONS", "JDK_JAVA_OPTIONS", "CLASSPATH"}; + for (String affectingEnvironmentVariable : jvmAffectingEnvironmentVariables) { + environment.remove(affectingEnvironmentVariable); + } + } + private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { Map restrictedEnvironment = new HashMap<>(); List jvmRequiredEnvironmentVariables = new ArrayList<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); From 59b91a86d303836cd600fa6dbbae43e82a50fa13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Mar 2023 13:23:45 +0100 Subject: [PATCH 13/16] Be explicit about the meaning of the values in imageBuilderEnvironment --- .../src/com/oracle/svm/driver/DefaultOptionHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index b4f701b5fd62..bba85d0dd5cb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -172,7 +172,8 @@ public boolean consume(ArgumentQueue args) { args.poll(); String envVarSetting = headArg.substring(ADD_ENV_VAR_OPTION.length()); String[] keyValue = envVarSetting.split("=", 2); - nativeImage.imageBuilderEnvironment.put(keyValue[0], keyValue.length > 1 ? keyValue[1] : null); + String valueDefinedOrInherited = keyValue.length > 1 ? keyValue[1] : null; + nativeImage.imageBuilderEnvironment.put(keyValue[0], valueDefinedOrInherited); return true; } if (headArg.startsWith("-J")) { From e2f60d2e26f9a203ebf9e96ffc1a20c194e58176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Mar 2023 14:58:16 +0100 Subject: [PATCH 14/16] Provide CHANGELOG entry --- substratevm/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 9adf1883eafd..90848b4cae5b 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -26,6 +26,7 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-40641) Dynamic linking of AWT libraries on Linux. * (GR-40463) Red Hat added experimental support for JMX, which can be enabled with the `--enable-monitoring` option (e.g. `--enable-monitoring=jmxclient,jmxserver`). * (GR-44110) Native Image now targets `x86-64-v3` by default on AMD64 and supports a new `-march` option. Use `-march=compatibility` for best compatibility (previous default) or `-march=native` for best performance if the native executable is deployed on the same machine or on a machine with the same CPU features. To list all available machine types, use `-march=list`. +* (GR-43971) Add native-image option `-E[=]` and support environment variable capturing in bundles. Previously almost all environment variables were available in the builder. To temporarily revert back to the old behaviour, env setting `NATIVE_IMAGE_SLOPPY_BUILDER_SANITATION=true` can be used. The old behaviour will be removed in a future release. ## Version 22.3.0 * (GR-35721) Remove old build output style and the `-H:±BuildOutputUseNewStyle` option. From ec609c5d352bf9a5264484bc1418aaa529ce5050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Mar 2023 18:19:39 +0100 Subject: [PATCH 15/16] Ensure bundle gets completed in case of image-build errors --- .../src/com/oracle/svm/driver/NativeImage.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 03538d7e5e20..4131809839c0 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1584,7 +1584,10 @@ private static void performBuild(BuildConfiguration config, Function Date: Thu, 2 Mar 2023 19:57:49 +0100 Subject: [PATCH 16/16] [GR-44027] Fix manifest Class-Path handling of relative paths in bundle-case --- .../com/oracle/svm/driver/BundleSupport.java | 14 ++++++++++++-- .../src/com/oracle/svm/driver/NativeImage.java | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 6160b40ff6a0..157370117e00 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -416,8 +416,8 @@ private Path substitutePath(Path origPath, Path destinationDir) { return origPath; } - // TODO Report error if overlapping dir-trees are passed in - // TODO add .endsWith(ClasspathUtils.cpWildcardSubstitute) handling (copy whole directory) + // TODO: Report error if overlapping dir-trees are passed in + String origFileName = origPath.getFileName().toString(); int extensionPos = origFileName.lastIndexOf('.'); String baseName; @@ -447,6 +447,16 @@ private Path substitutePath(Path origPath, Path destinationDir) { return substitutedPath; } + Path originalPath(Path substitutedPath) { + Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); + for (Map.Entry entry : pathSubstitutions.entrySet()) { + if (entry.getValue().equals(relativeSubstitutedPath)) { + return entry.getKey(); + } + } + return null; + } + private void copyFiles(Path source, Path target, boolean overwrite) { nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "> Copy files from " + source + " to " + target); if (Files.isDirectory(source)) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 4131809839c0..d679a25b08eb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -971,14 +971,27 @@ void handleClassPathAttribute(LinkedHashSet destination, Path jarFilePath, String classPathValue = mainAttributes.getValue("Class-Path"); /* Missing Class-Path Attribute is tolerable */ if (classPathValue != null) { + /* Cache expensive reverse lookup in bundle-case */ + Path origJarFilePath = null; for (String cp : classPathValue.split(" +")) { Path manifestClassPath = Path.of(cp); if (!manifestClassPath.isAbsolute()) { /* Resolve relative manifestClassPath against directory containing jar */ - manifestClassPath = jarFilePath.getParent().resolve(manifestClassPath); + Path relativeManifestClassPath = manifestClassPath; + manifestClassPath = jarFilePath.getParent().resolve(relativeManifestClassPath); + if (useBundle() && !Files.exists(manifestClassPath)) { + if (origJarFilePath == null) { + origJarFilePath = bundleSupport.originalPath(jarFilePath); + } + if (origJarFilePath == null) { + assert false : "Manifest Class-Path handling failed. No original path for " + jarFilePath + " available."; + break; + } + manifestClassPath = origJarFilePath.getParent().resolve(relativeManifestClassPath); + } } /* Invalid entries in Class-Path are allowed (i.e. use strict false) */ - addImageClasspathEntry(destination, manifestClassPath, false); + addImageClasspathEntry(destination, manifestClassPath.normalize(), false); } } }