From 7b9112116eb0681f3ee61ed740ed0aa158801dbe Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Mon, 4 Mar 2024 13:07:14 -0500 Subject: [PATCH 1/2] add some support for physical memory usage and tests for every chunk periodic native events --- .../oracle/svm/core/heap/PhysicalMemory.java | 40 +++++++++ .../EveryChunkNativePeriodicEvents.java | 6 +- .../TestEveryChunkNativePeriodicEvents.java | 82 +++++++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java index 2f234bcd8548..a4f25481c438 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java @@ -24,9 +24,15 @@ */ package com.oracle.svm.core.heap; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; import java.util.concurrent.locks.ReentrantLock; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -39,6 +45,8 @@ import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; +import com.sun.management.OperatingSystemMXBean; + /** * Contains static methods to get configuration of physical memory. */ @@ -50,6 +58,8 @@ public interface PhysicalMemorySupport { UnsignedWord size(); } + private static final long K = 1024; + private static final ReentrantLock LOCK = new ReentrantLock(); private static final UnsignedWord UNSET_SENTINEL = UnsignedUtils.MAX_VALUE; private static UnsignedWord cachedSize = UNSET_SENTINEL; @@ -107,6 +117,36 @@ public static UnsignedWord size() { return cachedSize; } + /** + * Returns the amount of used physical memory in bytes, or -1 if not supported yet. + */ + public static long usedSize() { + // Containerized Linux, Windows and Mac OS X use the OS bean + if ((Containers.isContainerized() && Containers.memoryLimitInBytes() > 0) || + Platform.includedIn(Platform.WINDOWS.class) || + Platform.includedIn(Platform.MACOS.class)) { + OperatingSystemMXBean osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + return osBean.getTotalMemorySize() - osBean.getFreeMemorySize(); + } + // Non-containerized Linux uses MemAvailable from /proc/meminfo + if (Platform.includedIn(Platform.LINUX.class)) { + try { + List lines = Files.readAllLines(Paths.get("/proc/meminfo")); + for (String line : lines) { + if (!line.contains("MemAvailable")) { + continue; + } + String memAvailable = line.replaceAll("\\D", ""); + if (!memAvailable.isEmpty()) { + return size().rawValue() - Long.parseLong(memAvailable) * K; + } + } + } catch (IOException e) { + } + } + return -1; + } + /** * Returns the size of physical memory in bytes that has been previously cached. This method * must not be called if {@link #isInitialized()} is still false. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java index db0a82d9c853..254c7ef2f873 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java @@ -51,7 +51,7 @@ public class EveryChunkNativePeriodicEvents extends Event { public static void emit() { emitJavaThreadStats(); - emitPhysicalMemory(); + emitPhysicalMemory(PhysicalMemory.usedSize()); emitClassLoadingStatistics(); emitPerThreadEvents(); } @@ -75,7 +75,7 @@ private static void emitJavaThreadStats() { } @Uninterruptible(reason = "Accesses a JFR buffer.") - private static void emitPhysicalMemory() { + private static void emitPhysicalMemory(long usedSize) { if (JfrEvent.PhysicalMemory.shouldEmit()) { JfrNativeEventWriterData data = StackValue.get(JfrNativeEventWriterData.class); JfrNativeEventWriterDataAccess.initializeThreadLocalNativeBuffer(data); @@ -83,7 +83,7 @@ private static void emitPhysicalMemory() { JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.PhysicalMemory); JfrNativeEventWriter.putLong(data, JfrTicks.elapsedTicks()); JfrNativeEventWriter.putLong(data, PhysicalMemory.getCachedSize().rawValue()); - JfrNativeEventWriter.putLong(data, 0); /* used size */ + JfrNativeEventWriter.putLong(data, usedSize); /* used size */ JfrNativeEventWriter.endSmallEvent(data); } } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java new file mode 100644 index 000000000000..55ea74e70eea --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.test.jfr; + +import static org.junit.Assert.assertTrue; + +import com.oracle.svm.core.jfr.JfrEvent; + +import java.util.List; + +import jdk.jfr.Recording; +import jdk.jfr.consumer.RecordedEvent; +import org.junit.Test; + +/** + * This tests built in native JFR events that are sent periodically upon every chunk. The events + * ThreadCPULoad and ThreadAllocationStatistics are not tested here since they already have their + * own individual tests. + */ +public class TestEveryChunkNativePeriodicEvents extends JfrRecordingTest { + + @Test + public void test() throws Throwable { + String[] events = new String[]{ + JfrEvent.JavaThreadStatistics.getName(), + JfrEvent.PhysicalMemory.getName(), + JfrEvent.ClassLoadingStatistics.getName(), + }; + Recording recording = startRecording(events); + + stopRecording(recording, this::validateEvents); + } + + private void validateEvents(List events) { + boolean foundJavaThreadStatistics = false; + boolean foundPhysicalMemory = false; + boolean foundClassLoadingStatistics = false; + for (RecordedEvent e : events) { + if (e.getEventType().getName().equals(JfrEvent.JavaThreadStatistics.getName())) { + foundJavaThreadStatistics = true; + assertTrue(e.getLong("activeCount") > 1); + assertTrue(e.getLong("daemonCount") > 0); + assertTrue(e.getLong("accumulatedCount") > 1); + assertTrue(e.getLong("peakCount") > 1); + } else if (e.getEventType().getName().equals(JfrEvent.PhysicalMemory.getName())) { + foundPhysicalMemory = true; + assertTrue(e.getLong("totalSize") > 0); + assertTrue(e.getLong("usedSize") > 0 || e.getLong("usedSize") == -1); + + } else if (e.getEventType().getName().equals(JfrEvent.ClassLoadingStatistics.getName())) { + foundClassLoadingStatistics = true; + assertTrue(e.getLong("loadedClassCount") > 0); + } + } + assertTrue(foundJavaThreadStatistics && foundPhysicalMemory && foundClassLoadingStatistics); + } + +} From 1f19cf18944ea07795281feca40f153a9d553de5 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 27 Mar 2024 17:07:07 +0100 Subject: [PATCH 2/2] Cleanups. --- .../oracle/svm/core/heap/PhysicalMemory.java | 82 ++++++++++++++----- .../TestEveryChunkNativePeriodicEvents.java | 42 +++++----- 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java index a4f25481c438..bbb12df9e4a4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PhysicalMemory.java @@ -24,10 +24,12 @@ */ package com.oracle.svm.core.heap; +import java.io.BufferedReader; +import java.io.FileReader; import java.io.IOException; import java.lang.management.ManagementFactory; -import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.locks.ReentrantLock; @@ -44,7 +46,6 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; - import com.sun.management.OperatingSystemMXBean; /** @@ -117,33 +118,70 @@ public static UnsignedWord size() { return cachedSize; } - /** - * Returns the amount of used physical memory in bytes, or -1 if not supported yet. - */ + /** Returns the amount of used physical memory in bytes, or -1 if not supported. */ public static long usedSize() { - // Containerized Linux, Windows and Mac OS X use the OS bean - if ((Containers.isContainerized() && Containers.memoryLimitInBytes() > 0) || - Platform.includedIn(Platform.WINDOWS.class) || - Platform.includedIn(Platform.MACOS.class)) { + // Windows, macOS, and containerized Linux use the OS bean. + if (Platform.includedIn(Platform.WINDOWS.class) || + Platform.includedIn(Platform.MACOS.class) || + (Containers.isContainerized() && Containers.memoryLimitInBytes() > 0)) { OperatingSystemMXBean osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); return osBean.getTotalMemorySize() - osBean.getFreeMemorySize(); } - // Non-containerized Linux uses MemAvailable from /proc/meminfo + + // Non-containerized Linux uses /proc/meminfo. if (Platform.includedIn(Platform.LINUX.class)) { - try { - List lines = Files.readAllLines(Paths.get("/proc/meminfo")); - for (String line : lines) { - if (!line.contains("MemAvailable")) { - continue; - } - String memAvailable = line.replaceAll("\\D", ""); - if (!memAvailable.isEmpty()) { - return size().rawValue() - Long.parseLong(memAvailable) * K; - } + return getUsedSizeFromProcMemInfo(); + } + + return -1L; + } + + // Will be removed as part of GR-51479. + private static long getUsedSizeFromProcMemInfo() { + try { + List lines = readAllLines("/proc/meminfo"); + for (String line : lines) { + if (line.contains("MemAvailable")) { + return size().rawValue() - parseFirstNumber(line) * K; + } + } + } catch (Exception e) { + /* Nothing to do. */ + } + return -1L; + } + + private static List readAllLines(String fileName) throws IOException { + List lines = new ArrayList<>(); + try (BufferedReader bufferedReader = new BufferedReader(new FileReader(fileName, StandardCharsets.UTF_8))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + lines.add(line); + } + } + return lines; + } + + /** Parses the first number in the String as a long value. */ + private static long parseFirstNumber(String str) { + int firstDigit = -1; + int lastDigit = -1; + + for (int i = 0; i < str.length(); i++) { + if (Character.isDigit(str.charAt(i))) { + if (firstDigit == -1) { + firstDigit = i; } - } catch (IOException e) { + lastDigit = i; + } else if (firstDigit != -1) { + break; } } + + if (firstDigit >= 0) { + String number = str.substring(firstDigit, lastDigit + 1); + return Long.parseLong(number); + } return -1; } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java index 55ea74e70eea..4189c585fe47 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/jfr/TestEveryChunkNativePeriodicEvents.java @@ -28,55 +28,53 @@ import static org.junit.Assert.assertTrue; -import com.oracle.svm.core.jfr.JfrEvent; - import java.util.List; +import org.junit.Test; + +import com.oracle.svm.core.jfr.JfrEvent; + import jdk.jfr.Recording; import jdk.jfr.consumer.RecordedEvent; -import org.junit.Test; /** - * This tests built in native JFR events that are sent periodically upon every chunk. The events - * ThreadCPULoad and ThreadAllocationStatistics are not tested here since they already have their - * own individual tests. + * Tests the VM-level JFR events that are created periodically upon every chunk. Note that the + * events ThreadCPULoad and ThreadAllocationStatistics are not tested here since they already have + * their own individual tests. */ public class TestEveryChunkNativePeriodicEvents extends JfrRecordingTest { - @Test public void test() throws Throwable { - String[] events = new String[]{ - JfrEvent.JavaThreadStatistics.getName(), - JfrEvent.PhysicalMemory.getName(), - JfrEvent.ClassLoadingStatistics.getName(), - }; + String[] events = new String[]{JfrEvent.JavaThreadStatistics.getName(), JfrEvent.PhysicalMemory.getName(), JfrEvent.ClassLoadingStatistics.getName()}; Recording recording = startRecording(events); - - stopRecording(recording, this::validateEvents); + stopRecording(recording, TestEveryChunkNativePeriodicEvents::validateEvents); } - private void validateEvents(List events) { + private static void validateEvents(List events) { boolean foundJavaThreadStatistics = false; boolean foundPhysicalMemory = false; boolean foundClassLoadingStatistics = false; + for (RecordedEvent e : events) { - if (e.getEventType().getName().equals(JfrEvent.JavaThreadStatistics.getName())) { + String eventName = e.getEventType().getName(); + if (eventName.equals(JfrEvent.JavaThreadStatistics.getName())) { foundJavaThreadStatistics = true; assertTrue(e.getLong("activeCount") > 1); assertTrue(e.getLong("daemonCount") > 0); assertTrue(e.getLong("accumulatedCount") > 1); assertTrue(e.getLong("peakCount") > 1); - } else if (e.getEventType().getName().equals(JfrEvent.PhysicalMemory.getName())) { + } else if (eventName.equals(JfrEvent.PhysicalMemory.getName())) { foundPhysicalMemory = true; assertTrue(e.getLong("totalSize") > 0); - assertTrue(e.getLong("usedSize") > 0 || e.getLong("usedSize") == -1); - - } else if (e.getEventType().getName().equals(JfrEvent.ClassLoadingStatistics.getName())) { + assertTrue(e.getLong("usedSize") > 0); + } else if (eventName.equals(JfrEvent.ClassLoadingStatistics.getName())) { foundClassLoadingStatistics = true; assertTrue(e.getLong("loadedClassCount") > 0); } } - assertTrue(foundJavaThreadStatistics && foundPhysicalMemory && foundClassLoadingStatistics); - } + assertTrue(foundJavaThreadStatistics); + assertTrue(foundPhysicalMemory); + assertTrue(foundClassLoadingStatistics); + } }