Properly handle OOM error (#136)

Author: k-sareen

mmtk/Cargo.toml

@@ -20,7 +20,7 @@ lazy_static = "1.1"
 # - change branch
 # - change repo name
 # But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "29afb0c2fc8896db95e967062a892d39589bae4d" }
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "c38365be73e83dfd8a41e901d4537e87e0eb0ffe" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 

mmtk/src/collection.rs

@@ -1,5 +1,6 @@
 use mmtk::scheduler::WorkBucketStage;
 use mmtk::scheduler::{ProcessEdgesWork, ScanStackRoot};
+use mmtk::util::alloc::AllocationError;
 use mmtk::util::opaque_pointer::*;
 use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding};
 use mmtk::{Mutator, MutatorContext};
@@ -75,6 +76,12 @@ impl Collection<OpenJDK> for VMCollection {
         // unimplemented!()
     }
 
+    fn out_of_memory(tls: VMThread, err_kind: AllocationError) {
+        unsafe {
+            ((*UPCALLS).out_of_memory)(tls, err_kind);
+        }
+    }
+
     fn schedule_finalization(_tls: VMWorkerThread) {
         unsafe {
             ((*UPCALLS).schedule_finalizer)();

mmtk/src/lib.rs

@@ -13,6 +13,7 @@ extern crate lazy_static;
 use std::ptr::null_mut;
 
 use libc::{c_char, c_void, uintptr_t};
+use mmtk::util::alloc::AllocationError;
 use mmtk::util::opaque_pointer::*;
 use mmtk::util::{Address, ObjectReference};
 use mmtk::vm::VMBinding;
@@ -46,6 +47,7 @@ pub struct OpenJDK_Upcalls {
     pub resume_mutators: extern "C" fn(tls: VMWorkerThread),
     pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void),
     pub block_for_gc: extern "C" fn(),
+    pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError),
     pub get_next_mutator: extern "C" fn() -> *mut Mutator<OpenJDK>,
     pub reset_mutator_iterator: extern "C" fn(),
     pub compute_static_roots: extern "C" fn(trace: *mut c_void, tls: OpaquePointer),

openjdk/mmtk.h

@@ -12,6 +12,12 @@ extern "C" {
 typedef void* MMTk_Mutator;
 typedef void* MMTk_TraceLocal;
 
+// This has the same layout as mmtk::util::alloc::AllocationError
+typedef enum {
+    HeapOutOfMemory,
+    MmapOutOfMemory,
+} MMTkAllocationError;
+
 extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS;
@@ -102,6 +108,7 @@ typedef struct {
     void (*resume_mutators) (void *tls);
     void (*spawn_collector_thread) (void *tls, int kind, void *ctx);
     void (*block_for_gc) ();
+    void (*out_of_memory) (void *tls, MMTkAllocationError err_kind);
     void* (*get_next_mutator) ();
     void (*reset_mutator_iterator) ();
     void (*compute_static_roots) (void* trace, void* tls);

openjdk/mmtkHeap.cpp

@@ -114,7 +114,7 @@ jint MMTkHeap::initialize() {
 
   _companion_thread = new MMTkVMCompanionThread();
   if (!os::create_thread(_companion_thread, os::pgc_thread)) {
-    printf("Failed to create thread");
+    fprintf(stderr, "Failed to create thread");
     guarantee(false, "panic");
   }
   os::start_thread(_companion_thread);
@@ -503,9 +503,6 @@ void MMTkHeap::scan_roots(OopClosure& cl) {
 
 HeapWord* MMTkHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) {
   HeapWord* obj = Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize);
-  // post_alloc(Thread::current()->mmtk_mutator(), obj_ptr, NULL, size << LogHeapWordSize, 0);
-  // printf("offset: %ld\n", ((size_t) &Thread::current()->third_party_heap_mutator) - ((size_t) Thread::current()));
-  guarantee(obj, "MMTk gave us null!");
   return obj;
 }
 

openjdk/mmtkMutator.cpp

@@ -19,9 +19,11 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) {
 
   // FIXME: Proper use of slow-path api
   HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator);
-  // Post allococation. Currently we are only calling post_alloc in slowpath here.
-  // TODO: We also need to call them in the fastpath.
-  ::post_alloc((MMTk_Mutator) this, o, bytes, allocator);
+  // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM.
+  // Hence, only call post allocation hooks if we have a proper object.
+  if (o != nullptr) {
+    ::post_alloc((MMTk_Mutator) this, o, bytes, allocator);
+  }
   return o;
 }
 

openjdk/mmtkUpcalls.cpp

@@ -39,6 +39,7 @@
 #include "runtime/thread.hpp"
 #include "runtime/threadSMR.hpp"
 #include "runtime/vmThread.hpp"
+#include "utilities/debug.hpp"
 
 static size_t mmtk_start_the_world_count = 0;
 
@@ -131,6 +132,25 @@ static void mmtk_block_for_gc() {
   log_debug(gc)("Thread (id=%d) resumed after GC finished.", Thread::current()->osthread()->thread_id());
 }
 
+static void mmtk_out_of_memory(void* tls, MMTkAllocationError err_kind) {
+  switch (err_kind) {
+  case HeapOutOfMemory :
+    // Note that we have to do nothing for the case that the Java heap is too small. Since mmtk-core already
+    // returns a nullptr back to the JVM, it automatically triggers an OOM exception since the JVM checks for
+    // OOM every (slowpath) allocation [1]. In fact, if we report and throw an OOM exception here, the VM will
+    // complain since a pending exception bit was already set when it was trying to check for OOM [2]. Hence,
+    // it is best to let the JVM take care of reporting OOM itself.
+    //
+    // [1]: https://github.com/mmtk/openjdk/blob/e4dbe9909fa5c21685a20a1bc541fcc3b050dac4/src/hotspot/share/gc/shared/memAllocator.cpp#L83
+    // [2]: https://github.com/mmtk/openjdk/blob/e4dbe9909fa5c21685a20a1bc541fcc3b050dac4/src/hotspot/share/gc/shared/memAllocator.cpp#L117
+    break;
+  case MmapOutOfMemory :
+    // Abort the VM immediately due to insufficient system resources.
+    vm_exit_out_of_memory(0, OOM_MMAP_ERROR, "MMTk: Unable to acquire more memory from the OS. Out of system resources.");
+    break;
+  }
+}
+
 static void* mmtk_get_mmtk_mutator(void* tls) {
   return (void*) &((Thread*) tls)->third_party_heap_mutator;
 }
@@ -304,6 +324,7 @@ OpenJDK_Upcalls mmtk_upcalls = {
   mmtk_resume_mutators,
   mmtk_spawn_collector_thread,
   mmtk_block_for_gc,
+  mmtk_out_of_memory,
   mmtk_get_next_mutator,
   mmtk_reset_mutator_iterator,
   mmtk_compute_static_roots,