Integrate ThirdPartyHeap into aarch64

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -343,7 +343,7 @@ class Address {
 
   enum mode { no_mode, base_plus_offset, pre, post, post_reg,
               base_plus_offset_reg, literal };
-
+  Register _obj_start = noreg;
   // Shift and extend for base reg + reg offset addressing
   class extend {
     int _option, _shift;
@@ -464,7 +464,7 @@ class Address {
     }
   }
 
-  Address(const Address& a) : _mode(a._mode) { copy_data(a); }
+  Address(const Address& a) : _obj_start(a._obj_start), _mode(a._mode) { copy_data(a); }
 
   // Verify the value is trivially destructible regardless of mode, so our
   // destructor can also be trivial, and so our assignment operator doesn't
@@ -474,6 +474,7 @@ class Address {
 
   Address& operator=(const Address& a) {
     _mode = a._mode;
+    _obj_start = a._obj_start;
     copy_data(a);
     return *this;
   }

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -174,7 +174,11 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
   if (UseTLAB) {
     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
   } else {
-    b(slow_case);
+    if (UseThirdPartyHeap) { 
+      eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
+    } else {
+      b(slow_case);
+    }
   }
 }
 

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp

@@ -47,7 +47,7 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                   Register src, Register dst, Register count, RegSet saved_regs) {}
   virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
-                                  Register start, Register count, Register tmp, RegSet saved_regs) {}
+                                  Register src, Register dst, Register count, Register tmp, RegSet saved_regs) {}
 
   virtual void copy_load_at(MacroAssembler* masm,
                             DecoratorSet decorators,
@@ -111,6 +111,17 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
     Label&   slow_case                 // continuation point if fast allocation fails
   );
 
+  virtual void eden_allocate(MacroAssembler* masm,
+    Register obj,                      // result: pointer to object after successful allocation
+    Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
+    int      con_size_in_bytes,        // object size in bytes if   known at compile time
+    Register t1,                       // temp register
+    Register t2,                       // temp register
+    Label&   slow_case                 // continuation point if fast allocation fails
+  ) {
+    assert(false, "To be overriden by MMTk");
+  }
+
   virtual void barrier_stubs_init() {}
 
   virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::stw_instruction_and_data_patch; }

src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp

@@ -37,10 +37,10 @@ void ModRefBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decorat
 }
 
 void ModRefBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
-                                                   Register start, Register count, Register tmp,
+                                                   Register src, Register dst, Register count, Register tmp,
                                                    RegSet saved_regs) {
   if (is_oop) {
-    gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+    gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp, saved_regs);
   }
 }
 

src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp

@@ -46,7 +46,7 @@ class ModRefBarrierSetAssembler: public BarrierSetAssembler {
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                   Register src, Register dst, Register count, RegSet saved_regs);
   virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
-                                  Register start, Register count, Register tmp, RegSet saved_regs);
+                                  Register src, Register dst, Register count, Register tmp, RegSet saved_regs);
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
 };

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -4782,6 +4782,16 @@ void MacroAssembler::tlab_allocate(Register obj,
   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 }
 
+void MacroAssembler::eden_allocate(Register obj,
+                                   Register var_size_in_bytes,
+                                  int con_size_in_bytes,
+                                   Register t1,
+                                   Register t2,
+                                   Label& slow_case) {
+  BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
+}
+
 void MacroAssembler::verify_tlab() {
 #ifdef ASSERT
   if (UseTLAB && VerifyOops) {

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -933,6 +933,15 @@ class MacroAssembler: public Assembler {
   );
   void verify_tlab();
 
+  void eden_allocate(
+    Register obj,                      // result: pointer to object after successful allocation
+    Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
+    int      con_size_in_bytes,        // object size in bytes if   known at compile time
+    Register t1,                       // temp register
+    Register t2,                       // temp register
+    Label&   slow_case                 // continuation point if fast allocation fails
+  );
+
   // interface method calling
   void lookup_interface_method(Register recv_klass,
                                Register intf_klass,

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -1514,7 +1514,7 @@ class StubGenerator: public StubCodeGenerator {
         verify_oop_array(size, d, count, r16);
     }
 
-    bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
+    bs->arraycopy_epilogue(_masm, decorators, is_oop, s, d, count, rscratch1, RegSet());
 
     __ leave();
     __ mov(r0, zr); // return 0
@@ -1583,7 +1583,7 @@ class StubGenerator: public StubCodeGenerator {
       if (VerifyOops)
         verify_oop_array(size, d, count, r16);
     }
-    bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
+    bs->arraycopy_epilogue(_masm, decorators, is_oop, s, d, count, rscratch1, RegSet());
     __ leave();
     __ mov(r0, zr); // return 0
     __ ret(lr);
@@ -1865,6 +1865,7 @@ class StubGenerator: public StubCodeGenerator {
     const Register copied_oop  = r22;       // actual oop copied
     const Register count_save  = r21;       // orig elementscount
     const Register start_to    = r20;       // destination array start address
+    const Register start_from  = r23;       // source array start address
     const Register r19_klass   = r19;       // oop._klass
 
     // Registers used as gc temps (r5, r6, r7 are save-on-call)
@@ -1904,7 +1905,7 @@ class StubGenerator: public StubCodeGenerator {
 
      // Empty array:  Nothing to do.
     __ cbz(count, L_done);
-    __ push(RegSet::of(r19, r20, r21, r22), sp);
+    __ push(RegSet::of(r19, r20, r21, r22, r23), sp);
 
 #ifdef ASSERT
     BLOCK_COMMENT("assert consistent ckoff/ckval");
@@ -1935,6 +1936,7 @@ class StubGenerator: public StubCodeGenerator {
 
     // Copy from low to high addresses
     __ mov(start_to, to);              // Save destination array start address
+    __ mov(start_from, from);
     __ b(L_load_element);
 
     // ======== begin loop ========
@@ -1975,10 +1977,10 @@ class StubGenerator: public StubCodeGenerator {
     __ br(Assembler::EQ, L_done_pop);
 
     __ BIND(L_do_card_marks);
-    bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, rscratch1, wb_post_saved_regs);
+    bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, start_from, count_save, rscratch1, wb_post_saved_regs);
 
     __ bind(L_done_pop);
-    __ pop(RegSet::of(r19, r20, r21, r22), sp);
+    __ pop(RegSet::of(r19, r20, r21, r22, r23), sp);
     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
 
     __ bind(L_done);

src/hotspot/cpu/aarch64/templateTable_aarch64.cpp

@@ -1105,6 +1105,7 @@ void TemplateTable::aastore() {
   __ ldr(r3, at_tos_p2()); // array
 
   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
+  element_address._obj_start = r3;
 
   index_check(r3, r2);     // kills r1
   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
@@ -2724,6 +2725,7 @@ void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteContr
 
   // field address
   const Address field(obj, off);
+  field._obj_start = obj;
 
   Label notByte, notBool, notInt, notShort, notChar,
         notLong, notFloat, notObj, notDouble;
@@ -2992,7 +2994,8 @@ void TemplateTable::fast_storefield(TosState state)
   pop_and_check_object(r2);
 
   // field address
-  const Address field(r2, r1);
+  Address field(r2, r1);
+  field._obj_start = r2;
 
   // access field
   switch (bytecode()) {
@@ -3558,8 +3561,12 @@ void TemplateTable::_new() {
   //
   //  Go to slow path.
 
-  if (UseTLAB) {
-    __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
+  if (UseTLAB || UseThirdPartyHeap) {
+    if (UseTLAB) {
+      __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
+    } else {
+      __ eden_allocate(r0, r3, 0, noreg, r1, slow_case);
+    }
 
     if (ZeroTLAB) {
       // the fields have been already cleared