diff --git a/.github/scripts/ci-test-global-alloc-bit.sh b/.github/scripts/ci-test-vo-bit.sh similarity index 99% rename from .github/scripts/ci-test-global-alloc-bit.sh rename to .github/scripts/ci-test-vo-bit.sh index d38c71bf..7c8d43fb 100755 --- a/.github/scripts/ci-test-global-alloc-bit.sh +++ b/.github/scripts/ci-test-vo-bit.sh @@ -5,7 +5,7 @@ set -xe unset JAVA_TOOL_OPTIONS export MMTK_EXTREME_ASSERTIONS=0 -export GLOBAL_ALLOC_BIT=1 +export VO_BIT=1 . $(dirname "$0")/ci-build.sh cd $OPENJDK_PATH diff --git a/.github/scripts/ci-test.sh b/.github/scripts/ci-test.sh index 1de23a83..6112d1b9 100755 --- a/.github/scripts/ci-test.sh +++ b/.github/scripts/ci-test.sh @@ -10,6 +10,6 @@ cd $cur cd $cur ./ci-test-assertions.sh cd $cur -./ci-test-global-alloc-bit.sh +./ci-test-vo-bit.sh cd $cur ./ci-test-malloc-mark-sweep.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38b9252f..7c86df77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,8 +24,8 @@ jobs: run: | ./.github/scripts/ci-build.sh ./.github/scripts/ci-test-only-normal.sh - - name: DaCapo Tests with global allocation bit - run: ./.github/scripts/ci-test-global-alloc-bit.sh + - name: DaCapo Tests with valid object bit (VO bit) + run: ./.github/scripts/ci-test-vo-bit.sh - name: DaCapo Tests with malloc Mark-Sweep run: ./.github/scripts/ci-test-malloc-mark-sweep.sh diff --git a/README.md b/README.md index bb7c89bd..6e5c071e 100644 --- a/README.md +++ b/README.md @@ -197,12 +197,14 @@ building OpenJDK, we can change its location to be in the object's header: $ MARK_IN_HEADER=1 make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk ``` -### Alloc bit -To support the `global_alloc_bit` feature in mmtk-core, you can set the environment variable `GLOBAL_ALLOC_BIT=1` when -building OpenJDK. This will set the feature for mmtk-core, as well as compiling the fastpath for the alloc bit. +### Valid object bit + +To support the `vo_bit` (valid object bit) feature in mmtk-core, you can set the +environment variable `VO_BIT=1` when building OpenJDK. This will set the feature +for mmtk-core, as well as compiling the fastpath for the VO bit. ```console -$ GLOBAL_ALLOC_BIT=1 make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk +$ VO_BIT=1 make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk ``` ## Test diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0f560737..e5c02fa6 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -95,6 +95,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "crossbeam" version = "0.8.2" @@ -162,6 +168,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "delegate" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d358e0ec5c59a5e1603b933def447096886121660fc680dc1e64a0753981fe3c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "downcast-rs" version = "1.2.0" @@ -209,13 +226,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -307,19 +324,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "is-terminal" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -348,9 +365,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libgit2-sys" @@ -378,9 +395,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -419,7 +436,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=f8469b0a019bea58cba26e9fb5c70fb7398ccd31#f8469b0a019bea58cba26e9fb5c70fb7398ccd31" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=4873b4ab4016a2a5ef413463443856efa373e90c#4873b4ab4016a2a5ef413463443856efa373e90c" dependencies = [ "atomic", "atomic-traits", @@ -427,9 +444,11 @@ dependencies = [ "built 0.6.0", "cfg-if", "crossbeam", + "delegate", "downcast-rs", "enum-map", "env_logger", + "is-terminal", "itertools", "lazy_static", "libc", @@ -437,18 +456,19 @@ dependencies = [ "mmtk-macros", "num-traits", "num_cpus", + "probe", "regex", "spin", "static_assertions", "strum", "strum_macros", - "sys-info", + "sysinfo", ] [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=f8469b0a019bea58cba26e9fb5c70fb7398ccd31#f8469b0a019bea58cba26e9fb5c70fb7398ccd31" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=4873b4ab4016a2a5ef413463443856efa373e90c#4873b4ab4016a2a5ef413463443856efa373e90c" dependencies = [ "proc-macro-error", "proc-macro2", @@ -467,6 +487,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -514,6 +543,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "probe" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e81fcf486280f0b8b18ca43ceafd32739eb0eb703eb024a8d00814eeba556" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -556,6 +591,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "regex" version = "1.7.1" @@ -584,16 +641,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.37.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -701,13 +758,18 @@ dependencies = [ ] [[package]] -name = "sys-info" -version = "0.9.1" +name = "sysinfo" +version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +checksum = "6b949f01f9c23823744b71e0060472ecbde578ef68cc2a9e46d114efd77c3034" dependencies = [ - "cc", + "cfg-if", + "core-foundation-sys", "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", ] [[package]] @@ -850,7 +912,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.1", ] [[package]] @@ -859,13 +930,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -874,38 +960,80 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 344d8044..7975fb9b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -30,7 +30,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "f8469b0a019bea58cba26e9fb5c70fb7398ccd31" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "4873b4ab4016a2a5ef413463443856efa373e90c" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -42,9 +42,9 @@ default = [] mmtk_extreme_assertions = ["mmtk/extreme_assertions"] nogc_lock_free = ["mmtk/nogc_lock_free"] nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] -# Use the env var GLOBAL_ALLOC_BIT=1 when building OpenJDK so the fastpath for setting the alloc bit will be compiled in. +# Use the env var VO_BIT=1 when building OpenJDK so the fastpath for setting the alloc bit will be compiled in. # See README. -global_alloc_bit = ["mmtk/global_alloc_bit"] +vo_bit = ["mmtk/vo_bit"] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] @@ -62,5 +62,7 @@ gencopy = [] marksweep = [] pageprotect = [] immix = [] +genimmix = [] +stickyimmix = [] jfr = [] diff --git a/mmtk/src/active_plan.rs b/mmtk/src/active_plan.rs index 88d12958..5ecaa92d 100644 --- a/mmtk/src/active_plan.rs +++ b/mmtk/src/active_plan.rs @@ -1,3 +1,4 @@ +use crate::MutatorClosure; use crate::OpenJDK; use crate::SINGLETON; use crate::UPCALLS; @@ -5,7 +6,36 @@ use mmtk::util::opaque_pointer::*; use mmtk::vm::ActivePlan; use mmtk::Mutator; use mmtk::Plan; -use std::sync::Mutex; +use std::collections::VecDeque; +use std::marker::PhantomData; + +struct OpenJDKMutatorIterator<'a> { + mutators: VecDeque<&'a mut Mutator>, + phantom_data: PhantomData<&'a ()>, +} + +impl<'a> OpenJDKMutatorIterator<'a> { + fn new() -> Self { + let mut mutators = VecDeque::new(); + unsafe { + ((*UPCALLS).get_mutators)(MutatorClosure::from_rust_closure(&mut |mutator| { + mutators.push_back(mutator); + })); + } + Self { + mutators, + phantom_data: PhantomData, + } + } +} + +impl<'a> Iterator for OpenJDKMutatorIterator<'a> { + type Item = &'a mut Mutator; + + fn next(&mut self) -> Option { + self.mutators.pop_front() + } +} pub struct VMActivePlan {} @@ -25,29 +55,11 @@ impl ActivePlan for VMActivePlan { } } - fn reset_mutator_iterator() { - unsafe { - ((*UPCALLS).reset_mutator_iterator)(); - } - } - - fn get_next_mutator() -> Option<&'static mut Mutator> { - let _guard = MUTATOR_ITERATOR_LOCK.lock().unwrap(); - unsafe { - let m = ((*UPCALLS).get_next_mutator)(); - if m.is_null() { - None - } else { - Some(&mut *m) - } - } + fn mutators<'a>() -> Box> + 'a> { + Box::new(OpenJDKMutatorIterator::new()) } fn number_of_mutators() -> usize { unsafe { ((*UPCALLS).number_of_mutators)() } } } - -lazy_static! { - pub static ref MUTATOR_ITERATOR_LOCK: Mutex<()> = Mutex::new(()); -} diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5bd6b820..58949a56 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -72,6 +72,10 @@ pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls) { Some(PlanSelector::PageProtect) } else if cfg!(feature = "immix") { Some(PlanSelector::Immix) + } else if cfg!(feature = "genimmix") { + Some(PlanSelector::GenImmix) + } else if cfg!(feature = "stickyimmix") { + Some(PlanSelector::StickyImmix) } else { None }; @@ -129,7 +133,7 @@ pub extern "C" fn alloc( mutator: *mut Mutator, size: usize, align: usize, - offset: isize, + offset: usize, allocator: AllocationSemantics, ) -> Address { memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) @@ -286,6 +290,20 @@ pub extern "C" fn process(name: *const c_char, value: *const c_char) -> bool { ) } +/// Pass hotspot `ParallelGCThreads` flag to mmtk +#[no_mangle] +pub extern "C" fn mmtk_builder_set_threads(value: usize) { + let mut builder = BUILDER.lock().unwrap(); + builder.options.threads.set(value); +} + +/// Pass hotspot `UseTransparentHugePages` flag to mmtk +#[no_mangle] +pub extern "C" fn mmtk_builder_set_transparent_hugepages(value: bool) { + let mut builder = BUILDER.lock().unwrap(); + builder.options.transparent_hugepages.set(value); +} + #[no_mangle] // We trust the name/value pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -325,7 +343,7 @@ pub extern "C" fn mmtk_object_reference_write_pre( ) { mutator .barrier() - .object_reference_write_pre(src, slot, target); + .object_reference_write_pre(src, slot.into(), target); } /// Full post barrier @@ -338,7 +356,7 @@ pub extern "C" fn mmtk_object_reference_write_post( ) { mutator .barrier() - .object_reference_write_post(src, slot, target); + .object_reference_write_post(src, slot.into(), target); } /// Barrier slow-path call @@ -351,7 +369,7 @@ pub extern "C" fn mmtk_object_reference_write_slow( ) { mutator .barrier() - .object_reference_write_slow(src, slot, target); + .object_reference_write_slow(src, slot.into(), target); } /// Array-copy pre-barrier @@ -365,7 +383,7 @@ pub extern "C" fn mmtk_array_copy_pre( let bytes = count << LOG_BYTES_IN_ADDRESS; mutator .barrier() - .memory_region_copy_pre(src..src + bytes, dst..dst + bytes); + .memory_region_copy_pre((src..src + bytes).into(), (dst..dst + bytes).into()); } /// Array-copy post-barrier @@ -379,7 +397,7 @@ pub extern "C" fn mmtk_array_copy_post( let bytes = count << LOG_BYTES_IN_ADDRESS; mutator .barrier() - .memory_region_copy_post(src..src + bytes, dst..dst + bytes); + .memory_region_copy_post((src..src + bytes).into(), (dst..dst + bytes).into()); } /// C2 Slowpath allocation barrier diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index f02e3f56..776c737a 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,53 +1,25 @@ use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; -use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding}; -use mmtk::{Mutator, MutatorContext}; +use mmtk::vm::{Collection, GCThreadContext}; +use mmtk::Mutator; use crate::UPCALLS; use crate::{MutatorClosure, OpenJDK}; pub struct VMCollection {} -extern "C" fn report_mutator_stop( - mutator: *mut Mutator, - callback_ptr: *mut libc::c_void, -) where - F: FnMut(&'static mut Mutator), -{ - let callback: &mut F = unsafe { &mut *(callback_ptr as *mut F) }; - callback(unsafe { &mut *mutator }); -} - -fn to_mutator_closure(callback: &mut F) -> MutatorClosure -where - F: FnMut(&'static mut Mutator), -{ - MutatorClosure { - func: report_mutator_stop::, - data: callback as *mut F as *mut libc::c_void, - } -} - const GC_THREAD_KIND_CONTROLLER: libc::c_int = 0; const GC_THREAD_KIND_WORKER: libc::c_int = 1; impl Collection for VMCollection { - /// With the presence of the "VM companion thread", - /// the OpenJDK binding allows any MMTk GC thread to stop/start the world. - const COORDINATOR_ONLY_STW: bool = false; - fn stop_all_mutators(tls: VMWorkerThread, mut mutator_visitor: F) where F: FnMut(&'static mut Mutator), { - let scan_mutators_in_safepoint = - ::VMScanning::SCAN_MUTATORS_IN_SAFEPOINT; - unsafe { ((*UPCALLS).stop_all_mutators)( tls, - scan_mutators_in_safepoint, - to_mutator_closure(&mut mutator_visitor), + MutatorClosure::from_rust_closure(&mut mutator_visitor), ); } } @@ -79,14 +51,6 @@ impl Collection for VMCollection { } } - fn prepare_mutator>( - _tls_w: VMWorkerThread, - _tls_m: VMMutatorThread, - _m: &T, - ) { - // unimplemented!() - } - fn out_of_memory(tls: VMThread, err_kind: AllocationError) { unsafe { ((*UPCALLS).out_of_memory)(tls, err_kind); diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs new file mode 100644 index 00000000..44f666d1 --- /dev/null +++ b/mmtk/src/edges.rs @@ -0,0 +1,103 @@ +use std::ops::Range; + +use mmtk::{ + util::{Address, ObjectReference}, + vm::edge_shape::{AddressRangeIterator, Edge, MemorySlice}, +}; + +/// The type of edges in OpenJDK. +/// Currently it has the same layout as `Address`, but we override its load and store methods. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +#[repr(transparent)] +pub struct OpenJDKEdge { + pub addr: Address, +} + +impl From
for OpenJDKEdge { + fn from(value: Address) -> Self { + Self { addr: value } + } +} + +impl Edge for OpenJDKEdge { + fn load(&self) -> ObjectReference { + if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + // Workaround: On x86 (including x86_64), machine instructions may contain pointers as + // immediates, and they may be unaligned. It is an undefined behavior in Rust to + // dereference unaligned pointers. We have to explicitly use unaligned memory access + // methods. On x86, ordinary MOV instructions can load and store memory at unaligned + // addresses, so we expect `ptr.read_unaligned()` to have no performance penalty over + // `ptr.read()` if `ptr` is actually aligned. + unsafe { + let ptr = self.addr.to_ptr::(); + ptr.read_unaligned() + } + } else { + unsafe { self.addr.load() } + } + } + + fn store(&self, object: ObjectReference) { + if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + unsafe { + let ptr = self.addr.to_mut_ptr::(); + ptr.write_unaligned(object) + } + } else { + unsafe { self.addr.store(object) } + } + } +} + +/// A range of OpenJDKEdge, usually used for arrays. +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct OpenJDKEdgeRange { + range: Range
, +} + +impl From> for OpenJDKEdgeRange { + fn from(value: Range
) -> Self { + Self { range: value } + } +} + +pub struct OpenJDKEdgeRangeIterator { + inner: AddressRangeIterator, +} + +impl Iterator for OpenJDKEdgeRangeIterator { + type Item = OpenJDKEdge; + + fn next(&mut self) -> Option { + self.inner.next().map(|a| a.into()) + } +} + +// Note that we cannot implement MemorySlice for `Range` because neither +// `MemorySlice` nor `Range` are defined in the `mmtk-openjdk` crate. ("orphan rule") +impl MemorySlice for OpenJDKEdgeRange { + type Edge = OpenJDKEdge; + type EdgeIterator = OpenJDKEdgeRangeIterator; + + fn iter_edges(&self) -> Self::EdgeIterator { + OpenJDKEdgeRangeIterator { + inner: self.range.iter_edges(), + } + } + + fn object(&self) -> Option { + self.range.object() + } + + fn start(&self) -> Address { + self.range.start() + } + + fn bytes(&self) -> usize { + self.range.bytes() + } + + fn copy(src: &Self, tgt: &Self) { + MemorySlice::copy(&src.range, &tgt.range) + } +} diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 6eb195ce..16b1687e 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -52,7 +52,7 @@ impl> GCWork for ScanCodeCacheRoots let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(*r) + edges.push((*r).into()) } } // Create work packet diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index a7cf02c7..f7ae852a 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -2,11 +2,11 @@ extern crate lazy_static; use std::collections::HashMap; -use std::ops::Range; use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; use std::sync::Mutex; +use edges::{OpenJDKEdge, OpenJDKEdgeRange}; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; @@ -19,6 +19,7 @@ pub mod active_plan; pub mod api; mod build_info; pub mod collection; +mod edges; mod gc_work; pub mod object_model; mod object_scanning; @@ -39,6 +40,28 @@ pub struct MutatorClosure { pub data: *mut libc::c_void, } +impl MutatorClosure { + fn from_rust_closure(callback: &mut F) -> Self + where + F: FnMut(&'static mut Mutator), + { + Self { + func: Self::call_rust_closure::, + data: callback as *mut F as *mut libc::c_void, + } + } + + extern "C" fn call_rust_closure( + mutator: *mut Mutator, + callback_ptr: *mut libc::c_void, + ) where + F: FnMut(&'static mut Mutator), + { + let callback: &mut F = unsafe { &mut *(callback_ptr as *mut F) }; + callback(unsafe { &mut *mutator }); + } +} + /// A closure for reporting root edges. The C++ code should pass `data` back as the last argument. #[repr(C)] pub struct EdgesClosure { @@ -53,17 +76,12 @@ pub struct EdgesClosure { #[repr(C)] pub struct OpenJDK_Upcalls { - pub stop_all_mutators: extern "C" fn( - tls: VMWorkerThread, - scan_mutators_in_safepoint: bool, - closure: MutatorClosure, - ), + pub stop_all_mutators: extern "C" fn(tls: VMWorkerThread, closure: MutatorClosure), pub resume_mutators: extern "C" fn(tls: VMWorkerThread), pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), - pub get_next_mutator: extern "C" fn() -> *mut Mutator, - pub reset_mutator_iterator: extern "C" fn(), + pub get_mutators: extern "C" fn(closure: MutatorClosure), pub scan_object: extern "C" fn(trace: *mut c_void, object: ObjectReference, tls: OpaquePointer), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, @@ -77,8 +95,8 @@ pub struct OpenJDK_Upcalls { pub referent_offset: extern "C" fn() -> i32, pub discovered_offset: extern "C" fn() -> i32, pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char, - pub scan_all_thread_roots: extern "C" fn(closure: EdgesClosure), - pub scan_thread_roots: extern "C" fn(closure: EdgesClosure, tls: VMMutatorThread), + pub scan_roots_in_all_mutator_threads: extern "C" fn(closure: EdgesClosure), + pub scan_roots_in_mutator_thread: extern "C" fn(closure: EdgesClosure, tls: VMMutatorThread), pub scan_code_cache_roots: extern "C" fn(closure: EdgesClosure), pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure), pub scan_oop_storage_set_roots: extern "C" fn(closure: EdgesClosure), @@ -101,8 +119,8 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); #[no_mangle] -pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = - mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); +pub static VO_BIT_ADDRESS: uintptr_t = + mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR.as_usize(); #[no_mangle] pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = @@ -111,13 +129,6 @@ pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = #[derive(Default)] pub struct OpenJDK; -/// The type of edges in OpenJDK. -/// -/// TODO: We currently make it an alias to Address to make the change minimal. -/// If we support CompressedOOPs, we should define an enum type to support both -/// compressed and uncompressed OOPs. -pub type OpenJDKEdge = Address; - impl VMBinding for OpenJDK { type VMObjectModel = object_model::VMObjectModel; type VMScanning = scanning::VMScanning; @@ -126,7 +137,7 @@ impl VMBinding for OpenJDK { type VMReferenceGlue = reference_glue::VMReferenceGlue; type VMEdge = OpenJDKEdge; - type VMMemorySlice = Range
; + type VMMemorySlice = OpenJDKEdgeRange; const MIN_ALIGNMENT: usize = 8; const MAX_ALIGNMENT: usize = 8; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 7ee39948..3cfe50fd 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -72,7 +72,7 @@ impl ObjectModel for VMObjectModel { ::std::mem::size_of::() } - fn get_align_offset_when_copied(_object: ObjectReference) -> isize { + fn get_align_offset_when_copied(_object: ObjectReference) -> usize { 0 } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 57e87610..189aa4e6 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -16,7 +16,7 @@ impl OopIterate for OopMapBlock { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = start + (i << LOG_BYTES_IN_ADDRESS); + let edge = (start + (i << LOG_BYTES_IN_ADDRESS)).into(); closure.visit_edge(edge); } } @@ -66,7 +66,7 @@ impl OopIterate for InstanceMirrorKlass { let len = Self::static_oop_field_count(oop); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for oop in slice { - closure.visit_edge(Address::from_ref(oop as &Oop)); + closure.visit_edge(Address::from_ref(oop as &Oop).into()); } } } @@ -88,7 +88,7 @@ impl OopIterate for ObjArrayKlass { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let array = unsafe { oop.as_array_oop() }; for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(Address::from_ref(oop as &Oop)); + closure.visit_edge(Address::from_ref(oop as &Oop).into()); } } } @@ -133,9 +133,9 @@ impl InstanceRefKlass { } fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { let referent_addr = Self::referent_address(oop); - closure.visit_edge(referent_addr); + closure.visit_edge(referent_addr.into()); let discovered_addr = Self::discovered_address(oop); - closure.visit_edge(discovered_addr); + closure.visit_edge(discovered_addr.into()); } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 479755a2..166600e5 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -20,7 +20,9 @@ extern "C" fn report_edges_and_renew_buffer>( factory_ptr: *mut libc::c_void, ) -> NewBuffer { if !ptr.is_null() { - let buf = unsafe { Vec::
::from_raw_parts(ptr, length, capacity) }; + // Note: Currently OpenJDKEdge has the same layout as Address. If the layout changes, we + // should fix the Rust-to-C interface. + let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf); } @@ -42,9 +44,6 @@ pub(crate) fn to_edges_closure>(factory: &mut F } impl Scanning for VMScanning { - const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; - const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object>( tls: VMWorkerThread, object: ObjectReference, @@ -58,20 +57,14 @@ impl Scanning for VMScanning { // TODO } - fn scan_thread_roots(_tls: VMWorkerThread, mut factory: impl RootsWorkFactory) { - unsafe { - ((*UPCALLS).scan_all_thread_roots)(to_edges_closure(&mut factory)); - } - } - - fn scan_thread_root( + fn scan_roots_in_mutator_thread( _tls: VMWorkerThread, mutator: &'static mut Mutator, mut factory: impl RootsWorkFactory, ) { let tls = mutator.get_tls(); unsafe { - ((*UPCALLS).scan_thread_roots)(to_edges_closure(&mut factory), tls); + ((*UPCALLS).scan_roots_in_mutator_thread)(to_edges_closure(&mut factory), tls); } } @@ -84,15 +77,9 @@ impl Scanning for VMScanning { Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, Box::new(ScanOopStorageSetRoots::new(factory.clone())) as _, // FIXME17: Several removed roots are all put to this work packet, may cause slowdown. Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _, + Box::new(ScanVMThreadRoots::new(factory)) as _, ], ); - if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { - memory_manager::add_work_packet( - &SINGLETON, - WorkBucketStage::Prepare, - ScanVMThreadRoots::new(factory), - ); - } } fn supports_return_barrier() -> bool { diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index 48867f50..0c5a9a70 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -19,11 +19,11 @@ ifdef MMTK_PLAN GC_FEATURES=--features $(MMTK_PLAN) endif -ifeq ($(GLOBAL_ALLOC_BIT), 1) +ifeq ($(VO_BIT), 1) ifndef GC_FEATURES - GC_FEATURES=--features global_alloc_bit + GC_FEATURES=--features vo_bit else - GC_FEATURES:=$(strip $(GC_FEATURES))",global_alloc_bit" + GC_FEATURES:=$(strip $(GC_FEATURES))",vo_bit" endif endif @@ -90,8 +90,8 @@ $(LIB_MMTK): FORCE echo -e "Only cross compiling to riscv64 is supported"; \ exit 1; \ fi - echo "cd $(MMTK_RUST_ROOT) && $(CARGO_EXECUTABLE) build $(CARGO_TARGET_FLAG) $(CARGO_PROFILE_FLAG) $(GC_FEATURES)" - cd $(MMTK_RUST_ROOT) && $(CARGO_EXECUTABLE) build $(CARGO_TARGET_FLAG) $(CARGO_PROFILE_FLAG) $(GC_FEATURES) + echo "cd $(MMTK_RUST_ROOT) && $(CARGO_EXECUTABLE) --version && $(CARGO_EXECUTABLE) build $(CARGO_TARGET_FLAG) $(CARGO_PROFILE_FLAG) $(GC_FEATURES)" + cd $(MMTK_RUST_ROOT) && $(CARGO_EXECUTABLE) --version && $(CARGO_EXECUTABLE) build $(CARGO_TARGET_FLAG) $(CARGO_PROFILE_FLAG) $(GC_FEATURES) mkdir -p $(JVM_LIB_OUTPUTDIR) && cp $(MMTK_RUST_ROOT)/target/$(CARGO_TARGET)/$(CARGO_PROFILE)/libmmtk_openjdk.so $(LIB_MMTK) JVM_LIBS += -L$(JVM_LIB_OUTPUTDIR) -lmmtk_openjdk @@ -102,8 +102,8 @@ FORCE: TARGETS += $(LIB_MMTK) JVM_SRC_DIRS += $(MMTK_CPP_ROOT)/share $(MMTK_CPP_ROOT)/share/barriers $(MMTK_CPP_ROOT)/cpu/$(HOTSPOT_TARGET_CPU_ARCH) JVM_CFLAGS += -std=c++14 -fpermissive -DINCLUDE_THIRD_PARTY_HEAP -DTHIRD_PARTY_HEAP_SRC=$(MMTK_CPP_ROOT)/share -I$(MMTK_CPP_ROOT)/share -I$(MMTK_CPP_ROOT)/cpu/$(HOTSPOT_TARGET_CPU_ARCH) -ifeq ($(GLOBAL_ALLOC_BIT), 1) - JVM_CFLAGS += -DMMTK_ENABLE_GLOBAL_ALLOC_BIT +ifeq ($(VO_BIT), 1) + JVM_CFLAGS += -DMMTK_ENABLE_VO_BIT endif $(BUILD_LIBJVM): $(LIB_MMTK) diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index 7991f580..d810b7af 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -98,11 +98,11 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa __ jcc(Assembler::above, slow_case); // lab.cursor = end __ movptr(cursor, end); - bool enable_global_alloc_bit = false; - #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT - enable_global_alloc_bit = true; + bool enable_vo_bit = false; + #ifdef MMTK_ENABLE_VO_BIT + enable_vo_bit = true; #endif - if (enable_global_alloc_bit || selector.tag == TAG_MARK_COMPACT) { + if (enable_vo_bit || selector.tag == TAG_MARK_COMPACT) { Register tmp3 = rdi; Register tmp2 = rscratch1; assert_different_registers(obj, tmp2, tmp3, rcx); @@ -110,7 +110,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ movptr(tmp3, obj); __ shrptr(tmp3, 6); - __ movptr(tmp2, ALLOC_BIT_BASE_ADDRESS); + __ movptr(tmp2, VO_BIT_BASE_ADDRESS); __ movb(tmp2, Address(tmp2, tmp3)); // tmp3 = 1 << ((obj >> 3) & 7) // 1. rcx = (obj >> 3) & 7 @@ -126,7 +126,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // store-byte tmp2 (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)) __ movptr(tmp3, obj); __ shrptr(tmp3, 6); - __ movptr(rcx, ALLOC_BIT_BASE_ADDRESS); + __ movptr(rcx, VO_BIT_BASE_ADDRESS); __ movb(Address(rcx, tmp3), tmp2); } diff --git a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp index a496191e..ca881ced 100644 --- a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp @@ -56,9 +56,41 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* #endif } +void MMTkObjectBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + // `count` or `dst` register values may get overwritten after the array copy, and `arraycopy_epilogue` can receive invalid addresses. + // Save the register values here and restore them in `arraycopy_epilogue`. + // See https://github.com/openjdk/jdk/blob/jdk-11%2B19/src/hotspot/cpu/x86/gc/shared/modRefBarrierSetAssembler_x86.cpp#L37-L50 + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); + if (type == T_OBJECT || type == T_ARRAY) { + if (!checkcast) { + if (!obj_int) { + // Save count for barrier + __ movptr(r11, count); + } else if (disjoint) { + // Save dst in r11 in the disjoint case + __ movq(r11, dst); + } + } + } +} + void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) { + if (!checkcast) { + if (!obj_int) { + // Save count for barrier + count = r11; + } else if (disjoint) { + // Use the saved dst in the disjoint case + dst = r11; + } + } __ pusha(); __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); diff --git a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.hpp b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.hpp index a6686e0d..b6f6bd4f 100644 --- a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.hpp +++ b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.hpp @@ -5,6 +5,7 @@ class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; #endif // MMTK_OPENJDK_MMTK_OBJECT_BARRIER_SET_ASSEMBLER_X86_HPP diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 119cd510..6e76b7a5 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -20,7 +20,7 @@ typedef enum { extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; -extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; +extern const uintptr_t VO_BIT_ADDRESS; extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; @@ -44,7 +44,7 @@ extern void* alloc_slow_largeobject(MMTk_Mutator mutator, size_t size, size_t align, size_t offset); extern void post_alloc(MMTk_Mutator mutator, void* refer, - int bytes, int allocator); + size_t bytes, int allocator); /// Full pre-barrier extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, void* slot, void* target); @@ -144,13 +144,12 @@ struct EdgesClosure { * OpenJDK-specific */ typedef struct { - void (*stop_all_mutators) (void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure); + void (*stop_all_mutators) (void *tls, MutatorClosure closure); void (*resume_mutators) (void *tls); void (*spawn_gc_thread) (void *tls, int kind, void *ctx); void (*block_for_gc) (); void (*out_of_memory) (void *tls, MMTkAllocationError err_kind); - void* (*get_next_mutator) (); - void (*reset_mutator_iterator) (); + void (*get_mutators) (MutatorClosure closure); void (*scan_object) (void* trace, void* object, void* tls); void (*dump_object) (void* object); size_t (*get_object_size) (void* object); @@ -164,8 +163,8 @@ typedef struct { int (*referent_offset) (); int (*discovered_offset) (); char* (*dump_object_string) (void* object); - void (*scan_all_thread_roots)(EdgesClosure closure); - void (*scan_thread_roots)(EdgesClosure closure, void* tls); + void (*scan_roots_in_all_mutator_threads)(EdgesClosure closure); + void (*scan_roots_in_mutator_thread)(EdgesClosure closure, void* tls); void (*scan_code_cache_roots) (EdgesClosure closure); void (*scan_class_loader_data_graph_roots) (EdgesClosure closure); void (*scan_oop_storage_set_roots) (EdgesClosure closure); @@ -205,6 +204,9 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); +extern void mmtk_builder_set_threads(size_t value); +extern void mmtk_builder_set_transparent_hugepages(bool value); + #ifdef __cplusplus } #endif diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 1d538795..d4fffe8d 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -41,7 +41,7 @@ #define MMTK_ENABLE_ALLOCATION_FASTPATH true #define MMTK_ENABLE_BARRIER_FASTPATH true -const intptr_t ALLOC_BIT_BASE_ADDRESS = GLOBAL_ALLOC_BIT_ADDRESS; +const intptr_t VO_BIT_BASE_ADDRESS = VO_BIT_ADDRESS; struct MMTkAllocatorOffsets { int tlab_top_offset; diff --git a/openjdk/share/mmtkBarrierSetC2.cpp b/openjdk/share/mmtkBarrierSetC2.cpp index 53680c25..6bb4241f 100644 --- a/openjdk/share/mmtkBarrierSetC2.cpp +++ b/openjdk/share/mmtkBarrierSetC2.cpp @@ -274,14 +274,14 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, fast_oop_ctrl = needgc_false; // No contention, so this is the fast path fast_oop_rawmem = store_eden_top; - bool enable_global_alloc_bit = false; - #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT - enable_global_alloc_bit = true; + bool enable_vo_bit = false; + #ifdef MMTK_ENABLE_VO_BIT + enable_vo_bit = true; #endif - if (enable_global_alloc_bit || selector.tag == TAG_MARK_COMPACT) { + if (enable_vo_bit || selector.tag == TAG_MARK_COMPACT) { // set the alloc bit: // intptr_t addr = (intptr_t) (void*) fast_oop; - // uint8_t* meta_addr = (uint8_t*) (ALLOC_BIT_BASE_ADDRESS + (addr >> 6)); + // uint8_t* meta_addr = (uint8_t*) (VO_BIT_BASE_ADDRESS + (addr >> 6)); // intptr_t shift = (addr >> 3) & 0b111; // uint8_t byte_val = *meta_addr; // uint8_t new_byte_val = byte_val | (1 << shift); @@ -295,7 +295,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, Node *meta_offset = new URShiftLNode(obj_addr, addr_shift); x->transform_later(meta_offset); - Node *meta_base = ConLNode::make(ALLOC_BIT_BASE_ADDRESS); + Node *meta_base = ConLNode::make(VO_BIT_BASE_ADDRESS); x->transform_later(meta_base); Node *meta_addr = new AddLNode(meta_base, meta_offset); @@ -331,10 +331,10 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, Node *new_meta_val = new OrINode(meta_val, set_bit); x->transform_later(new_meta_val); - Node *set_alloc_bit = new StoreBNode(fast_oop_ctrl, fast_oop_rawmem, meta_addr_p, TypeRawPtr::BOTTOM, new_meta_val, MemNode::unordered); - x->transform_later(set_alloc_bit); + Node *set_vo_bit = new StoreBNode(fast_oop_ctrl, fast_oop_rawmem, meta_addr_p, TypeRawPtr::BOTTOM, new_meta_val, MemNode::unordered); + x->transform_later(set_vo_bit); - fast_oop_rawmem = set_alloc_bit; + fast_oop_rawmem = set_vo_bit; } InitializeNode* init = alloc->initialization(); diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index a53f8de4..488da474 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -94,6 +94,8 @@ jint MMTkHeap::initialize() { // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); // Set options + mmtk_builder_set_threads(ParallelGCThreads); + mmtk_builder_set_transparent_hugepages(UseTransparentHugePages); if (ThirdPartyHeapOptions != NULL) { bool set_options = process_bulk(os::strdup(ThirdPartyHeapOptions)); guarantee(set_options, "Failed to set MMTk options. Please check if the options are valid: %s\n", ThirdPartyHeapOptions); @@ -423,7 +425,7 @@ void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { VMThread::vm_thread()->oops_do(&cl, NULL); } -void MMTkHeap::scan_thread_roots(OopClosure& cl) { +void MMTkHeap::scan_roots_in_all_mutator_threads(OopClosure& cl) { ResourceMark rm; Threads::possibly_parallel_oops_do(false, &cl, NULL); } diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 0b201a5e..44e340b9 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -198,7 +198,7 @@ class MMTkHeap : public CollectedHeap { void scan_roots(OopClosure& cl); - void scan_thread_roots(OopClosure& cl); + void scan_roots_in_all_mutator_threads(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); void scan_class_loader_data_graph_roots(OopClosure& cl); diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index dc6d9f8f..6c50ceef 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -51,7 +51,7 @@ // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; -static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { +static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { ClassLoaderDataGraph::clear_claimed_marks(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); @@ -61,12 +61,11 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); - if (!scan_mutators_in_safepoint) { - JavaThreadIteratorWithHandle jtiwh; - while (JavaThread *cur = jtiwh.next()) { - closure.invoke((void*)&cur->third_party_heap_mutator); - } + JavaThreadIteratorWithHandle jtiwh; + while (JavaThread *cur = jtiwh.next()) { + closure.invoke((void*)&cur->third_party_heap_mutator); } + log_debug(gc)("Finished enumerating threads."); nmethod::oops_do_marking_prologue(); } @@ -184,46 +183,19 @@ static bool mmtk_is_mutator(void* tls) { return ((Thread*) tls)->third_party_heap_collector == NULL; } -template -struct MaybeUninit { - MaybeUninit() {} - T* operator->() { - return (T*) &_data; - } - T& operator*() { - return *((T*) &_data); +static void mmtk_get_mutators(MutatorClosure closure) { + JavaThread *thr; + for (JavaThreadIteratorWithHandle jtiwh; thr = jtiwh.next();) { + closure.invoke(&thr->third_party_heap_mutator); } -private: - char _data[sizeof(T)]; -}; - -static MaybeUninit jtiwh; -static bool mutator_iteration_start = true; - -static void* mmtk_get_next_mutator() { - if (mutator_iteration_start) { - *jtiwh = JavaThreadIteratorWithHandle(); - mutator_iteration_start = false; - } - JavaThread *thr = jtiwh->next(); - if (thr == NULL) { - mutator_iteration_start = true; - return NULL; - } - return (void*) &thr->third_party_heap_mutator; } -static void mmtk_reset_mutator_iterator() { - mutator_iteration_start = true; -} - - -static void mmtk_scan_all_thread_roots(EdgesClosure closure) { +static void mmtk_scan_roots_in_all_mutator_threads(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); - MMTkHeap::heap()->scan_thread_roots(cl); + MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } -static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { +static void mmtk_scan_roots_in_mutator_thread(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure2 cl(closure); @@ -358,8 +330,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_spawn_gc_thread, mmtk_block_for_gc, mmtk_out_of_memory, - mmtk_get_next_mutator, - mmtk_reset_mutator_iterator, + mmtk_get_mutators, mmtk_scan_object, mmtk_dump_object, mmtk_get_object_size, @@ -373,8 +344,8 @@ OpenJDK_Upcalls mmtk_upcalls = { referent_offset, discovered_offset, dump_object_string, - mmtk_scan_all_thread_roots, - mmtk_scan_thread_roots, + mmtk_scan_roots_in_all_mutator_threads, + mmtk_scan_roots_in_mutator_thread, mmtk_scan_code_cache_roots, mmtk_scan_class_loader_data_graph_roots, mmtk_scan_oop_storage_set_roots, diff --git a/openjdk/share/thirdPartyHeapArguments.cpp b/openjdk/share/thirdPartyHeapArguments.cpp index 3785f458..d4b84c21 100644 --- a/openjdk/share/thirdPartyHeapArguments.cpp +++ b/openjdk/share/thirdPartyHeapArguments.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "gc/shared/adaptiveSizePolicy.hpp" #include "gc/shared/gcArguments.hpp" +#include "gc/shared/workerPolicy.hpp" #include "mmtkHeap.hpp" #include "runtime/globals_extension.hpp" #include "runtime/globals.hpp" @@ -43,6 +44,12 @@ void ThirdPartyHeapArguments::initialize() { FLAG_SET_DEFAULT(UseTLAB, false); FLAG_SET_DEFAULT(UseCompressedOops, false); FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + FLAG_SET_DEFAULT(ParallelGCThreads, WorkerPolicy::parallel_worker_threads()); + if (ParallelGCThreads == 0) { + assert(!FLAG_IS_DEFAULT(ParallelGCThreads), "ParallelGCThreads should not be 0."); + vm_exit_during_initialization("The flag -XX:+UseUseThirdPartyHeap can not be combined with -XX:ParallelGCThreads=0", NULL); + } + } void ThirdPartyHeapArguments::initialize_alignments() {