Use macro to generate process edges for plans (#575)

This PR adds `PlanProcessEdges`, and a few macros/traits that are used in `PlanProcessEdges`. This PR removes policy-specific code from plan, and use macros to generate trace object work for each plan. This PR closes #258, and closes #576.

* add a trait `PolicyTraceObject`. Each policy provides an implementation of this. With this trait, a plan no longer include any policy-specific code in trace object.
* add a trait `PlanTraceObject` and related macros. With the macro, plan implementer can declaratively specify trace object behavior with attributes, and the implementation of `PlanTraceObject` will be generated by the macro.
* add a type `PlanProcessEdges`. This uses `PlanTraceObject` to provide a general implementation of `ProcessEdgesWork`. We no longer need any plan-specific process edges work.

Author: qinsoon

.github/workflows/cargo-publish.yml

@@ -19,5 +19,9 @@ jobs:
           override: true
       - name: Cargo login
         run: cargo login ${{ secrets.CI_CARGO_LOGIN }}
-      - name: Cargo publish
-        run: cargo publish
+      - name: Publish sub crates
+        run: |
+          cargo publish --manifest-path=macros/Cargo.toml
+      - name: Public mmtk-core
+        run: |
+          cargo publish

Cargo.toml

@@ -18,6 +18,9 @@ crate-type = ["rlib"]
 doctest = false
 
 [dependencies]
+# MMTk macros
+mmtk-macros = { version = "0.11.0", path = "macros/" }
+
 custom_derive = "0.1"
 enum_derive = "0.1"
 libc = "0.2"

docs/tutorial/code/mygc_semispace/gc_work.rs

@@ -5,11 +5,95 @@ use crate::vm::VMBinding;
 use std::ops::{Deref, DerefMut};
 // ANCHOR_END: imports
 
-// ANCHOR: workcontext
+// ANCHOR: workcontext_sft
 pub struct MyGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext<VM> {
     type VM = VM;
     type PlanType = MyGC<VM>;
     type ProcessEdgesWorkType = SFTProcessEdges<Self::VM>;
 }
-// ANCHOR_END: workcontext
+// ANCHOR_END: workcontext_sft
+
+// ANCHOR: workcontext_plan
+use crate::scheduler::gc_work::PlanProcessEdges;
+use crate::policy::gc_work::DEFAULT_TRACE;
+pub struct MyGCWorkContext2<VM: VMBinding>(std::marker::PhantomData<VM>);
+impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext2<VM> {
+    type VM = VM;
+    type PlanType = MyGC<VM>;
+    type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, MyGC<VM>, DEFAULT_TRACE>;
+}
+// ANCHOR: workcontext_plan
+
+use crate::util::{Address, ObjectReference};
+use crate::util::copy::CopySemantics;
+use crate::MMTK;
+use crate::policy::space::Space;
+
+// ANCHOR: mygc_process_edges
+pub struct MyGCProcessEdges<VM: VMBinding> {
+    plan: &'static MyGC<VM>,
+    base: ProcessEdgesBase<VM>,
+}
+// ANCHOR_END: mygc_process_edges
+
+// ANCHOR: mygc_process_edges_impl
+impl<VM:VMBinding> ProcessEdgesWork for MyGCProcessEdges<VM> {
+    type VM = VM;
+    fn new(edges: Vec<Address>, roots: bool, mmtk: &'static MMTK<VM>) -> Self {
+        let base = ProcessEdgesBase::new(edges, roots, mmtk);
+        let plan = base.plan().downcast_ref::<MyGC<VM>>().unwrap();
+        Self { base, plan }
+    }
+
+    #[inline]
+    fn trace_object(&mut self, object: ObjectReference) -> ObjectReference {
+        if object.is_null() {
+            return object;
+        }
+        if self.plan.tospace().in_space(object) {
+            self.plan.tospace().trace_object::<Self>(
+                self,
+                object,
+                Some(CopySemantics::DefaultCopy),
+                self.worker(),
+            )
+        } else if self.plan.fromspace().in_space(object) {
+            self.plan.fromspace().trace_object::<Self>(
+                self,
+                object,
+                Some(CopySemantics::DefaultCopy),
+                self.worker(),
+            )
+        } else {
+            self.plan.common.trace_object::<Self>(self, object)
+        }
+    }
+}
+// ANCHOR_END: mygc_process_edges_impl
+
+// ANCHOR: mygc_process_edges_deref
+impl<VM: VMBinding> Deref for MyGCProcessEdges<VM> {
+    type Target = ProcessEdgesBase<VM>;
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.base
+    }
+}
+
+impl<VM: VMBinding> DerefMut for MyGCProcessEdges<VM> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.base
+    }
+}
+// ANCHOR_END: mygc_process_edges_deref
+
+// ANCHOR: workcontext_mygc
+pub struct MyGCWorkContext3<VM: VMBinding>(std::marker::PhantomData<VM>);
+impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext3<VM> {
+    type VM = VM;
+    type PlanType = MyGC<VM>;
+    type ProcessEdgesWorkType = MyGCProcessEdges<Self::VM>;
+}
+// ANCHOR: workcontext_mygc

docs/tutorial/code/mygc_semispace/global.rs

@@ -29,12 +29,18 @@ use std::sync::Arc;
 // Remove #[allow(unused_imports)].
 // Remove handle_user_collection_request().
 
+use mmtk_macros::PlanTraceObject;
+
 // Modify
 // ANCHOR: plan_def
+#[derive(PlanTraceObject)]
 pub struct MyGC<VM: VMBinding> {
     pub hi: AtomicBool,
+    #[trace(CopySemantics::DefaultCopy)]
     pub copyspace0: CopySpace<VM>,
+    #[trace(CopySemantics::DefaultCopy)]
     pub copyspace1: CopySpace<VM>,
+    #[fallback_trace]
     pub common: CommonPlan<VM>,
 }
 // ANCHOR_END: plan_def

docs/tutorial/src/mygc/ss/collection.md

@@ -43,14 +43,15 @@ method `schedule_common_work()` that will add common work packets for you.
 
 To use `schedule_common_work()`, first we need to create a type `MyGCWorkContext`
 and implement the trait `GCWorkContext` for it. We create `gc_work.rs` and add the
-following implementation. Note that we do not set a specific `ProcessEdgesWorkType`
-and we will use the default [`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html),
+following implementation. Note that we will use the default
+[`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html),
 which is a general work packet that a plan can use to trace objects. For plans
 like semispace, `SFTProcessEdges` is sufficient. For more complex GC plans,
 one can create and write their own work packet that implements the `ProcessEdgesWork` trait.
+We will discuss about this later, and discuss the alternatives.
 
 ```rust
-{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext}}
+{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_sft}}
 ```
 
 Then we implement `schedule_collection()` using `MyGCWorkContext` and `schedule_common_work()`.
@@ -111,35 +112,6 @@ there aren't any preparation steps for the mutator in this GC.
 In `create_mygc_mutator()`, find the field `prep_func` and change it from
 `mygc_mutator_noop()` to `mygc_mutator_prepare()`.
 
-
-## Scan objects
-
-Next, we'll add the code to allow the plan to collect garbage - filling out 
-functions for work packets.
-
-In `gc_work.rs`, add a new method to `ProcessEdgesWork for MyGCProcessEdges`,
-`trace_object(&mut self, object: ObjectReference)`.
-This method should return an ObjectReference, and use the 
-inline attribute.
-Check if the object passed into the function is null 
-(`object.is_null()`). If it is, return the object.
-Otherwise, check which space the object is in, and forward the call to the 
-policy-specific object tracing code. If it is in neither space, forward the 
-call to the common space and let the common space to handle object tracing in 
-its spaces (e.g. immortal or large object space):
-
-```rust
-{{#include ../../../code/mygc_semispace/gc_work.rs:trace_object}}
-```
-
-Add two new implementation blocks, `Deref` and `DerefMut` for 
-`MyGCProcessEdges`. These allow `MyGCProcessEdges` to be dereferenced to 
-`ProcessEdgesBase`, and allows easy access to fields in `ProcessEdgesBase`.
-
-```rust
-{{#include ../../../code/mygc_semispace/gc_work.rs:deref}}
-```
-
 ## Release
 
 Finally, we need to fill out the functions that are, roughly speaking, 
@@ -178,6 +150,91 @@ will then go to the new tospace.
 Delete `mygc_mutator_noop()`. It was a placeholder for the prepare and 
 release functions that you have now added, so it is now dead code.
 
+## ProcessEdgesWork for MyGC
+
+[`ProcessEdgesWork`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/trait.ProcessEdgesWork.html)
+is the key work packet for tracing objects in a GC. A `ProcessEdgesWork` implementation
+defines how to trace objects, and how to generate more work packets based on the current tracing
+to finish the object closure.
+
+`GCWorkContext` specifies a type
+that implements `ProcessEdgesWork`, and we used `SFTProcessEdges` earlier. In
+this section, we discuss what `SFTProcessEdges` does, and what the alternatives
+are.
+
+### Approach 1: Use `SFTProcessEdges`
+
+[`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html) dispatches
+the tracing of objects to their respective spaces through [Space Function Table (SFT)](https://www.mmtk.io/mmtk-core/mmtk/policy/space/trait.SFT.html).
+As long as all the policies in a plan provide an implementation of `sft_trace_object()` in their SFT implementations,
+the plan can use `SFTProcessEdges`. Currently most policies provide an implementation for `sft_trace_object()`, except
+mark compact and immix. Those two policies use multiple GC traces, and due to the limitation of SFT, SFT does not allow
+multiple `sft_trace_object()` for a policy.
+
+`SFTProcessEdges` is the simplest approach when all the policies support it. Fortunately, we can use it for our GC, semispace.
+
+### Approach 2: Derive `PlanTraceObject` and use `PlanProcessEdges`
+
+`PlanProcessEdges` is another general `ProcessEdgesWork` implementation that can be used by most plans. When a plan
+implements the [`PlanTraceObject`](https://www.mmtk.io/mmtk-core/mmtk/plan/transitive_closure/trait.PlanTraceObject.html),
+it can use `PlanProcessEdges`.
+
+You can manually provide an implementation of `PlanTraceObject` for `MyGC`. But you can also use the derive macro MMTK provides,
+and the macro will generate an implementation of `PlanTraceObject`:
+* add `#[derive(PlanTraceObject)]` for `MyGC` (import the macro properly: `use mmtk_macros::PlanTraceObject`)
+* add `#[trace(CopySemantics::Default)]` to both copy space fields, `copyspace0` and `copyspace1`. This tells the macro to generate
+  trace code for both spaces, and for any copying in the spaces, use `CopySemantics::DefaultCopy` that we have configured early.
+* add `#[fallback_trace]` to `common`. This tells the macro that if an object is not found in any space with `#[trace]` in ths plan,
+  try find the space for the object in the 'parent' plan. In our case, we fall back to the `CommonPlan`, as the object may be
+  in large object space or immortal space in the common plan. `CommonPlan` also implements `PlanTraceObject`, so it knows how to
+  find a space for the object and trace it in the same way.
+
+With the derive macro, your `MyGC` struct should look like this:
+```rust
+{{#include ../../../code/mygc_semispace/global.rs:plan_def}}
+```
+
+Once this is done, you can specify `PlanProcessEdges` as the `ProcessEdgesWorkType` in your GC work context:
+```rust
+{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_plan}}
+```
+
+### Approach 3: Implement your own `ProcessEdgesWork`
+
+Apart from the two approaches above, you can always implement your own `ProcessEdgesWork`. This is
+an overkill for simple plans like semi space, but might be necessary for more complex plans.
+We discuss how to implement it for `MyGC`.
+
+Create a struct `MyGCProcessEdges<VM: VMBinding>` in the `gc_work` module. It includes a reference
+back to the plan, and a `ProcessEdgesBase` field:
+```rust
+{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges}}
+```
+
+Implement `ProcessEdgesWork` for `MyGCProcessEdges`. As most methods in the trait have a default
+implemetation, we only need to implement `new()` and `trace_object()` for our plan. However, this
+may not be true when you implement it for other GC plans. It would be better to check the default
+implementation of `ProcessEdgesWork`.
+
+For `trace_object()`, what we do is similar to the approach above (except that we need to write the code
+ourselves rather than letting the macro to generate it for us). We try to figure out
+which space the object is in, and invoke `trace_object()` for the object on that space. If the
+object is not in any of the semi spaces in the plan, we forward the call to `CommonPlan`.
+```rust
+{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges_impl}}
+```
+
+We would also need to implement `Deref` and `DerefMut` to our `ProcessEdgesWork` impl to be
+dereferenced as `ProcessEdgesBase`.
+```rust
+{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges_deref}}
+```
+
+In the end, use `MyGCProcessEdges` as `ProcessEdgesWorkType` in the `GCWorkContext`:
+```rust
+{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_mygc}}
+```
+
 ## Summary
 
 You should now have MyGC working and able to collect garbage. All three

macros/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "mmtk-macros"
+# the macro crate uses the same version as mmtk-core
+version = "0.11.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+description = "MMTk macros provides procedural macros used by mmtk-core."
+homepage = "https://www.mmtk.io"
+repository = "https://github.com/mmtk/mmtk-core/tree/master/macros"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1.0.37"
+syn = { version = "1.0.91", features = ["extra-traits"] }
+quote = "1.0.18"
+proc-macro-error = "1.0.4"

macros/src/lib.rs

@@ -0,0 +1,68 @@
+extern crate proc_macro;
+extern crate syn;
+extern crate proc_macro_error;
+extern crate quote;
+
+use proc_macro::TokenStream;
+use proc_macro_error::proc_macro_error;
+use syn::{parse_macro_input};
+use proc_macro_error::abort_call_site;
+use quote::quote;
+use syn::DeriveInput;
+
+mod util;
+mod plan_trace_object_impl;
+
+const DEBUG_MACRO_OUTPUT: bool = false;
+
+/// Generally a plan needs to add these attributes in order for the macro to work. The macro will
+/// generate an implementation of `PlanTraceObject` for the plan. With `PlanTraceObject`, the plan use
+/// `PlanProcessEdges` for GC tracing. The attributes only affects code generation in the macro, thus
+/// only affects the generated `PlanTraceObject` implementation.
+/// * add `#[derive(PlanTraceObject)]` to the plan struct.
+/// * add `#[trace]` to each space field the plan struct has. If the policy is a copying policy,
+///   it needs to further specify the copy semantic (`#[trace(CopySemantics::X)]`)
+/// * add `#[fallback_trace]` to the parent plan if the plan is composed with other plans (or parent plans).
+///   For example, `GenImmix` is composed with `Gen`, `Gen` is composed with `CommonPlan`, `CommonPlan` is composed
+///   with `BasePlan`.
+/// * add `#[post_scan]` to any space field that has some policy-specific post_scan_object(). For objects in those spaces,
+///   `post_scan_object()` in the policy will be called after `VM::VMScanning::scan_object()`.
+#[proc_macro_error]
+#[proc_macro_derive(PlanTraceObject, attributes(trace, post_scan, fallback_trace))]
+pub fn derive_plan_trace_object(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    let ident = input.ident;
+    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+
+    let output = if let syn::Data::Struct(syn::DataStruct {
+        fields: syn::Fields::Named(ref fields),
+        ..
+    }) = input.data {
+        let spaces = util::get_fields_with_attribute(fields, "trace");
+        let post_scan_spaces = util::get_fields_with_attribute(fields, "post_scan");
+        let fallback = util::get_unique_field_with_attribute(fields, "fallback_trace");
+
+        let trace_object_function = plan_trace_object_impl::generate_trace_object(&spaces, &fallback, &ty_generics);
+        let post_scan_object_function = plan_trace_object_impl::generate_post_scan_object(&post_scan_spaces, &ty_generics);
+        let may_move_objects_function = plan_trace_object_impl::generate_may_move_objects(&spaces, &fallback, &ty_generics);
+        quote!{
+            impl #impl_generics crate::plan::PlanTraceObject #ty_generics for #ident #ty_generics #where_clause {
+                #trace_object_function
+
+                #post_scan_object_function
+
+                #may_move_objects_function
+            }
+        }
+    } else {
+        abort_call_site!("`#[derive(PlanTraceObject)]` only supports structs with named fields.")
+    };
+
+    // Debug the output - use the following code to debug the generated code (when cargo exapand is not working)
+    if DEBUG_MACRO_OUTPUT {
+        use quote::ToTokens;
+        println!("{}", output.to_token_stream());
+    }
+
+    output.into()
+}