cancel theap on multi-ractors

accessing theap needs complicating synchronization but it reduce
performance on multi-ractor mode. So simply stop using theap
on multi-ractor mode. In future, theap should be replaced with
more cleaver memory strategy.

common.mk

@@ -10384,6 +10384,7 @@ ractor.$(OBJEXT): {$(VPATH)}subst.h
 ractor.$(OBJEXT): {$(VPATH)}thread.h
 ractor.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h
 ractor.$(OBJEXT): {$(VPATH)}thread_native.h
+ractor.$(OBJEXT): {$(VPATH)}transient_heap.h
 ractor.$(OBJEXT): {$(VPATH)}variable.h
 ractor.$(OBJEXT): {$(VPATH)}vm_core.h
 ractor.$(OBJEXT): {$(VPATH)}vm_debug.h

ractor.c

@@ -14,6 +14,7 @@
 #include "internal/struct.h"
 #include "variable.h"
 #include "gc.h"
+#include "transient_heap.h"
 
 VALUE rb_cRactor;
 static VALUE rb_eRactorError;
@@ -1127,15 +1128,32 @@ ractor_next_id(void)
 }
 
 static void
-vm_insert_ractor0(rb_vm_t *vm, rb_ractor_t *r)
+vm_insert_ractor0(rb_vm_t *vm, rb_ractor_t *r, bool single_ractor_mode)
 {
     RUBY_DEBUG_LOG("r:%u ractor.cnt:%u++", r->id, vm->ractor.cnt);
-    VM_ASSERT(!rb_multi_ractor_p() || RB_VM_LOCKED_P());
+    VM_ASSERT(single_ractor_mode || RB_VM_LOCKED_P());
 
     list_add_tail(&vm->ractor.set, &r->vmlr_node);
     vm->ractor.cnt++;
 }
 
+static void
+cancel_single_ractor_mode(void)
+{
+    // enable multi-ractor mode
+    RUBY_DEBUG_LOG("enable multi-ractor mode", 0);
+
+    rb_gc_start();
+    rb_transient_heap_evacuate();
+
+    if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) {
+        rb_warn("Ractor is experimental, and the behavior may change in future versions of Ruby! "
+                "Also there are many implementation issues.");
+    }
+
+    ruby_single_main_ractor = NULL;
+}
+
 static void
 vm_insert_ractor(rb_vm_t *vm, rb_ractor_t *r)
 {
@@ -1144,29 +1162,22 @@ vm_insert_ractor(rb_vm_t *vm, rb_ractor_t *r)
     if (rb_multi_ractor_p()) {
         RB_VM_LOCK();
         {
-            vm_insert_ractor0(vm, r);
+            vm_insert_ractor0(vm, r, false);
             vm_ractor_blocking_cnt_inc(vm, r, __FILE__, __LINE__);
         }
         RB_VM_UNLOCK();
     }
     else {
-        vm_insert_ractor0(vm, r);
-
-        if (vm->ractor.cnt == 1) {
+        if (vm->ractor.cnt == 0) {
             // main ractor
+            vm_insert_ractor0(vm, r, true);
             ractor_status_set(r, ractor_blocking);
             ractor_status_set(r, ractor_running);
         }
         else {
+            cancel_single_ractor_mode();
+            vm_insert_ractor0(vm, r, true);
             vm_ractor_blocking_cnt_inc(vm, r, __FILE__, __LINE__);
-
-            // enable multi-ractor mode
-            RUBY_DEBUG_LOG("enable multi-ractor mode", 0);
-            ruby_single_main_ractor = NULL;
-
-            if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) {
-                rb_warn("Ractor is experimental, and the behavior may change in future versions of Ruby! Also there are many implementation issues.");
-            }
         }
     }
 }

transient_heap.c

@@ -364,10 +364,10 @@ transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
 void *
 rb_transient_heap_alloc(VALUE obj, size_t req_size)
 {
-    void *ret;
+    // only on single main ractor
+    if (ruby_single_main_ractor == NULL) return NULL;
 
-    RB_VM_LOCK_ENTER();
-    {
+    void *ret;
     struct transient_heap* theap = transient_heap_get();
     size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
 
@@ -430,8 +430,6 @@ rb_transient_heap_alloc(VALUE obj, size_t req_size)
             ret = NULL;
         }
     }
-    }
-    RB_VM_LOCK_LEAVE();
 
     return ret;
 }
@@ -774,17 +772,17 @@ transient_heap_update_status(struct transient_heap* theap, enum transient_heap_s
 
 static void
 transient_heap_evacuate(void *dmy)
-{
-    RB_VM_LOCK_ENTER();
-    rb_vm_barrier();
 {
     struct transient_heap* theap = transient_heap_get();
 
+    if (theap->total_marked_objects == 0) return;
+    if (ruby_single_main_ractor == NULL) rb_bug("not single ractor mode");
     if (theap->status == transient_heap_marking) {
         if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
     }
     else {
         VALUE gc_disabled = rb_gc_disable_no_rest();
+        {
             struct transient_heap_block* block;
 
             RUBY_DEBUG_LOG("start gc_disabled:%d", RTEST(gc_disabled));
@@ -825,12 +823,16 @@ transient_heap_evacuate(void *dmy)
 
             transient_heap_verify(theap);
             transient_heap_update_status(theap, transient_heap_none);
-
+        }
         if (gc_disabled != Qtrue) rb_gc_enable();
         RUBY_DEBUG_LOG("finish", 0);
     }
 }
-    RB_VM_LOCK_LEAVE();
+
+void
+rb_transient_heap_evacuate(void)
+{
+    transient_heap_evacuate(NULL);
 }
 
 static void
@@ -964,7 +966,7 @@ rb_transient_heap_finish_marking(void)
 
     struct transient_heap* theap = transient_heap_get();
 
-    if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",
+    RUBY_DEBUG_LOG("objects:%d, marked:%d",
                    theap->total_objects,
                    theap->total_marked_objects);
     if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);

transient_heap.h

@@ -31,6 +31,9 @@ void rb_transient_heap_start_marking(int full_marking);
 void rb_transient_heap_finish_marking(void);
 void rb_transient_heap_update_references(void);
 
+/* used by ractor.c */
+void rb_transient_heap_evacuate(void);
+
 /* for debug API */
 void rb_transient_heap_dump(void);
 void rb_transient_heap_verify(void);
@@ -49,6 +52,7 @@ void rb_struct_transient_heap_evacuate(VALUE st, int promote);
 #define rb_transient_heap_promote(obj) ((void)0)
 #define rb_transient_heap_start_marking(full_marking) ((void)0)
 #define rb_transient_heap_update_references() ((void)0)
+#define rb_transient_heap_evacuate() ((void)0)
 #define rb_transient_heap_finish_marking() ((void)0)
 #define rb_transient_heap_mark(obj, ptr) ((void)0)