Add RVALUE_OVERHEAD and move ractor_belonging_id

This commit adds RVALUE_OVERHEAD for storing metadata at the end of the
slot. This commit moves the ractor_belonging_id in debug builds from the
flags to RVALUE_OVERHEAD which frees the 16 bits in the headers for
object shapes.

gc.c

@@ -621,17 +621,29 @@ typedef struct RVALUE {
             VALUE v3;
         } values;
     } as;
+
+    /* Start of RVALUE_OVERHEAD.
+     * Do not directly read these members from the RVALUE as they're located
+     * at the end of the slot (which may differ in size depending on the size
+     * pool). */
+#if RACTOR_CHECK_MODE
+    uint32_t _ractor_belonging_id;
+#endif
 #if GC_DEBUG
     const char *file;
     int line;
 #endif
 } RVALUE;
 
-#if GC_DEBUG
+#if RACTOR_CHECK_MODE
-STATIC_ASSERT(sizeof_rvalue, offsetof(RVALUE, file) == SIZEOF_VALUE * 5);
+# define RVALUE_OVERHEAD (sizeof(RVALUE) - offsetof(RVALUE, _ractor_belonging_id))
+#elif GC_DEBUG
+# define RVALUE_OVERHEAD (sizeof(RVALUE) - offsetof(RVALUE, file))
 #else
-STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == SIZEOF_VALUE * 5);
+# define RVALUE_OVERHEAD 0
 #endif
+
+STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == (SIZEOF_VALUE * 5) + RVALUE_OVERHEAD);
 STATIC_ASSERT(alignof_rvalue, RUBY_ALIGNOF(RVALUE) == SIZEOF_VALUE);
 
 typedef uintptr_t bits_t;
@@ -2576,7 +2588,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
 size_t
 rb_gc_obj_slot_size(VALUE obj)
 {
-    return GET_HEAP_PAGE(obj)->slot_size;
+    return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
 }
 
 static inline size_t
@@ -2591,6 +2603,8 @@ size_pool_slot_size(unsigned char pool_id)
     GC_ASSERT(size_pools[pool_id].slot_size == (short)slot_size);
 #endif
 
+    slot_size -= RVALUE_OVERHEAD;
+
     return slot_size;
 }
 
@@ -2705,6 +2719,8 @@ static inline size_t
 size_pool_idx_for_size(size_t size)
 {
 #if USE_RVARGC
+    size += RVALUE_OVERHEAD;
+
     size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
 
     /* size_pool_idx is ceil(log2(slot_count)) */
@@ -2914,7 +2930,7 @@ rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flag
 VALUE
 rb_newobj(void)
 {
-    return newobj_of(0, T_NONE, 0, 0, 0, FALSE, sizeof(RVALUE));
+    return newobj_of(0, T_NONE, 0, 0, 0, FALSE, RVALUE_SIZE);
 }
 
 static size_t
@@ -2966,7 +2982,7 @@ rb_newobj_of(VALUE klass, VALUE flags)
         return rb_class_instance_allocate_internal(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED, flags & FL_WB_PROTECTED);
     }
     else {
-        return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, sizeof(RVALUE));
+        return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, RVALUE_SIZE);
     }
 }
 
@@ -3004,7 +3020,7 @@ rb_imemo_name(enum imemo_type type)
 VALUE
 rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
 {
-    size_t size = sizeof(RVALUE);
+    size_t size = RVALUE_SIZE;
     VALUE flags = T_IMEMO | (type << FL_USHIFT);
     return newobj_of(v0, flags, v1, v2, v3, TRUE, size);
 }
@@ -3012,7 +3028,7 @@ rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
 static VALUE
 rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0)
 {
-    size_t size = sizeof(RVALUE);
+    size_t size = sizeof(struct rb_imemo_tmpbuf_struct);
     VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT);
     return newobj_of(v0, flags, v1, v2, v3, FALSE, size);
 }
@@ -3093,7 +3109,7 @@ rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FU
 {
     RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1);
     if (klass) rb_data_object_check(klass);
-    return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE, sizeof(RVALUE));
+    return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE, sizeof(struct RTypedData));
 }
 
 VALUE
@@ -3109,7 +3125,7 @@ rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
 {
     RBIMPL_NONNULL_ARG(type);
     if (klass) rb_data_object_check(klass);
-    return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED, sizeof(RVALUE));
+    return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED, sizeof(struct RTypedData));
 }
 
 VALUE
@@ -4974,7 +4990,7 @@ obj_memsize_of(VALUE obj, int use_all_types)
                BUILTIN_TYPE(obj), (void*)obj);
     }
 
-    return size + GET_HEAP_PAGE(obj)->slot_size;
+    return size + rb_gc_obj_slot_size(obj);
 }
 
 size_t
@@ -9883,6 +9899,14 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s
 
     /* Move the object */
     memcpy(dest, src, MIN(src_slot_size, slot_size));
+
+    if (RVALUE_OVERHEAD > 0) {
+        void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
+        void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
+
+        memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
+    }
+
     memset(src, 0, src_slot_size);
 
     /* Set bits for object in new location */
@@ -10005,7 +10029,7 @@ gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
         }
 
 #if USE_RVARGC
-        if ((size_t)GET_HEAP_PAGE(v)->slot_size >= rb_ary_size_as_embedded(v)) {
+        if (rb_gc_obj_slot_size(v) >= rb_ary_size_as_embedded(v)) {
             if (rb_ary_embeddable_p(v)) {
                 rb_ary_make_embedded(v);
             }
@@ -10516,7 +10540,7 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
 
                 // if, after move the string is not embedded, and can fit in the
                 // slot it's been placed in, then re-embed it
-                if ((size_t)GET_HEAP_PAGE(obj)->slot_size >= rb_str_size_as_embedded(obj)) {
+                if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
                     if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) {
                         rb_str_make_embedded(obj);
                     }
@@ -14339,7 +14363,8 @@ Init_GC(void)
 
     gc_constants = rb_hash_new();
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), RBOOL(GC_DEBUG));
-    rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE));
+    rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
+    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(RVALUE)));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));

ractor_core.h

@@ -288,6 +288,8 @@ rb_ractor_id(const rb_ractor_t *r)
 }
 
 #if RACTOR_CHECK_MODE > 0
+# define RACTOR_BELONGING_ID(obj) (*(uint32_t *)(((uintptr_t)(obj)) + rb_gc_obj_slot_size(obj)))
+
 uint32_t rb_ractor_current_id(void);
 // If ractor check mode is enabled, shape bits needs to be smaller
 STATIC_ASSERT(shape_bits, SHAPE_ID_NUM_BITS == 16);
@@ -295,8 +297,7 @@ STATIC_ASSERT(shape_bits, SHAPE_ID_NUM_BITS == 16);
 static inline void
 rb_ractor_setup_belonging_to(VALUE obj, uint32_t rid)
 {
-    VALUE flags = RBASIC(obj)->flags & 0xffff0000ffffffff; // 4B
+    RACTOR_BELONGING_ID(obj) = rid;
-    RBASIC(obj)->flags = flags | ((VALUE)rid << 32);
 }
 
 static inline void
@@ -312,7 +313,7 @@ rb_ractor_belonging(VALUE obj)
         return 0;
     }
     else {
-        return RBASIC(obj)->flags >> 32 & 0xFFFF;
+        return RACTOR_BELONGING_ID(obj);
     }
 }
 

test/objspace/test_objspace.rb

@@ -480,8 +480,8 @@ class TestObjSpace < Test::Unit::TestCase
         obj = JSON.parse(l)
         next if obj["type"] == "ROOT"
 
-        assert(obj["slot_size"] != nil)
+        assert_not_nil obj["slot_size"]
-        assert(obj["slot_size"] % GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] == 0)
+        assert_equal 0, obj["slot_size"] % GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
       }
     end
   end

test/ruby/test_gc.rb

@@ -152,7 +152,7 @@ class TestGc < Test::Unit::TestCase
       GC.stat_heap(i, stat_heap)
       GC.stat(stat)
 
-      assert_equal GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] * (2**i), stat_heap[:slot_size]
+      assert_equal GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] * (2**i), stat_heap[:slot_size]
       assert_operator stat_heap[:heap_allocatable_pages], :<=, stat[:heap_allocatable_pages]
       assert_operator stat_heap[:heap_eden_pages], :<=, stat[:heap_eden_pages]
       assert_operator stat_heap[:heap_eden_slots], :>=, 0

test/ruby/test_objectspace.rb

@@ -67,7 +67,7 @@ End
 
   def test_id2ref_invalid_symbol_id
     msg = /is not symbol id value/
-    assert_raise_with_message(RangeError, msg) { ObjectSpace._id2ref(:a.object_id + 40) }
+    assert_raise_with_message(RangeError, msg) { ObjectSpace._id2ref(:a.object_id + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]) }
   end
 
   def test_count_objects