1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow newobj_of0 and newobj_slowpath to allocate into multiple heap slots

Browse Source
This commit is contained in:
Matt Valentine-House 2021-03-30 13:34:14 +01:00 committed by Peter Zhu
parent ddc29e2989
commit 8bbd319806
Notes: git 2021-05-06 22:18:49 +09:00 
Merged: https://github.com/ruby/ruby/pull/4391
5 changed files with 436 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ext/objspace/objspace.c
View File

@ -66,6 +66,7 @@ total_i(VALUE v, void *ptr)
case T_IMEMO: case T_IMEMO:
case T_ICLASS: case T_ICLASS:
case T_NODE: case T_NODE:
case T_PAYLOAD:
case T_ZOMBIE: case T_ZOMBIE:
return; return;
default: default:
@ -224,6 +225,7 @@ type2sym(enum ruby_value_type i)
CASE_TYPE(T_ICLASS); CASE_TYPE(T_ICLASS);
CASE_TYPE(T_MOVED); CASE_TYPE(T_MOVED);
CASE_TYPE(T_ZOMBIE); CASE_TYPE(T_ZOMBIE);
CASE_TYPE(T_PAYLOAD);
#undef CASE_TYPE #undef CASE_TYPE
default: rb_bug("type2sym: unknown type (%d)", i); default: rb_bug("type2sym: unknown type (%d)", i);
} }

449
gc.c
View File

@ -551,6 +551,7 @@ typedef struct gc_profile_record {
} gc_profile_record; } gc_profile_record;
#define FL_FROM_FREELIST FL_USER0 #define FL_FROM_FREELIST FL_USER0
#define FL_FROM_PAYLOAD FL_USER0
struct RMoved { struct RMoved {
VALUE flags; VALUE flags;
@ -564,12 +565,30 @@ struct RMoved {
#pragma pack(push, 4) /* == SIZEOF_VALUE: magic for reducing sizeof(RVALUE): 24 -> 20 */ #pragma pack(push, 4) /* == SIZEOF_VALUE: magic for reducing sizeof(RVALUE): 24 -> 20 */
#endif #endif
struct RPayload {
VALUE flags;
};
#define RPAYLOAD(obj) ((struct RPayload *)obj)
static unsigned short
RPAYLOAD_LEN(VALUE obj) {
unsigned short len = (unsigned short)(RPAYLOAD(obj)->flags >> FL_USHIFT);
return len;
}
static void
RPAYLOAD_FLAGS_SET(VALUE obj, unsigned short len)
{
// as len is the only thing in the user bits, we can overwrite it every time
RPAYLOAD(obj)->flags = T_PAYLOAD | (len << FL_USHIFT);
}
typedef struct RVALUE { typedef struct RVALUE {
union { union {
struct { struct {
VALUE flags; /* always 0 for freed obj */ VALUE flags; /* always 0 for freed obj */
struct RVALUE *next; struct RVALUE *next;
} free; } free;
struct RPayload payload;
struct RMoved moved; struct RMoved moved;
struct RBasic basic; struct RBasic basic;
struct RObject object; struct RObject object;
@ -1266,6 +1285,35 @@ RVALUE_FLAGS_AGE(VALUE flags)
return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT); return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
} }
#if USE_RVARGC
static VALUE
payload_or_self(VALUE obj)
{
struct heap_page *p = GET_HEAP_PAGE(obj);
VALUE cur = (VALUE)p->start;
while (cur != obj && GET_HEAP_PAGE(cur) == p) {
VALUE p = cur;
void *poisoned = asan_poisoned_object_p((VALUE)p);
asan_unpoison_object((VALUE)p, false);
if (BUILTIN_TYPE(cur) == T_PAYLOAD) {
if (cur < obj && obj < cur + RPAYLOAD_LEN(cur) * sizeof(RVALUE)) {
return cur;
}
cur += RPAYLOAD_LEN(cur) * sizeof(RVALUE);
} else {
cur += sizeof(RVALUE);
}
if (poisoned) {
asan_poison_object((VALUE)p);
}
}
return obj;
}
#endif
static int static int
check_rvalue_consistency_force(const VALUE obj, int terminate) check_rvalue_consistency_force(const VALUE obj, int terminate)
{ {
@ -1473,6 +1521,18 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag
{ {
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++; objspace->rgengc.old_objects++;
#if USE_RVARGC
if (BUILTIN_TYPE(obj) == T_PAYLOAD) {
int plen = RPAYLOAD_LEN(obj);
for (int i = 1; i < plen; i++) {
VALUE pbody = obj + i * sizeof(RVALUE);
MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(pbody), pbody);
}
objspace->rgengc.old_objects += plen - 1;
}
#endif
rb_transient_heap_promote(obj); rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2 #if RGENGC_PROFILE >= 2
@ -1562,6 +1622,11 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
if (RVALUE_MARKED(obj)) { if (RVALUE_MARKED(obj)) {
objspace->rgengc.old_objects--; objspace->rgengc.old_objects--;
#if USE_RVARGC
if (BUILTIN_TYPE(obj) == T_PAYLOAD) {
objspace->rgengc.old_objects -= RPAYLOAD_LEN(obj) - 1;
}
#endif
} }
check_rvalue_consistency(obj); check_rvalue_consistency(obj);
@ -2203,10 +2268,144 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
return obj; return obj;
} }
static inline VALUE static unsigned long
ractor_cached_freeobj(rb_objspace_t *objspace, rb_ractor_t *cr) rvargc_slot_count(size_t size)
{ {
// roomof == ceiling division, so we don't have to do div then mod
return roomof(size + sizeof(struct RPayload), sizeof(RVALUE));
}
static RVALUE *
rvargc_find_contiguous_slots(int slots, RVALUE *freelist)
{
RVALUE *cursor = freelist;
RVALUE *previous_region = NULL;
while(cursor) {
int i;
RVALUE *search = cursor;
for (i = 0; i < (slots - 1); i++) {
// Peek ahead to see if the region is contiguous
if (search->as.free.next == (search - 1)) {
search = search->as.free.next;
} else {
// Next slot is not contiguous
if (search->as.free.next) {
cursor = search->as.free.next;
previous_region = search;
break;
} else {
// Hit the end of the free list
return NULL;
}
}
}
if (i == slots - 1) {
if (previous_region) {
previous_region->as.free.next = search->as.free.next;
search->as.free.next = freelist;
}
return search;
}
}
rb_bug("rvargc_find_contiguous_slots: unreachable");
}
static inline bool heap_add_freepage(rb_heap_t *heap, struct heap_page *page);
static struct heap_page * heap_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap);
static inline void ractor_set_cache(rb_ractor_t *cr, struct heap_page *page);
static inline void *
rvargc_find_region(size_t size, rb_ractor_t *cr, RVALUE *freelist)
{
// maintain master behaviour when we only need one slot
if (size == sizeof(RVALUE))
return freelist;
if (!freelist) return freelist;
rb_objspace_t *objspace = &rb_objspace;
int slots = (int)rvargc_slot_count(size);
RVALUE * p = rvargc_find_contiguous_slots(slots, freelist);
// We found a contiguous space on the freelist stored in the ractor cache
if (p) {
struct heap_page *page = GET_HEAP_PAGE(p);
page->free_slots -= slots;
asan_unpoison_memory_region(p, sizeof(RVALUE) * slots, false);
return p;
}
else {
struct heap_page *search_page;
heap_allocatable_pages_set(objspace, heap_allocatable_pages + 1);
while (!p) {
// search_page is the page we're going to search for contiguous slots
search_page = heap_next_freepage(objspace, heap_eden);
p = rvargc_find_contiguous_slots(slots, search_page->freelist);
if (p) {
// Remove the region from the freelist
search_page->freelist = p->as.free.next;
search_page->free_slots -= slots;
// If we started sweeping, the object cache can be removed
// from the ractor. Set it to the page we found
if (!cr->newobj_cache.using_page) {
ractor_set_cache(cr, search_page);
}
// Otherwise we need to add this page back to the list of free
// pages.
else {
// make this pointer point at the Ractor's freelist
p->as.free.next = freelist;
}
asan_unpoison_memory_region(p, sizeof(RVALUE) * slots, false);
return p;
}
}
}
return NULL;
}
int
rb_slot_size()
{
return sizeof(RVALUE);
}
VALUE
rb_rvargc_payload_init(VALUE obj, size_t size)
{
rb_objspace_t * objspace = &rb_objspace;
struct RPayload *ph = (struct RPayload *)obj;
memset(ph, 0, rvargc_slot_count(size) * sizeof(RVALUE));
RPAYLOAD_FLAGS_SET((VALUE)ph, rvargc_slot_count(size));
objspace->total_allocated_objects += rvargc_slot_count(size);
return (VALUE)ph;
}
void *
rb_rvargc_payload_data_ptr(VALUE phead)
{
return (void *)(phead + sizeof(struct RPayload));
}
static inline VALUE
ractor_cached_free_region(rb_objspace_t *objspace, rb_ractor_t *cr, size_t size)
{
#if USE_RVARGC
RVALUE *p = rvargc_find_region(size, cr, cr->newobj_cache.freelist);
#else
RVALUE *p = cr->newobj_cache.freelist; RVALUE *p = cr->newobj_cache.freelist;
#endif
if (p) { if (p) {
VALUE obj = (VALUE)p; VALUE obj = (VALUE)p;
@ -2241,13 +2440,9 @@ heap_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
} }
static inline void static inline void
ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr) ractor_set_cache(rb_ractor_t *cr, struct heap_page *page)
{ {
ASSERT_vm_locking(); gc_report(3, &rb_objspace, "ractor_set_cache: Using page %p\n", (void *)GET_PAGE_BODY(page->start));
GC_ASSERT(cr->newobj_cache.freelist == NULL);
struct heap_page *page = heap_next_freepage(objspace, heap_eden);
cr->newobj_cache.using_page = page; cr->newobj_cache.using_page = page;
cr->newobj_cache.freelist = page->freelist; cr->newobj_cache.freelist = page->freelist;
page->free_slots = 0; page->free_slots = 0;
@ -2258,6 +2453,16 @@ ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr)
asan_poison_object((VALUE)cr->newobj_cache.freelist); asan_poison_object((VALUE)cr->newobj_cache.freelist);
} }
static inline void
ractor_cache_slots(rb_objspace_t *objspace, rb_ractor_t *cr)
{
ASSERT_vm_locking();
struct heap_page *page = heap_next_freepage(objspace, heap_eden);
ractor_set_cache(cr, page);
}
static inline VALUE static inline VALUE
newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3) newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
{ {
@ -2268,10 +2473,10 @@ newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
return obj; return obj;
} }
ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected)); ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected, size_t alloc_size));
static inline VALUE static inline VALUE
newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected) newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected, size_t alloc_size)
{ {
VALUE obj; VALUE obj;
unsigned int lev; unsigned int lev;
@ -2293,11 +2498,16 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *
} }
// allocate new slot // allocate new slot
while ((obj = ractor_cached_freeobj(objspace, cr)) == Qfalse) { while ((obj = ractor_cached_free_region(objspace, cr, alloc_size)) == Qfalse) {
ractor_cache_slots(objspace, cr); ractor_cache_slots(objspace, cr);
} }
GC_ASSERT(obj != 0); GC_ASSERT(obj != 0);
newobj_init(klass, flags, wb_protected, objspace, obj); newobj_init(klass, flags, wb_protected, objspace, obj);
#if USE_RVARGC
if (alloc_size > sizeof(RVALUE))
rb_rvargc_payload_init(obj + sizeof(RVALUE), alloc_size - sizeof(RVALUE));
#endif
gc_event_hook_prep(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj, newobj_fill(obj, 0, 0, 0)); gc_event_hook_prep(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj, newobj_fill(obj, 0, 0, 0));
} }
RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev); RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev);
@ -2306,24 +2516,24 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *
} }
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, rb_ractor_t *cr)); rb_objspace_t *objspace, rb_ractor_t *cr, size_t alloc_size));
NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, rb_ractor_t *cr)); rb_objspace_t *objspace, rb_ractor_t *cr, size_t alloc_size));
static VALUE static VALUE
newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr) newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, size_t alloc_size)
{ {
return newobj_slowpath(klass, flags, objspace, cr, TRUE); return newobj_slowpath(klass, flags, objspace, cr, TRUE, alloc_size);
} }
static VALUE static VALUE
newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr) newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, size_t alloc_size)
{ {
return newobj_slowpath(klass, flags, objspace, cr, FALSE); return newobj_slowpath(klass, flags, objspace, cr, FALSE, alloc_size);
} }
static inline VALUE static inline VALUE
newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr) newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr, size_t alloc_size)
{ {
VALUE obj; VALUE obj;
rb_objspace_t *objspace = &rb_objspace; rb_objspace_t *objspace = &rb_objspace;
@ -2344,54 +2554,61 @@ newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr)
ruby_gc_stressful || ruby_gc_stressful ||
gc_event_hook_available_p(objspace)) && gc_event_hook_available_p(objspace)) &&
wb_protected && wb_protected &&
(obj = ractor_cached_freeobj(objspace, cr)) != Qfalse)) { (obj = ractor_cached_free_region(objspace, cr, alloc_size)) != Qfalse)) {
newobj_init(klass, flags, wb_protected, objspace, obj); newobj_init(klass, flags, wb_protected, objspace, obj);
#if USE_RVARGC
if (alloc_size > sizeof(RVALUE))
rb_rvargc_payload_init(obj + sizeof(RVALUE), alloc_size - sizeof(RVALUE));
#endif
} }
else { else {
RB_DEBUG_COUNTER_INC(obj_newobj_slowpath); RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
obj = wb_protected ? obj = wb_protected ?
newobj_slowpath_wb_protected(klass, flags, objspace, cr) : newobj_slowpath_wb_protected(klass, flags, objspace, cr, alloc_size) :
newobj_slowpath_wb_unprotected(klass, flags, objspace, cr); newobj_slowpath_wb_unprotected(klass, flags, objspace, cr, alloc_size);
} }
return obj; return obj;
} }
static inline VALUE static inline VALUE
newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected) newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t alloc_size)
{ {
VALUE obj = newobj_of0(klass, flags, wb_protected, GET_RACTOR()); VALUE obj = newobj_of0(klass, flags, wb_protected, GET_RACTOR(), alloc_size);
return newobj_fill(obj, v1, v2, v3); return newobj_fill(obj, v1, v2, v3);
} }
static inline VALUE static inline VALUE
newobj_of_cr(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected) newobj_of_cr(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t alloc_size)
{ {
VALUE obj = newobj_of0(klass, flags, wb_protected, cr); VALUE obj = newobj_of0(klass, flags, wb_protected, cr, alloc_size);
return newobj_fill(obj, v1, v2, v3); return newobj_fill(obj, v1, v2, v3);
} }
VALUE VALUE
rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags) rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags, size_t size)
{ {
GC_ASSERT((flags & FL_WB_PROTECTED) == 0); GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, FALSE); size = size + sizeof(RVALUE);
return newobj_of(klass, flags, 0, 0, 0, FALSE, size);
} }
VALUE VALUE
rb_wb_protected_newobj_of(VALUE klass, VALUE flags) rb_wb_protected_newobj_of(VALUE klass, VALUE flags, size_t size)
{ {
GC_ASSERT((flags & FL_WB_PROTECTED) == 0); GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, TRUE); size = size + sizeof(RVALUE);
return newobj_of(klass, flags, 0, 0, 0, TRUE, size);
} }
VALUE VALUE
rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags) rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
{ {
GC_ASSERT((flags & FL_WB_PROTECTED) == 0); GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of_cr(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE); size = size + sizeof(RVALUE);
return newobj_of_cr(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE, size);
} }
/* for compatibility */ /* for compatibility */
@ -2399,7 +2616,7 @@ rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flag
VALUE VALUE
rb_newobj(void) rb_newobj(void)
{ {
return newobj_of(0, T_NONE, 0, 0, 0, FALSE); return newobj_of(0, T_NONE, 0, 0, 0, FALSE, sizeof(RVALUE));
} }
VALUE VALUE
@ -2408,7 +2625,7 @@ rb_newobj_of(VALUE klass, VALUE flags)
if ((flags & RUBY_T_MASK) == T_OBJECT) { if ((flags & RUBY_T_MASK) == T_OBJECT) {
st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass); st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
VALUE obj = newobj_of(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED , Qundef, Qundef, Qundef, flags & FL_WB_PROTECTED); VALUE obj = newobj_of(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED , Qundef, Qundef, Qundef, flags & FL_WB_PROTECTED, sizeof(RVALUE));
if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) { if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
rb_init_iv_list(obj); rb_init_iv_list(obj);
@ -2416,7 +2633,7 @@ rb_newobj_of(VALUE klass, VALUE flags)
return obj; return obj;
} }
else { else {
return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED); return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, sizeof(RVALUE));
} }
} }
@ -2454,15 +2671,17 @@ rb_imemo_name(enum imemo_type type)
VALUE VALUE
rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0) rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{ {
size_t size = sizeof(RVALUE);
VALUE flags = T_IMEMO | (type << FL_USHIFT); VALUE flags = T_IMEMO | (type << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, TRUE); return newobj_of(v0, flags, v1, v2, v3, TRUE, size);
} }
static VALUE static VALUE
rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0) rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{ {
size_t size = sizeof(RVALUE);
VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT); VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, FALSE); return newobj_of(v0, flags, v1, v2, v3, FALSE, size);
} }
static VALUE static VALUE
@ -2528,7 +2747,7 @@ rb_class_allocate_instance(VALUE klass)
VALUE flags = T_OBJECT | ROBJECT_EMBED; VALUE flags = T_OBJECT | ROBJECT_EMBED;
VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, RGENGC_WB_PROTECTED_OBJECT); VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, RGENGC_WB_PROTECTED_OBJECT, sizeof(RVALUE));
if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) { if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
rb_init_iv_list(obj); rb_init_iv_list(obj);
@ -2542,7 +2761,7 @@ rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FU
{ {
RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1); RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1);
if (klass) Check_Type(klass, T_CLASS); if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE); return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE, sizeof(RVALUE));
} }
VALUE VALUE
@ -2558,7 +2777,7 @@ rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
{ {
RUBY_ASSERT_ALWAYS(type); RUBY_ASSERT_ALWAYS(type);
if (klass) Check_Type(klass, T_CLASS); if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED); return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED, sizeof(RVALUE));
} }
VALUE VALUE
@ -2899,7 +3118,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj); rb_class_remove_from_super_subclasses(obj);
if (RCLASS_EXT(obj)) if (RCLASS_EXT(obj))
xfree(RCLASS_EXT(obj));
RCLASS_EXT(obj) = NULL; RCLASS_EXT(obj) = NULL;
(void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE); (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
@ -3066,7 +3284,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
cc_table_free(objspace, obj, FALSE); cc_table_free(objspace, obj, FALSE);
rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj); rb_class_remove_from_super_subclasses(obj);
xfree(RCLASS_EXT(obj));
RCLASS_EXT(obj) = NULL; RCLASS_EXT(obj) = NULL;
RB_DEBUG_COUNTER_INC(obj_iclass_ptr); RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
@ -3316,11 +3533,34 @@ objspace_each_objects_try(VALUE arg)
RVALUE *pstart = page->start; RVALUE *pstart = page->start;
RVALUE *pend = pstart + page->total_slots; RVALUE *pend = pstart + page->total_slots;
RVALUE *cursor_end = pstart;
if ((*data->callback)(pstart, pend, sizeof(RVALUE), data->data)) { while (cursor_end < pend) {
int payload_len = 0;
while(cursor_end < pend && BUILTIN_TYPE((VALUE)cursor_end) != T_PAYLOAD) {
cursor_end++;
}
#if USE_RVARGC
//Make sure the Payload header slot is yielded
if (cursor_end < pend && BUILTIN_TYPE((VALUE)cursor_end) == T_PAYLOAD) {
payload_len = RPAYLOAD_LEN((VALUE)cursor_end);
cursor_end++;
}
#endif
if ((*data->callback)(pstart, cursor_end, sizeof(RVALUE), data->data)) {
break; break;
} }
// Move the cursor over the rest of the payload body
if (payload_len) {
cursor_end += (payload_len - 1);
pstart = cursor_end;
}
}
page = list_next(&heap_eden->pages, page, page_node); page = list_next(&heap_eden->pages, page, page_node);
} }
@ -3443,6 +3683,7 @@ internal_object_p(VALUE obj)
case T_IMEMO: case T_IMEMO:
case T_ICLASS: case T_ICLASS:
case T_ZOMBIE: case T_ZOMBIE:
case T_PAYLOAD:
break; break;
case T_CLASS: case T_CLASS:
if (!p->as.basic.klass) break; if (!p->as.basic.klass) break;
@ -4403,6 +4644,7 @@ obj_memsize_of(VALUE obj, int use_all_types)
case T_ZOMBIE: case T_ZOMBIE:
case T_MOVED: case T_MOVED:
case T_PAYLOAD:
break; break;
default: default:
@ -4460,6 +4702,7 @@ type_sym(size_t type)
COUNT_TYPE(T_ICLASS); COUNT_TYPE(T_ICLASS);
COUNT_TYPE(T_ZOMBIE); COUNT_TYPE(T_ZOMBIE);
COUNT_TYPE(T_MOVED); COUNT_TYPE(T_MOVED);
COUNT_TYPE(T_PAYLOAD);
#undef COUNT_TYPE #undef COUNT_TYPE
default: return SIZET2NUM(type); break; default: return SIZET2NUM(type); break;
} }
@ -4523,12 +4766,21 @@ count_objects(int argc, VALUE *argv, VALUE os)
for (i = 0; i < heap_allocated_pages; i++) { for (i = 0; i < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i]; struct heap_page *page = heap_pages_sorted[i];
RVALUE *p, *pend; RVALUE *p, *pend;
int stride = 1;
p = page->start; pend = p + page->total_slots; p = page->start; pend = p + page->total_slots;
for (;p < pend; p++) { for (;p < pend; p += stride) {
stride = 1;
VALUE vp = (VALUE)p; VALUE vp = (VALUE)p;
void *poisoned = asan_poisoned_object_p(vp); void *poisoned = asan_poisoned_object_p(vp);
asan_unpoison_object(vp, false); asan_unpoison_object(vp, false);
#if USE_RVARGC
if (RB_TYPE_P(vp, T_PAYLOAD)) {
stride = RPAYLOAD_LEN(vp);
counts[BUILTIN_TYPE(vp)] += RPAYLOAD_LEN(vp);
} else
#endif
if (p->as.basic.flags) { if (p->as.basic.flags) {
counts[BUILTIN_TYPE(vp)]++; counts[BUILTIN_TYPE(vp)]++;
} }
@ -4999,7 +5251,6 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64 if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
bits[BITMAP_INDEX(p) + sweep_page->total_slots / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1); bits[BITMAP_INDEX(p) + sweep_page->total_slots / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
} }
for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) { for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
bitset = ~bits[i]; bitset = ~bits[i];
if (bitset) { if (bitset) {
@ -5036,6 +5287,34 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
break; break;
/* minor cases */ /* minor cases */
case T_PAYLOAD:
{
int plen = RPAYLOAD_LEN(vp);
freed_slots += plen;
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)vp, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, vp);
// This loop causes slots *following this slot* to be marked as
// T_NONE. On the next iteration of this sweep loop, the T_NONE slots
// can be double counted. Mutating the bit plane is difficult because it's
// copied to a local variable. So we would need special logic to mutate
// local bitmap plane (stored in `bitset`) plane, versus T_PAYLOAD objects that span
// bitplanes. (Imagine a T_PAYLOAD at positions 0-3 versus positions 62-65,
// their mark bits would be on different planes. We would have to mutate only `bitset`
// for the first case, but `bitset` and `bits[i+1]` for the second
for (int i = 1; i < plen; i++) {
VALUE pbody = vp + i * sizeof(RVALUE);
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)pbody, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, pbody);
// Lets set a bit on the object so that the T_NONE branch
// will know to avoid double counting this slot.
FL_SET(pbody, FL_FROM_PAYLOAD);
}
}
break;
case T_MOVED: case T_MOVED:
if (objspace->flags.during_compacting) { if (objspace->flags.during_compacting) {
/* The sweep cursor shouldn't have made it to any /* The sweep cursor shouldn't have made it to any
@ -5062,9 +5341,16 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
/* We *want* to fill this slot */ /* We *want* to fill this slot */
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(vp), vp); MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(vp), vp);
} }
else {
// This slot came from a T_PAYLOAD object and
// has already been counted
if (FL_TEST(vp, FL_FROM_PAYLOAD)) {
FL_UNSET(vp, FL_FROM_PAYLOAD);
}
else { else {
empty_slots++; /* already freed */ empty_slots++; /* already freed */
} }
}
break; break;
} }
} }
@ -6071,12 +6357,18 @@ rb_mark_tbl_no_pin(st_table *tbl)
mark_tbl_no_pin(&rb_objspace, tbl); mark_tbl_no_pin(&rb_objspace, tbl);
} }
static void gc_mark_payload(rb_objspace_t *objspace, VALUE obj);
static void static void
gc_mark_maybe(rb_objspace_t *objspace, VALUE obj) gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
{ {
(void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj)); (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
if (is_pointer_to_heap(objspace, (void *)obj)) { if (is_pointer_to_heap(objspace, (void *)obj)) {
#if USE_RVARGC
obj = payload_or_self(obj);
#endif
void *ptr = __asan_region_is_poisoned((void *)obj, SIZEOF_VALUE); void *ptr = __asan_region_is_poisoned((void *)obj, SIZEOF_VALUE);
asan_unpoison_object(obj, false); asan_unpoison_object(obj, false);
@ -6207,6 +6499,17 @@ gc_aging(rb_objspace_t *objspace, VALUE obj)
GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE); GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj); RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
} }
#if USE_RVARGC
if (RVALUE_UNCOLLECTIBLE(obj) && BUILTIN_TYPE(obj) == T_PAYLOAD) {
int plen = RPAYLOAD_LEN(obj);
for (int i = 1; i < plen; i++) {
VALUE pbody = obj + i * sizeof(RVALUE);
MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(pbody), pbody);
}
}
#endif
} }
check_rvalue_consistency(obj); check_rvalue_consistency(obj);
@ -6388,6 +6691,22 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj)
} }
} }
static inline void
gc_mark_payload(rb_objspace_t *objspace, VALUE obj)
{
#if USE_RVARGC
GC_ASSERT(BUILTIN_TYPE(obj) == T_PAYLOAD);
// Mark payload head here
gc_mark_and_pin(objspace, obj);
for (int i = 1 ; i < RPAYLOAD_LEN(obj); i++) {
VALUE p = obj + i * sizeof(RVALUE);
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(p), p);
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(p), p);
}
#endif
}
static void static void
gc_mark_children(rb_objspace_t *objspace, VALUE obj) gc_mark_children(rb_objspace_t *objspace, VALUE obj)
{ {
@ -6423,9 +6742,14 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
break; break;
} }
if (BUILTIN_TYPE(obj) != T_PAYLOAD) {
gc_mark(objspace, any->as.basic.klass); gc_mark(objspace, any->as.basic.klass);
}
switch (BUILTIN_TYPE(obj)) { switch (BUILTIN_TYPE(obj)) {
case T_PAYLOAD:
gc_mark_payload(objspace, obj);
break;
case T_CLASS: case T_CLASS:
case T_MODULE: case T_MODULE:
if (RCLASS_SUPER(obj)) { if (RCLASS_SUPER(obj)) {
@ -6437,6 +6761,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
cc_table_mark(objspace, obj); cc_table_mark(objspace, obj);
mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj)); mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj));
mark_const_tbl(objspace, RCLASS_CONST_TBL(obj)); mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
gc_mark_payload(objspace, (VALUE)((uintptr_t)RCLASS(obj)->ptr - sizeof(struct RPayload)));
break; break;
case T_ICLASS: case T_ICLASS:
@ -6449,6 +6774,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
if (!RCLASS_EXT(obj)) break; if (!RCLASS_EXT(obj)) break;
mark_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj)); mark_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
cc_table_mark(objspace, obj); cc_table_mark(objspace, obj);
gc_mark_payload(objspace, (VALUE)((uintptr_t)RCLASS(obj)->ptr - sizeof(struct RPayload)));
break; break;
case T_ARRAY: case T_ARRAY:
@ -7082,6 +7408,14 @@ verify_internal_consistency_i(void *page_start, void *page_end, size_t stride, v
rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data); rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data);
} }
} }
/* make sure we have counted the payload body slots */
if (BUILTIN_TYPE(obj) == T_PAYLOAD) {
if (RVALUE_OLD_P(obj)) {
data->old_object_count += RPAYLOAD_LEN(obj) - 1;
}
data->live_object_count += RPAYLOAD_LEN(obj) - 1;
}
} }
else { else {
if (BUILTIN_TYPE(obj) == T_ZOMBIE) { if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
@ -7107,13 +7441,28 @@ gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
int remembered_old_objects = 0; int remembered_old_objects = 0;
int free_objects = 0; int free_objects = 0;
int zombie_objects = 0; int zombie_objects = 0;
int stride = 1;
#if USE_RVARGC
int default_stride = 1;
#endif
for (i=0; i<page->total_slots; i++) { for (i=0; i<page->total_slots; i+=stride) {
VALUE val = (VALUE)&page->start[i]; VALUE val = (VALUE)&page->start[i];
void *poisoned = asan_poisoned_object_p(val); void *poisoned = asan_poisoned_object_p(val);
asan_unpoison_object(val, false); asan_unpoison_object(val, false);
#if USE_RVARGC
if (BUILTIN_TYPE(val) == T_PAYLOAD) {
stride = RPAYLOAD_LEN(val);
} else {
stride = default_stride;
}
#endif
if (RBASIC(val) == 0) free_objects++; if (RBASIC(val) == 0) free_objects++;
#if USE_RVARGC
if (BUILTIN_TYPE(val) == T_PAYLOAD) stride = RPAYLOAD_LEN(val);
#endif
if (BUILTIN_TYPE(val) == T_ZOMBIE) zombie_objects++; if (BUILTIN_TYPE(val) == T_ZOMBIE) zombie_objects++;
if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) { if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
has_remembered_shady = TRUE; has_remembered_shady = TRUE;
@ -9475,6 +9824,9 @@ gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t * objspace,
case T_MOVED: case T_MOVED:
case T_ZOMBIE: case T_ZOMBIE:
break; break;
case T_PAYLOAD:
v += (stride * (RPAYLOAD_LEN(v) - 1));
break;
default: default:
if (RVALUE_WB_UNPROTECTED(v)) { if (RVALUE_WB_UNPROTECTED(v)) {
page->flags.has_uncollectible_shady_objects = TRUE; page->flags.has_uncollectible_shady_objects = TRUE;
@ -12339,6 +12691,7 @@ type_name(int type, VALUE obj)
TYPE_NAME(T_ICLASS); TYPE_NAME(T_ICLASS);
TYPE_NAME(T_MOVED); TYPE_NAME(T_MOVED);
TYPE_NAME(T_ZOMBIE); TYPE_NAME(T_ZOMBIE);
TYPE_NAME(T_PAYLOAD);
case T_DATA: case T_DATA:
if (obj && rb_objspace_data_type_name(obj)) { if (obj && rb_objspace_data_type_name(obj)) {
return rb_objspace_data_type_name(obj); return rb_objspace_data_type_name(obj);
@ -12452,6 +12805,9 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
if (internal_object_p(obj)) { if (internal_object_p(obj)) {
/* ignore */ /* ignore */
} }
else if (type == T_PAYLOAD) {
/* ignore */
}
else if (RBASIC(obj)->klass == 0) { else if (RBASIC(obj)->klass == 0) {
APPENDF((BUFF_ARGS, "(temporary internal)")); APPENDF((BUFF_ARGS, "(temporary internal)"));
} }
@ -12469,6 +12825,9 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
#endif #endif
switch (type) { switch (type) {
case T_PAYLOAD:
APPENDF((BUFF_ARGS, "len: %i", RPAYLOAD_LEN(obj)));
break;
case T_NODE: case T_NODE:
UNEXPECTED_NODE(rb_raw_obj_info); UNEXPECTED_NODE(rb_raw_obj_info);
break; break;

2
include/ruby/internal/value_type.h
View File

@ -80,6 +80,7 @@
#define T_TRUE RUBY_T_TRUE #define T_TRUE RUBY_T_TRUE
#define T_UNDEF RUBY_T_UNDEF #define T_UNDEF RUBY_T_UNDEF
#define T_ZOMBIE RUBY_T_ZOMBIE #define T_ZOMBIE RUBY_T_ZOMBIE
#define T_PAYLOAD RUBY_T_PAYLOAD
#define BUILTIN_TYPE RB_BUILTIN_TYPE #define BUILTIN_TYPE RB_BUILTIN_TYPE
#define DYNAMIC_SYM_P RB_DYNAMIC_SYM_P #define DYNAMIC_SYM_P RB_DYNAMIC_SYM_P
@ -132,6 +133,7 @@ ruby_value_type {
RUBY_T_SYMBOL = 0x14, /**< @see struct ::RSymbol */ RUBY_T_SYMBOL = 0x14, /**< @see struct ::RSymbol */
RUBY_T_FIXNUM = 0x15, /**< Integers formerly known as Fixnums. */ RUBY_T_FIXNUM = 0x15, /**< Integers formerly known as Fixnums. */
RUBY_T_UNDEF = 0x16, /**< @see ::RUBY_Qundef */ RUBY_T_UNDEF = 0x16, /**< @see ::RUBY_Qundef */
RUBY_T_PAYLOAD = 0x17, /**< @see ::RPayload */
RUBY_T_IMEMO = 0x1a, /**< @see struct ::RIMemo */ RUBY_T_IMEMO = 0x1a, /**< @see struct ::RIMemo */
RUBY_T_NODE = 0x1b, /**< @see struct ::RNode */ RUBY_T_NODE = 0x1b, /**< @see struct ::RNode */

29
internal/gc.h
View File

@ -19,6 +19,10 @@
struct rb_execution_context_struct; /* in vm_core.h */ struct rb_execution_context_struct; /* in vm_core.h */
struct rb_objspace; /* in vm_core.h */ struct rb_objspace; /* in vm_core.h */
#ifndef USE_RVARGC
#define USE_RVARGC 0
#endif
#ifdef NEWOBJ_OF #ifdef NEWOBJ_OF
# undef NEWOBJ_OF # undef NEWOBJ_OF
# undef RB_NEWOBJ_OF # undef RB_NEWOBJ_OF
@ -28,15 +32,21 @@ struct rb_objspace; /* in vm_core.h */
/* optimized version of NEWOBJ() */ /* optimized version of NEWOBJ() */
#define RB_NEWOBJ_OF(var, T, c, f) \ #define RB_NEWOBJ_OF(var, T, c, f) \
T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \
rb_wb_protected_newobj_of((c), (f) & ~FL_WB_PROTECTED) : \ rb_wb_protected_newobj_of((c), (f) & ~FL_WB_PROTECTED, 0) : \
rb_wb_unprotected_newobj_of((c), (f))) rb_wb_unprotected_newobj_of((c), (f), 0))
#define RB_EC_NEWOBJ_OF(ec, var, T, c, f) \ #define RB_EC_NEWOBJ_OF(ec, var, T, c, f) \
T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \
rb_ec_wb_protected_newobj_of((ec), (c), (f) & ~FL_WB_PROTECTED) : \ rb_ec_wb_protected_newobj_of((ec), (c), (f) & ~FL_WB_PROTECTED, 0) : \
rb_wb_unprotected_newobj_of((c), (f))) rb_wb_unprotected_newobj_of((c), (f), 0))
#define RB_RVARGC_NEWOBJ_OF(var, T, c, f, s) \
T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \
rb_wb_protected_newobj_of((c), (f) & ~FL_WB_PROTECTED, s) : \
rb_wb_unprotected_newobj_of((c), (f), s))
#define NEWOBJ_OF(var, T, c, f) RB_NEWOBJ_OF((var), T, (c), (f)) #define NEWOBJ_OF(var, T, c, f) RB_NEWOBJ_OF((var), T, (c), (f))
#define RVARGC_NEWOBJ_OF(var, T, c, f, s) RB_RVARGC_NEWOBJ_OF((var), T, (c), (f), (s))
#define RB_OBJ_GC_FLAGS_MAX 6 /* used in ext/objspace */ #define RB_OBJ_GC_FLAGS_MAX 6 /* used in ext/objspace */
#ifndef USE_UNALIGNED_MEMBER_ACCESS #ifndef USE_UNALIGNED_MEMBER_ACCESS
@ -61,6 +71,8 @@ struct rb_objspace; /* in vm_core.h */
rb_obj_write((VALUE)(a), UNALIGNED_MEMBER_ACCESS((VALUE *)(slot)), \ rb_obj_write((VALUE)(a), UNALIGNED_MEMBER_ACCESS((VALUE *)(slot)), \
(VALUE)(b), __FILE__, __LINE__) (VALUE)(b), __FILE__, __LINE__)
#define RVARGC_PAYLOAD_INIT(obj, size) (void *)rb_rvargc_payload_init((VALUE)obj, (size_t)size)
/* gc.c */ /* gc.c */
extern VALUE *ruby_initial_gc_stress_ptr; extern VALUE *ruby_initial_gc_stress_ptr;
extern int ruby_disable_gc; extern int ruby_disable_gc;
@ -89,9 +101,11 @@ VALUE rb_class_allocate_instance(VALUE klass);
RUBY_SYMBOL_EXPORT_BEGIN RUBY_SYMBOL_EXPORT_BEGIN
/* gc.c (export) */ /* gc.c (export) */
const char *rb_objspace_data_type_name(VALUE obj); const char *rb_objspace_data_type_name(VALUE obj);
VALUE rb_wb_protected_newobj_of(VALUE, VALUE); VALUE rb_wb_protected_newobj_of(VALUE, VALUE, size_t);
VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE); VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE, size_t);
VALUE rb_ec_wb_protected_newobj_of(struct rb_execution_context_struct *ec, VALUE klass, VALUE flags); VALUE rb_ec_wb_protected_newobj_of(struct rb_execution_context_struct *ec, VALUE klass, VALUE flags, size_t);
VALUE rb_rvargc_payload_init(VALUE obj, size_t size);
void * rb_rvargc_payload_data_ptr(VALUE obj);
size_t rb_obj_memsize_of(VALUE); size_t rb_obj_memsize_of(VALUE);
void rb_gc_verify_internal_consistency(void); void rb_gc_verify_internal_consistency(void);
size_t rb_obj_gc_flags(VALUE, ID[], size_t); size_t rb_obj_gc_flags(VALUE, ID[], size_t);
@ -100,6 +114,7 @@ void rb_gc_mark_vm_stack_values(long n, const VALUE *values);
void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2)); void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2));
void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3));
void ruby_sized_xfree(void *x, size_t size); void ruby_sized_xfree(void *x, size_t size);
int rb_slot_size();
RUBY_SYMBOL_EXPORT_END RUBY_SYMBOL_EXPORT_END
MJIT_SYMBOL_EXPORT_BEGIN MJIT_SYMBOL_EXPORT_BEGIN

1
vm_eval.c
View File

@ -729,6 +729,7 @@ rb_type_str(enum ruby_value_type type)
case type_case(T_ICLASS); case type_case(T_ICLASS);
case type_case(T_ZOMBIE); case type_case(T_ZOMBIE);
case type_case(T_MOVED); case type_case(T_MOVED);
case type_case(T_PAYLOAD);
case T_MASK: break; case T_MASK: break;
} }
#undef type_case #undef type_case