1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support an arbitrary number of header bits (< BITS_BITLENGTH)

Browse Source 
NUM_IN_PAGE(page->start) will sometimes return a 0 or a 1 depending on
how the alignment of the 40 byte slots work out.  This commit uses the
NUM_IN_PAGE function to shift the bitmap down on the first bitmap plane.
Iterating on the first bitmap plane is "special", but this commit allows
us to align object addresses on something besides 40 bytes, and also
eliminates the need to fill guard bits.
This commit is contained in:
Aaron Patterson 2021-06-01 16:28:35 -07:00
parent 93be7a4c6b
commit 38c5f2737f
No known key found for this signature in database
GPG Key ID: 953170BCB4FFAFC6
1 changed files with 319 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

357
gc.c
View File

@ -5191,33 +5191,19 @@ gc_compact_finish(rb_objspace_t *objspace, rb_heap_t *heap)
objspace->flags.during_compacting = FALSE;
}
static bool
gc_fill_swept_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page, int *freed_slots, int *empty_slots)
struct gc_sweep_context {
struct heap_page *page;
int final_slots;
int freed_slots;
int empty_slots;
};
static inline void
gc_fill_swept_page_plane(rb_objspace_t *objspace, rb_heap_t *heap, intptr_t p, bits_t bitset, bool * finished_compacting, struct gc_sweep_context *ctx)
{
/* Find any pinned but not marked objects and try to fill those slots */
int i;
int moved_slots = 0;
bool finished_compacting = false;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
RVALUE *p, *offset;
mark_bits = sweep_page->mark_bits;
pin_bits = sweep_page->pinned_bits;
p = sweep_page->start;
offset = p - NUM_IN_PAGE(p);
struct heap_page * cursor = heap->compact_cursor;
unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
/* *Want to move* objects are pinned but not marked. */
bitset = pin_bits[i] & ~mark_bits[i];
struct heap_page * sweep_page = ctx->page;
if (bitset) {
p = offset + i * BITS_BITLENGTH;
do {
if (bitset & 1) {
VALUE dest = (VALUE)p;
@ -5227,12 +5213,12 @@ gc_fill_swept_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *s
CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(dest), dest);
if (finished_compacting) {
if (*finished_compacting) {
if (BUILTIN_TYPE(dest) == T_NONE) {
(*empty_slots)++;
ctx->empty_slots++;
}
else {
(*freed_slots)++;
ctx->freed_slots++;
}
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)dest, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, dest);
@ -5242,76 +5228,71 @@ gc_fill_swept_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *s
* their memory until they have their finalizers run.*/
if (BUILTIN_TYPE(dest) != T_ZOMBIE) {
if(!try_move(objspace, heap, sweep_page, dest)) {
finished_compacting = true;
*finished_compacting = true;
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
gc_report(5, objspace, "Quit compacting, couldn't find an object to move\n");
if (BUILTIN_TYPE(dest) == T_NONE) {
(*empty_slots)++;
ctx->empty_slots++;
}
else {
(*freed_slots)++;
ctx->freed_slots++;
}
heap_page_add_freeobj(objspace, sweep_page, dest);
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(dest));
}
else {
moved_slots++;
//moved_slots++;
}
}
}
}
p++;
p += sizeof(RVALUE);
bitset >>= 1;
} while (bitset);
}
}
static bool
gc_fill_swept_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page, struct gc_sweep_context *ctx)
{
/* Find any pinned but not marked objects and try to fill those slots */
bool finished_compacting = false;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
RVALUE *p;
mark_bits = sweep_page->mark_bits;
pin_bits = sweep_page->pinned_bits;
p = sweep_page->start;
struct heap_page * cursor = heap->compact_cursor;
unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
/* *Want to move* objects are pinned but not marked. */
bitset = pin_bits[0] & ~mark_bits[0];
bitset >>= NUM_IN_PAGE(p); // Skip header / dead space bits
gc_fill_swept_page_plane(objspace ,heap, (intptr_t)p, bitset, &finished_compacting, ctx);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p));
for (int i = 1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
/* *Want to move* objects are pinned but not marked. */
bitset = pin_bits[i] & ~mark_bits[i];
gc_fill_swept_page_plane(objspace ,heap, (intptr_t)p, bitset, &finished_compacting, ctx);
p += BITS_BITLENGTH;
}
lock_page_body(objspace, GET_PAGE_BODY(heap->compact_cursor->start));
return finished_compacting;
}
static inline int
gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
static inline void
gc_plane_sweep(rb_objspace_t *objspace, rb_heap_t *heap, intptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
{
int i;
int empty_slots = 0, freed_slots = 0, final_slots = 0;
RVALUE *p, *offset;
bits_t *bits, bitset;
struct heap_page * sweep_page = ctx->page;
gc_report(2, objspace, "page_sweep: start.\n");
if (heap->compact_cursor) {
if (sweep_page == heap->compact_cursor) {
/* The compaction cursor and sweep page met, so we need to quit compacting */
gc_report(5, objspace, "Quit compacting, mark and compact cursor met\n");
gc_compact_finish(objspace, heap);
}
else {
/* We anticipate filling the page, so NULL out the freelist. */
asan_unpoison_memory_region(&sweep_page->freelist, sizeof(RVALUE*), false);
sweep_page->freelist = NULL;
asan_poison_memory_region(&sweep_page->freelist, sizeof(RVALUE*));
}
}
sweep_page->flags.before_sweep = FALSE;
p = sweep_page->start;
offset = p - NUM_IN_PAGE(p);
bits = sweep_page->mark_bits;
/* create guard : fill 1 out-of-range */
bits[BITMAP_INDEX(p)] |= BITMAP_BIT(p)-1;
int out_of_range_bits = (NUM_IN_PAGE(p) + sweep_page->total_slots) % BITS_BITLENGTH;
if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
bits[BITMAP_INDEX(p) + sweep_page->total_slots / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
}
for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
bitset = ~bits[i];
if (bitset) {
p = offset + i * BITS_BITLENGTH;
do {
VALUE vp = (VALUE)p;
asan_unpoison_object(vp, false);
@ -5326,7 +5307,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
}
#endif
if (obj_free(objspace, vp)) {
final_slots++;
ctx->final_slots++;
}
else {
if (heap->compact_cursor) {
@ -5337,7 +5318,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, vp);
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
freed_slots++;
ctx->freed_slots++;
}
}
@ -5347,7 +5328,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
case T_PAYLOAD:
{
int plen = RPAYLOAD_LEN(vp);
freed_slots += plen;
ctx->freed_slots += plen;
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)vp, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, vp);
@ -5383,10 +5364,10 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
}
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
if (FL_TEST(vp, FL_FROM_FREELIST)) {
empty_slots++;
ctx->empty_slots++;
}
else {
freed_slots++;
ctx->freed_slots++;
}
heap_page_add_freeobj(objspace, sweep_page, vp);
break;
@ -5405,20 +5386,75 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
FL_UNSET(vp, FL_FROM_PAYLOAD);
}
else {
empty_slots++; /* already freed */
ctx->empty_slots++; /* already freed */
}
}
break;
}
}
p++;
p += sizeof(RVALUE);
bitset >>= 1;
} while (bitset);
}
static inline int
gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
{
int i;
struct gc_sweep_context ctx;
ctx.page = sweep_page;
ctx.final_slots = 0;
ctx.freed_slots = 0;
ctx.empty_slots = 0;
RVALUE *p;
bits_t *bits, bitset;
gc_report(2, objspace, "page_sweep: start.\n");
if (heap->compact_cursor) {
if (sweep_page == heap->compact_cursor) {
/* The compaction cursor and sweep page met, so we need to quit compacting */
gc_report(5, objspace, "Quit compacting, mark and compact cursor met\n");
gc_compact_finish(objspace, heap);
}
else {
/* We anticipate filling the page, so NULL out the freelist. */
asan_unpoison_memory_region(&sweep_page->freelist, sizeof(RVALUE*), false);
sweep_page->freelist = NULL;
asan_poison_memory_region(&sweep_page->freelist, sizeof(RVALUE*));
}
}
sweep_page->flags.before_sweep = FALSE;
p = sweep_page->start;
bits = sweep_page->mark_bits;
int out_of_range_bits = (NUM_IN_PAGE(p) + sweep_page->total_slots) % BITS_BITLENGTH;
if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
bits[BITMAP_INDEX(p) + sweep_page->total_slots / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
}
// Skip out of range slots at the head of the page
bitset = ~bits[0];
bitset >>= NUM_IN_PAGE(p);
if (bitset) {
gc_plane_sweep(objspace, heap, (intptr_t)p, bitset, &ctx);
}
p += (BITS_BITLENGTH - NUM_IN_PAGE(p));
for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
bitset = ~bits[i];
if (bitset) {
gc_plane_sweep(objspace, heap, (intptr_t)p, bitset, &ctx);
}
p += BITS_BITLENGTH;
}
if (heap->compact_cursor) {
if (gc_fill_swept_page(objspace, heap, sweep_page, &freed_slots, &empty_slots)) {
if (gc_fill_swept_page(objspace, heap, sweep_page, &ctx)) {
gc_compact_finish(objspace, heap);
}
}
@ -5430,17 +5466,17 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->removing_objects += final_slots + freed_slots;
record->empty_objects += empty_slots;
record->removing_objects += ctx.final_slots + ctx.freed_slots;
record->empty_objects += ctx.empty_slots;
}
#endif
if (0) fprintf(stderr, "gc_page_sweep(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
rb_gc_count(),
sweep_page->total_slots,
freed_slots, empty_slots, final_slots);
ctx.freed_slots, ctx.empty_slots, ctx.final_slots);
sweep_page->free_slots = freed_slots + empty_slots;
objspace->profile.total_freed_objects += freed_slots;
sweep_page->free_slots = ctx.freed_slots + ctx.empty_slots;
objspace->profile.total_freed_objects += ctx.freed_slots;
if (heap_pages_deferred_final && !finalizing) {
rb_thread_t *th = GET_THREAD();
@ -5451,7 +5487,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
gc_report(2, objspace, "page_sweep: end.\n");
return freed_slots + empty_slots;
return ctx.freed_slots + ctx.empty_slots;
}
/* allocate additional minimum page to work */
@ -5662,27 +5698,9 @@ gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap)
}
static void
invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
invalidate_moved_plane(rb_objspace_t *objspace, intptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
{
int i;
int empty_slots = 0, freed_slots = 0;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
RVALUE *p, *offset;
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
p = page->start;
offset = p - NUM_IN_PAGE(p);
for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
/* Moved objects are pinned but never marked. We reuse the pin bits
* to indicate there is a moved object in this slot. */
bitset = pin_bits[i] & ~mark_bits[i];
if (bitset) {
p = offset + i * BITS_BITLENGTH;
do {
if (bitset & 1) {
VALUE forwarding_object = (VALUE)p;
@ -5697,10 +5715,10 @@ invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
object = rb_gc_location(forwarding_object);
if (FL_TEST(forwarding_object, FL_FROM_FREELIST)) {
empty_slots++; /* already freed */
ctx->empty_slots++; /* already freed */
}
else {
freed_slots++;
ctx->freed_slots++;
}
gc_move(objspace, object, forwarding_object);
@ -5713,14 +5731,48 @@ invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
}
}
p++;
p += sizeof(RVALUE);
bitset >>= 1;
} while (bitset);
}
}
page->free_slots += (empty_slots + freed_slots);
objspace->profile.total_freed_objects += freed_slots;
static void
invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
{
int i;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
RVALUE *p;
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
p = page->start;
struct gc_sweep_context ctx;
ctx.page = page;
ctx.final_slots = 0;
ctx.freed_slots = 0;
ctx.empty_slots = 0;
// Skip out of range slots at the head of the page
bitset = pin_bits[0] & ~mark_bits[0];
bitset >>= NUM_IN_PAGE(p);
invalidate_moved_plane(objspace, (intptr_t)p, bitset, &ctx);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p));
for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
/* Moved objects are pinned but never marked. We reuse the pin bits
* to indicate there is a moved object in this slot. */
bitset = pin_bits[i] & ~mark_bits[i];
invalidate_moved_plane(objspace, (intptr_t)p, bitset, &ctx);
p += BITS_BITLENGTH;
}
page->free_slots += (ctx.empty_slots + ctx.freed_slots);
objspace->profile.total_freed_objects += ctx.freed_slots;
}
static void
@ -7807,6 +7859,23 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark)
}
#if GC_ENABLE_INCREMENTAL_MARK
static inline void
gc_marks_wb_unprotected_objects_in_plane(rb_objspace_t *objspace, intptr_t p, bits_t bits)
{
if (bits) {
do {
if (bits & 1) {
gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", obj_info((VALUE)p));
GC_ASSERT(RVALUE_WB_UNPROTECTED((VALUE)p));
GC_ASSERT(RVALUE_MARKED((VALUE)p));
gc_mark_children(objspace, (VALUE)p);
}
p += sizeof(RVALUE);
bits >>= 1;
} while (bits);
}
}
static void
gc_marks_wb_unprotected_objects(rb_objspace_t *objspace)
{
@ -7816,26 +7885,18 @@ gc_marks_wb_unprotected_objects(rb_objspace_t *objspace)
bits_t *mark_bits = page->mark_bits;
bits_t *wbun_bits = page->wb_unprotected_bits;
RVALUE *p = page->start;
RVALUE *offset = p - NUM_IN_PAGE(p);
size_t j;
for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
bits_t bits = mark_bits[0] & wbun_bits[0];
bits >>= NUM_IN_PAGE(p);
gc_marks_wb_unprotected_objects_in_plane(objspace, (intptr_t)p, bits);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p));
for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
bits_t bits = mark_bits[j] & wbun_bits[j];
if (bits) {
p = offset + j * BITS_BITLENGTH;
do {
if (bits & 1) {
gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", obj_info((VALUE)p));
GC_ASSERT(RVALUE_WB_UNPROTECTED((VALUE)p));
GC_ASSERT(RVALUE_MARKED((VALUE)p));
gc_mark_children(objspace, (VALUE)p);
}
p++;
bits >>= 1;
} while (bits);
}
gc_marks_wb_unprotected_objects_in_plane(objspace, (intptr_t)p, bits);
p += BITS_BITLENGTH;
}
}
@ -8195,6 +8256,25 @@ rgengc_remembered(rb_objspace_t *objspace, VALUE obj)
#define PROFILE_REMEMBERSET_MARK 0
#endif
static inline void
rgengc_rememberset_mark_in_plane(rb_objspace_t *objspace, intptr_t p, bits_t bitset)
{
if (bitset) {
do {
if (bitset & 1) {
VALUE obj = (VALUE)p;
gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", obj_info(obj));
GC_ASSERT(RVALUE_UNCOLLECTIBLE(obj));
GC_ASSERT(RVALUE_OLD_P(obj) || RVALUE_WB_UNPROTECTED(obj));
gc_mark_children(objspace, obj);
}
p += sizeof(RVALUE);
bitset >>= 1;
} while (bitset);
}
}
static void
rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
@ -8208,7 +8288,6 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
list_for_each(&heap->pages, page, page_node) {
if (page->flags.has_remembered_objects | page->flags.has_uncollectible_shady_objects) {
RVALUE *p = page->start;
RVALUE *offset = p - NUM_IN_PAGE(p);
bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t *marking_bits = page->marking_bits;
bits_t *uncollectible_bits = page->uncollectible_bits;
@ -8224,25 +8303,15 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
}
page->flags.has_remembered_objects = FALSE;
for (j=0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
bitset = bits[0];
bitset >>= NUM_IN_PAGE(p);
rgengc_rememberset_mark_in_plane(objspace, (intptr_t)p, bitset);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p));
for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
bitset = bits[j];
if (bitset) {
p = offset + j * BITS_BITLENGTH;
do {
if (bitset & 1) {
VALUE obj = (VALUE)p;
gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", obj_info(obj));
GC_ASSERT(RVALUE_UNCOLLECTIBLE(obj));
GC_ASSERT(RVALUE_OLD_P(obj) || RVALUE_WB_UNPROTECTED(obj));
gc_mark_children(objspace, obj);
}
p++;
bitset >>= 1;
} while (bitset);
}
rgengc_rememberset_mark_in_plane(objspace, (intptr_t)p, bitset);
p += BITS_BITLENGTH;
}
}
#if PROFILE_REMEMBERSET_MARK