1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Combine call info and cache to speed up method invocation

Browse Source 
To perform a regular method call, the VM needs two structs,
`rb_call_info` and `rb_call_cache`. At the moment, we allocate these two
structures in separate buffers. In the worst case, the CPU needs to read
4 cache lines to complete a method call. Putting the two structures
together reduces the maximum number of cache line reads to 2.

Combining the structures also saves 8 bytes per call site as the current
layout uses separate two pointers for the call info and the call cache.
This saves about 2 MiB on Discourse.

This change improves the Optcarrot benchmark at least 3%. For more
details, see attached bugs.ruby-lang.org ticket.

Complications:
 - A new instruction attribute `comptime_sp_inc` is introduced to
 calculate SP increase at compile time without using call caches. At
 compile time, a `TS_CALLDATA` operand points to a call info struct, but
 at runtime, the same operand points to a call data struct. Instruction
 that explicitly define `sp_inc` also need to define `comptime_sp_inc`.
 - MJIT code for copying call cache becomes slightly more complicated.
 - This changes the bytecode format, which might break existing tools.

[Misc #16258]
This commit is contained in:
Alan Wu 2019-07-30 21:36:05 -04:00 committed by 卜部昌平
parent 38e931fa2c
commit 89e7997622
Notes: git 2019-10-24 18:04:08 +09:00 
Merged: https://github.com/ruby/ruby/pull/2564
17 changed files with 322 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common.mk
View File

@ -974,7 +974,7 @@ $(srcs_vpath)insns.inc: $(srcdir)/tool/ruby_vm/views/insns.inc.erb $(inc_common_
$(srcs_vpath)insns_info.inc: $(srcdir)/tool/ruby_vm/views/insns_info.inc.erb $(inc_common_headers) \ $(srcs_vpath)insns_info.inc: $(srcdir)/tool/ruby_vm/views/insns_info.inc.erb $(inc_common_headers) \
$(srcdir)/tool/ruby_vm/views/_insn_type_chars.erb $(srcdir)/tool/ruby_vm/views/_insn_name_info.erb \ $(srcdir)/tool/ruby_vm/views/_insn_type_chars.erb $(srcdir)/tool/ruby_vm/views/_insn_name_info.erb \
$(srcdir)/tool/ruby_vm/views/_insn_len_info.erb $(srcdir)/tool/ruby_vm/views/_insn_operand_info.erb \ $(srcdir)/tool/ruby_vm/views/_insn_len_info.erb $(srcdir)/tool/ruby_vm/views/_insn_operand_info.erb \
$(srcdir)/tool/ruby_vm/views/_attributes.erb $(srcdir)/tool/ruby_vm/views/_insn_stack_increase.erb \ $(srcdir)/tool/ruby_vm/views/_attributes.erb $(srcdir)/tool/ruby_vm/views/_comptime_insn_stack_increase.erb \
$(srcdir)/tool/ruby_vm/views/_insn_sp_pc_dependency.erb $(srcdir)/tool/ruby_vm/views/_insn_sp_pc_dependency.erb
$(srcs_vpath)vmtc.inc: $(srcdir)/tool/ruby_vm/views/vmtc.inc.erb $(inc_common_headers) $(srcs_vpath)vmtc.inc: $(srcdir)/tool/ruby_vm/views/vmtc.inc.erb $(inc_common_headers)
$(srcs_vpath)vm.inc: $(srcdir)/tool/ruby_vm/views/vm.inc.erb $(inc_common_headers) \ $(srcs_vpath)vm.inc: $(srcdir)/tool/ruby_vm/views/vm.inc.erb $(inc_common_headers) \

237
compile.c
View File

@ -546,6 +546,31 @@ verify_list(ISEQ_ARG_DECLARE const char *info, LINK_ANCHOR *const anchor)
#define verify_list(info, anchor) verify_list(iseq, (info), (anchor)) #define verify_list(info, anchor) verify_list(iseq, (info), (anchor))
#endif #endif
static void
verify_call_cache(rb_iseq_t *iseq)
{
return; /* comment out to enable */
VALUE *original = rb_iseq_original_iseq(iseq);
size_t i = 0;
while (i < iseq->body->iseq_size) {
VALUE insn = original[i];
const char *types = insn_op_types(insn);
for (int j=0; types[j]; j++) {
if (types[j] == TS_CALLDATA) {
struct rb_call_cache cc;
struct rb_call_data *cd = (struct rb_call_data *)original[i+j+1];
MEMZERO(&cc, cc, 1);
if (memcmp(&cc, &cd->cc, sizeof(cc))) {
rb_bug("call cache not zero for fresh iseq");
}
}
}
i += insn_len(insn);
}
}
/* /*
* elem1, elem2 => elem1, elem2, elem * elem1, elem2 => elem1, elem2, elem
*/ */
@ -1178,11 +1203,10 @@ new_callinfo(rb_iseq_t *iseq, ID mid, int argc, unsigned int flag, struct rb_cal
static INSN * static INSN *
new_insn_send(rb_iseq_t *iseq, int line_no, ID id, VALUE argc, const rb_iseq_t *blockiseq, VALUE flag, struct rb_call_info_kw_arg *keywords) new_insn_send(rb_iseq_t *iseq, int line_no, ID id, VALUE argc, const rb_iseq_t *blockiseq, VALUE flag, struct rb_call_info_kw_arg *keywords)
{ {
VALUE *operands = compile_data_alloc2(iseq, sizeof(VALUE), 3); VALUE *operands = compile_data_alloc2(iseq, sizeof(VALUE), 2);
operands[0] = (VALUE)new_callinfo(iseq, id, FIX2INT(argc), FIX2INT(flag), keywords, blockiseq != NULL); operands[0] = (VALUE)new_callinfo(iseq, id, FIX2INT(argc), FIX2INT(flag), keywords, blockiseq != NULL);
operands[1] = Qfalse; /* cache */ operands[1] = (VALUE)blockiseq;
operands[2] = (VALUE)blockiseq; return new_insn_core(iseq, line_no, BIN(send), 2, operands);
return new_insn_core(iseq, line_no, BIN(send), 3, operands);
} }
static rb_iseq_t * static rb_iseq_t *
@ -1358,6 +1382,7 @@ iseq_setup(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
VALUE str = rb_iseq_disasm(iseq); VALUE str = rb_iseq_disasm(iseq);
printf("%s\n", StringValueCStr(str)); printf("%s\n", StringValueCStr(str));
} }
verify_call_cache(iseq);
debugs("[compile step: finish]\n"); debugs("[compile step: finish]\n");
return COMPILE_OK; return COMPILE_OK;
@ -2090,13 +2115,10 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
insns_info = ALLOC_N(struct iseq_insn_info_entry, insn_num); insns_info = ALLOC_N(struct iseq_insn_info_entry, insn_num);
positions = ALLOC_N(unsigned int, insn_num); positions = ALLOC_N(unsigned int, insn_num);
body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, body->is_size); body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, body->is_size);
body->ci_entries = body->call_data =
rb_xmalloc_mul_add_mul( rb_xcalloc_mul_add_mul(
sizeof(struct rb_call_info), body->ci_size, sizeof(struct rb_call_data), body->ci_size,
sizeof(struct rb_call_info_with_kwarg), body->ci_kw_size); sizeof(struct rb_kwarg_call_data), body->ci_kw_size);
MEMZERO(body->ci_entries + body->ci_size, struct rb_call_info_with_kwarg, body->ci_kw_size); /* need to clear ci_kw entries */
body->cc_entries = ZALLOC_N(struct rb_call_cache, body->ci_size + body->ci_kw_size);
ISEQ_COMPILE_DATA(iseq)->ci_index = ISEQ_COMPILE_DATA(iseq)->ci_kw_index = 0; ISEQ_COMPILE_DATA(iseq)->ci_index = ISEQ_COMPILE_DATA(iseq)->ci_kw_index = 0;
list = FIRST_ELEMENT(anchor); list = FIRST_ELEMENT(anchor);
@ -2180,33 +2202,27 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
generated_iseq[code_index + 1 + j] = (VALUE)ic; generated_iseq[code_index + 1 + j] = (VALUE)ic;
break; break;
} }
case TS_CALLINFO: /* call info */ case TS_CALLDATA:
{ {
struct rb_call_info *base_ci = (struct rb_call_info *)operands[j]; struct rb_call_info *source_ci = (struct rb_call_info *)operands[j];
struct rb_call_info *ci; struct rb_call_data *cd;
if (base_ci->flag & VM_CALL_KWARG) { if (source_ci->flag & VM_CALL_KWARG) {
struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size];
struct rb_call_info_with_kwarg *ci_kw = &ci_kw_entries[ISEQ_COMPILE_DATA(iseq)->ci_kw_index++]; struct rb_kwarg_call_data *cd_kw = &kw_calls[ISEQ_COMPILE_DATA(iseq)->ci_kw_index++];
*ci_kw = *((struct rb_call_info_with_kwarg *)base_ci); cd_kw->ci_kw = *((struct rb_call_info_with_kwarg *)source_ci);
ci = (struct rb_call_info *)ci_kw; cd = (struct rb_call_data *)cd_kw;
assert(ISEQ_COMPILE_DATA(iseq)->ci_kw_index <= body->ci_kw_size); assert(ISEQ_COMPILE_DATA(iseq)->ci_kw_index <= body->ci_kw_size);
} }
else { else {
ci = &body->ci_entries[ISEQ_COMPILE_DATA(iseq)->ci_index++]; cd = &body->call_data[ISEQ_COMPILE_DATA(iseq)->ci_index++];
*ci = *base_ci; cd->ci = *source_ci;
assert(ISEQ_COMPILE_DATA(iseq)->ci_index <= body->ci_size); assert(ISEQ_COMPILE_DATA(iseq)->ci_index <= body->ci_size);
} }
generated_iseq[code_index + 1 + j] = (VALUE)ci; generated_iseq[code_index + 1 + j] = (VALUE)cd;
break; break;
} }
case TS_CALLCACHE:
{
struct rb_call_cache *cc = &body->cc_entries[ISEQ_COMPILE_DATA(iseq)->ci_index + ISEQ_COMPILE_DATA(iseq)->ci_kw_index - 1];
generated_iseq[code_index + 1 + j] = (VALUE)cc;
break;
}
case TS_ID: /* ID */ case TS_ID: /* ID */
generated_iseq[code_index + 1 + j] = SYM2ID(operands[j]); generated_iseq[code_index + 1 + j] = SYM2ID(operands[j]);
break; break;
@ -2530,7 +2546,7 @@ remove_unreachable_chunk(rb_iseq_t *iseq, LINK_ELEMENT *i)
case TS_OFFSET: case TS_OFFSET:
unref_destination((INSN *)i, pos); unref_destination((INSN *)i, pos);
break; break;
case TS_CALLINFO: case TS_CALLDATA:
if (((struct rb_call_info *)OPERAND_AT(i, pos))->flag & VM_CALL_KWARG) if (((struct rb_call_info *)OPERAND_AT(i, pos))->flag & VM_CALL_KWARG)
--(body->ci_kw_size); --(body->ci_kw_size);
else else
@ -3147,9 +3163,9 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal
} }
if (piobj) { if (piobj) {
struct rb_call_info *ci = (struct rb_call_info *)piobj->operands[0]; struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(piobj, 0);
if (IS_INSN_ID(piobj, send) || IS_INSN_ID(piobj, invokesuper)) { if (IS_INSN_ID(piobj, send) || IS_INSN_ID(piobj, invokesuper)) {
if (piobj->operands[2] == 0) { /* no blockiseq */ if (OPERAND_AT(piobj, 1) == 0) { /* no blockiseq */
ci->flag |= VM_CALL_TAILCALL; ci->flag |= VM_CALL_TAILCALL;
} }
} }
@ -3208,12 +3224,10 @@ insn_set_specialized_instruction(rb_iseq_t *iseq, INSN *iobj, int insn_id)
if (insn_id == BIN(opt_neq)) { if (insn_id == BIN(opt_neq)) {
VALUE *old_operands = iobj->operands; VALUE *old_operands = iobj->operands;
iobj->operand_size = 4; iobj->operand_size = 2;
iobj->operands = compile_data_alloc2(iseq, iobj->operand_size, sizeof(VALUE)); iobj->operands = compile_data_alloc2(iseq, iobj->operand_size, sizeof(VALUE));
iobj->operands[0] = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE); iobj->operands[0] = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE);
iobj->operands[1] = Qfalse; /* CALL_CACHE */ iobj->operands[1] = old_operands[0];
iobj->operands[2] = old_operands[0];
iobj->operands[3] = Qfalse; /* CALL_CACHE */
} }
return COMPILE_OK; return COMPILE_OK;
@ -3247,7 +3261,7 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj)
if (IS_INSN_ID(iobj, send)) { if (IS_INSN_ID(iobj, send)) {
struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, 0); struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, 0);
const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 2); const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 1);
#define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt)) #define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt))
if (ci->flag & VM_CALL_ARGS_SIMPLE) { if (ci->flag & VM_CALL_ARGS_SIMPLE) {
@ -4348,7 +4362,7 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const n
CHECK(COMPILE_POPPED(ret, "masgn lhs (NODE_ATTRASGN)", node)); CHECK(COMPILE_POPPED(ret, "masgn lhs (NODE_ATTRASGN)", node));
iobj = (INSN *)get_prev_insn((INSN *)LAST_ELEMENT(ret)); /* send insn */ iobj = (INSN *)get_prev_insn((INSN *)LAST_ELEMENT(ret)); /* send insn */
ci = (struct rb_call_info *)iobj->operands[0]; ci = (struct rb_call_info *)OPERAND_AT(iobj, 0);
ci->orig_argc += 1; ci->orig_argc += 1;
dupidx = INT2FIX(ci->orig_argc); dupidx = INT2FIX(ci->orig_argc);
@ -6667,14 +6681,12 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE
ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) {
VALUE str = rb_fstring(node->nd_recv->nd_lit); VALUE str = rb_fstring(node->nd_recv->nd_lit);
if (node->nd_mid == idUMinus) { if (node->nd_mid == idUMinus) {
ADD_INSN3(ret, line, opt_str_uminus, str, ADD_INSN2(ret, line, opt_str_uminus, str,
new_callinfo(iseq, idUMinus, 0, 0, NULL, FALSE), new_callinfo(iseq, idUMinus, 0, 0, NULL, FALSE));
Qundef /* CALL_CACHE */);
} }
else { else {
ADD_INSN3(ret, line, opt_str_freeze, str, ADD_INSN2(ret, line, opt_str_freeze, str,
new_callinfo(iseq, idFreeze, 0, 0, NULL, FALSE), new_callinfo(iseq, idFreeze, 0, 0, NULL, FALSE));
Qundef /* CALL_CACHE */);
} }
RB_OBJ_WRITTEN(iseq, Qundef, str); RB_OBJ_WRITTEN(iseq, Qundef, str);
if (popped) { if (popped) {
@ -6693,9 +6705,8 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE
ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) {
VALUE str = rb_fstring(node->nd_args->nd_head->nd_lit); VALUE str = rb_fstring(node->nd_args->nd_head->nd_lit);
CHECK(COMPILE(ret, "recv", node->nd_recv)); CHECK(COMPILE(ret, "recv", node->nd_recv));
ADD_INSN3(ret, line, opt_aref_with, str, ADD_INSN2(ret, line, opt_aref_with, str,
new_callinfo(iseq, idAREF, 1, 0, NULL, FALSE), new_callinfo(iseq, idAREF, 1, 0, NULL, FALSE));
NULL/* CALL_CACHE */);
RB_OBJ_WRITTEN(iseq, Qundef, str); RB_OBJ_WRITTEN(iseq, Qundef, str);
if (popped) { if (popped) {
ADD_INSN(ret, line, pop); ADD_INSN(ret, line, pop);
@ -7565,9 +7576,8 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in
ADD_INSN(ret, line, putself); ADD_INSN(ret, line, putself);
ADD_SEQ(ret, args); ADD_SEQ(ret, args);
ADD_INSN3(ret, line, invokesuper, ADD_INSN2(ret, line, invokesuper,
new_callinfo(iseq, 0, argc, flag | VM_CALL_SUPER | (type == NODE_ZSUPER ? VM_CALL_ZSUPER : 0) | VM_CALL_FCALL, keywords, parent_block != NULL), new_callinfo(iseq, 0, argc, flag | VM_CALL_SUPER | (type == NODE_ZSUPER ? VM_CALL_ZSUPER : 0) | VM_CALL_FCALL, keywords, parent_block != NULL),
Qnil, /* CALL_CACHE */
parent_block); parent_block);
if (popped) { if (popped) {
@ -8262,9 +8272,8 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in
ADD_INSN(ret, line, swap); ADD_INSN(ret, line, swap);
ADD_INSN1(ret, line, topn, INT2FIX(1)); ADD_INSN1(ret, line, topn, INT2FIX(1));
} }
ADD_INSN3(ret, line, opt_aset_with, str, ADD_INSN2(ret, line, opt_aset_with, str,
new_callinfo(iseq, idASET, 2, 0, NULL, FALSE), new_callinfo(iseq, idASET, 2, 0, NULL, FALSE));
NULL/* CALL_CACHE */);
RB_OBJ_WRITTEN(iseq, Qundef, str); RB_OBJ_WRITTEN(iseq, Qundef, str);
ADD_INSN(ret, line, pop); ADD_INSN(ret, line, pop);
break; break;
@ -8367,7 +8376,7 @@ insn_data_length(INSN *iobj)
static int static int
calc_sp_depth(int depth, INSN *insn) calc_sp_depth(int depth, INSN *insn)
{ {
return insn_stack_increase(depth, insn->insn_id, insn->operands); return comptime_insn_stack_increase(depth, insn->insn_id, insn->operands);
} }
static VALUE static VALUE
@ -8441,17 +8450,12 @@ insn_data_to_s_detail(INSN *iobj)
case TS_ISE: /* inline storage entry */ case TS_ISE: /* inline storage entry */
rb_str_catf(str, "<ic:%d>", FIX2INT(OPERAND_AT(iobj, j))); rb_str_catf(str, "<ic:%d>", FIX2INT(OPERAND_AT(iobj, j)));
break; break;
case TS_CALLINFO: /* call info */ case TS_CALLDATA: /* we store these as call infos at compile time */
{ {
struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, j); const struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, j);
rb_str_cat2(str, "<callinfo:"); rb_str_cat2(str, "<calldata:");
if (ci->mid) rb_str_catf(str, "%"PRIsVALUE, rb_id2str(ci->mid)); if (ci->mid) rb_str_catf(str, "%"PRIsVALUE, rb_id2str(ci->mid));
rb_str_catf(str, ", %d>", ci->orig_argc); rb_str_catf(str, ", %d>", ci->orig_argc);
break;
}
case TS_CALLCACHE: /* call cache */
{
rb_str_catf(str, "<call cache>");
break; break;
} }
case TS_CDHASH: /* case/when condition cache */ case TS_CDHASH: /* case/when condition cache */
@ -8832,12 +8836,9 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor,
iseq->body->is_size = NUM2INT(op) + 1; iseq->body->is_size = NUM2INT(op) + 1;
} }
break; break;
case TS_CALLINFO: case TS_CALLDATA:
argv[j] = iseq_build_callinfo_from_hash(iseq, op); argv[j] = iseq_build_callinfo_from_hash(iseq, op);
break; break;
case TS_CALLCACHE:
argv[j] = Qfalse;
break;
case TS_ID: case TS_ID:
argv[j] = rb_to_symbol_type(op); argv[j] = rb_to_symbol_type(op);
break; break;
@ -9487,9 +9488,9 @@ ibf_load_id(const struct ibf_load *load, const ID id_index)
/* dump/load: code */ /* dump/load: code */
static VALUE static VALUE
ibf_dump_callinfo(struct ibf_dump *dump, const struct rb_call_info *ci) ibf_dump_calldata(struct ibf_dump *dump, const struct rb_call_data *cd)
{ {
return (ci->flag & VM_CALL_KWARG) ? Qtrue : Qfalse; return (cd->ci.flag & VM_CALL_KWARG) ? Qtrue : Qfalse;
} }
static ibf_offset_t ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq); static ibf_offset_t ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq);
@ -9646,16 +9647,13 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq)
ibf_dump_write_small_value(dump, (VALUE)i); ibf_dump_write_small_value(dump, (VALUE)i);
} }
break; break;
case TS_CALLINFO: case TS_CALLDATA:
{ {
VALUE callinfo = ibf_dump_callinfo(dump, (const struct rb_call_info *)op); VALUE callinfo = ibf_dump_calldata(dump, (const struct rb_call_data *)op);
/* ibf_dump_callinfo() always returns either Qtrue or Qfalse */ /* ibf_dump_calldata() always returns either Qtrue or Qfalse */
ibf_dump_write_byte(dump, callinfo == Qtrue); ibf_dump_write_byte(dump, callinfo == Qtrue);
} }
break; break;
case TS_CALLCACHE:
/* do nothing */
break;
case TS_ID: case TS_ID:
ibf_dump_write_small_value(dump, ibf_dump_id(dump, (ID)op)); ibf_dump_write_small_value(dump, ibf_dump_id(dump, (ID)op));
break; break;
@ -9684,9 +9682,8 @@ ibf_load_code(const struct ibf_load *load, const rb_iseq_t *iseq, ibf_offset_t b
VALUE *code = ALLOC_N(VALUE, iseq_size); VALUE *code = ALLOC_N(VALUE, iseq_size);
struct rb_iseq_constant_body *load_body = iseq->body; struct rb_iseq_constant_body *load_body = iseq->body;
struct rb_call_info *ci_entries = load_body->ci_entries; struct rb_call_data *cd_entries = load_body->call_data;
struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&load_body->ci_entries[load_body->ci_size]; struct rb_kwarg_call_data *cd_kw_entries = (struct rb_kwarg_call_data *)&load_body->call_data[load_body->ci_size];
struct rb_call_cache *cc_entries = load_body->cc_entries;
union iseq_inline_storage_entry *is_entries = load_body->is_entries; union iseq_inline_storage_entry *is_entries = load_body->is_entries;
for (code_index=0; code_index<iseq_size;) { for (code_index=0; code_index<iseq_size;) {
@ -9730,15 +9727,12 @@ ibf_load_code(const struct ibf_load *load, const rb_iseq_t *iseq, ibf_offset_t b
code[code_index] = (VALUE)&is_entries[op]; code[code_index] = (VALUE)&is_entries[op];
} }
break; break;
case TS_CALLINFO: case TS_CALLDATA:
{ {
unsigned char op = ibf_load_byte(load, &reading_pos); unsigned char op = ibf_load_byte(load, &reading_pos);
code[code_index] = op ? (VALUE)ci_kw_entries++ : (VALUE)ci_entries++; /* op is 1 (kw) or 0 (!kw) */ code[code_index] = op ? (VALUE)cd_kw_entries++ : (VALUE)cd_entries++; /* op is 1 (kw) or 0 (!kw) */
} }
break; break;
case TS_CALLCACHE:
code[code_index] = (VALUE)cc_entries++;
break;
case TS_ID: case TS_ID:
{ {
VALUE op = ibf_load_small_value(load, &reading_pos); VALUE op = ibf_load_small_value(load, &reading_pos);
@ -10006,34 +10000,34 @@ ibf_dump_ci_entries(struct ibf_dump *dump, const rb_iseq_t *iseq)
const struct rb_iseq_constant_body *const body = iseq->body; const struct rb_iseq_constant_body *const body = iseq->body;
const unsigned int ci_size = body->ci_size; const unsigned int ci_size = body->ci_size;
const unsigned int ci_kw_size = body->ci_kw_size; const unsigned int ci_kw_size = body->ci_kw_size;
const struct rb_call_info *ci_entries = body->ci_entries; const struct rb_call_data *calls = body->call_data;
const struct rb_call_info_with_kwarg *ci_kw_entries = (const struct rb_call_info_with_kwarg *)&body->ci_entries[ci_size]; const struct rb_kwarg_call_data *kw_calls = (const struct rb_kwarg_call_data *)&body->call_data[ci_size];
ibf_offset_t offset = ibf_dump_pos(dump); ibf_offset_t offset = ibf_dump_pos(dump);
unsigned int i; unsigned int i;
for (i = 0; i < ci_size; i++) { for (i = 0; i < ci_size; i++) {
VALUE mid = ibf_dump_id(dump, ci_entries[i].mid); VALUE mid = ibf_dump_id(dump, calls[i].ci.mid);
ibf_dump_write_small_value(dump, mid); ibf_dump_write_small_value(dump, mid);
ibf_dump_write_small_value(dump, ci_entries[i].flag); ibf_dump_write_small_value(dump, calls[i].ci.flag);
ibf_dump_write_small_value(dump, ci_entries[i].orig_argc); ibf_dump_write_small_value(dump, calls[i].ci.orig_argc);
} }
for (i = 0; i < ci_kw_size; i++) { for (i = 0; i < ci_kw_size; i++) {
const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg;
VALUE mid = ibf_dump_id(dump, ci_kw_entries[i].ci.mid); VALUE mid = ibf_dump_id(dump, kw_calls[i].ci_kw.ci.mid);
ibf_dump_write_small_value(dump, mid); ibf_dump_write_small_value(dump, mid);
ibf_dump_write_small_value(dump, ci_kw_entries[i].ci.flag); ibf_dump_write_small_value(dump, kw_calls[i].ci_kw.ci.flag);
ibf_dump_write_small_value(dump, ci_kw_entries[i].ci.orig_argc); ibf_dump_write_small_value(dump, kw_calls[i].ci_kw.ci.orig_argc);
ibf_dump_write_small_value(dump, kw_arg->keyword_len); ibf_dump_write_small_value(dump, kw_arg->keyword_len);
int j; int j;
for (j = 0; j < ci_kw_entries[i].kw_arg->keyword_len; j++) { for (j = 0; j < kw_calls[i].ci_kw.kw_arg->keyword_len; j++) {
VALUE keyword = ibf_dump_object(dump, kw_arg->keywords[j]); /* kw_arg->keywords[n] is Symbol */ VALUE keyword = ibf_dump_object(dump, kw_arg->keywords[j]); /* kw_arg->keywords[n] is Symbol */
ibf_dump_write_small_value(dump, keyword); ibf_dump_write_small_value(dump, keyword);
@ -10043,7 +10037,8 @@ ibf_dump_ci_entries(struct ibf_dump *dump, const rb_iseq_t *iseq)
return offset; return offset;
} }
static struct rb_call_info * /* note that we dump out rb_call_info but load back rb_call_data */
static struct rb_call_data *
ibf_load_ci_entries(const struct ibf_load *load, ibf_load_ci_entries(const struct ibf_load *load,
ibf_offset_t ci_entries_offset, ibf_offset_t ci_entries_offset,
unsigned int ci_size, unsigned int ci_size,
@ -10053,43 +10048,43 @@ ibf_load_ci_entries(const struct ibf_load *load,
unsigned int i; unsigned int i;
struct rb_call_info *ci_entries = struct rb_call_data *calls =
rb_xmalloc_mul_add_mul( rb_xcalloc_mul_add_mul(
sizeof(struct rb_call_info), ci_size, sizeof(struct rb_call_data), ci_size,
sizeof(struct rb_call_info_with_kwarg), ci_kw_size); sizeof(struct rb_kwarg_call_data), ci_kw_size);
struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&ci_entries[ci_size]; struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&calls[ci_size];
for (i = 0; i < ci_size; i++) { for (i = 0; i < ci_size; i++) {
VALUE mid_index = ibf_load_small_value(load, &reading_pos); VALUE mid_index = ibf_load_small_value(load, &reading_pos);
ci_entries[i].mid = ibf_load_id(load, mid_index); calls[i].ci.mid = ibf_load_id(load, mid_index);
ci_entries[i].flag = (unsigned int)ibf_load_small_value(load, &reading_pos); calls[i].ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos);
ci_entries[i].orig_argc = (int)ibf_load_small_value(load, &reading_pos); calls[i].ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos);
} }
for (i = 0; i < ci_kw_size; i++) { for (i = 0; i < ci_kw_size; i++) {
VALUE mid_index = ibf_load_small_value(load, &reading_pos); VALUE mid_index = ibf_load_small_value(load, &reading_pos);
ci_kw_entries[i].ci.mid = ibf_load_id(load, mid_index); kw_calls[i].ci_kw.ci.mid = ibf_load_id(load, mid_index);
ci_kw_entries[i].ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos); kw_calls[i].ci_kw.ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos);
ci_kw_entries[i].ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos); kw_calls[i].ci_kw.ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos);
int keyword_len = (int)ibf_load_small_value(load, &reading_pos); int keyword_len = (int)ibf_load_small_value(load, &reading_pos);
ci_kw_entries[i].kw_arg = kw_calls[i].ci_kw.kw_arg =
rb_xmalloc_mul_add(keyword_len - 1, sizeof(VALUE), sizeof(struct rb_call_info_kw_arg)); rb_xmalloc_mul_add(keyword_len - 1, sizeof(VALUE), sizeof(struct rb_call_info_kw_arg));
ci_kw_entries[i].kw_arg->keyword_len = keyword_len; kw_calls[i].ci_kw.kw_arg->keyword_len = keyword_len;
int j; int j;
for (j = 0; j < ci_kw_entries[i].kw_arg->keyword_len; j++) { for (j = 0; j < kw_calls[i].ci_kw.kw_arg->keyword_len; j++) {
VALUE keyword = ibf_load_small_value(load, &reading_pos); VALUE keyword = ibf_load_small_value(load, &reading_pos);
ci_kw_entries[i].kw_arg->keywords[j] = ibf_load_object(load, keyword); kw_calls[i].ci_kw.kw_arg->keywords[j] = ibf_load_object(load, keyword);
} }
} }
return ci_entries; return calls;
} }
static ibf_offset_t static ibf_offset_t
@ -10343,8 +10338,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
load_body->catch_except_p = catch_except_p; load_body->catch_except_p = catch_except_p;
load_body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, is_size); load_body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, is_size);
load_body->ci_entries = ibf_load_ci_entries(load, ci_entries_offset, ci_size, ci_kw_size); load_body->call_data = ibf_load_ci_entries(load, ci_entries_offset, ci_size, ci_kw_size);
load_body->cc_entries = ZALLOC_N(struct rb_call_cache, ci_size + ci_kw_size);
load_body->param.opt_table = ibf_load_param_opt_table(load, param_opt_table_offset, param_opt_num); load_body->param.opt_table = ibf_load_param_opt_table(load, param_opt_table_offset, param_opt_num);
load_body->param.keyword = ibf_load_param_keyword(load, param_keyword_offset); load_body->param.keyword = ibf_load_param_keyword(load, param_keyword_offset);
load_body->param.flags.has_kw = (param_flags >> 4) & 1; load_body->param.flags.has_kw = (param_flags >> 4) & 1;
@ -10399,6 +10393,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
#if IBF_ISEQ_ENABLE_LOCAL_BUFFER #if IBF_ISEQ_ENABLE_LOCAL_BUFFER
load->current_buffer = saved_buffer; load->current_buffer = saved_buffer;
#endif #endif
verify_call_cache(iseq);
} }
static void static void

100
insns.def
View File

@ -783,13 +783,14 @@ definesmethod
/* invoke method. */ /* invoke method. */
DEFINE_INSN DEFINE_INSN
send send
(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) (CALL_DATA cd, ISEQ blockiseq)
(...) (...)
(VALUE val) (VALUE val)
// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); // attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci);
// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
{ {
VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, false); VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, false);
val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);
if (val == Qundef) { if (val == Qundef) {
RESTORE_REGS(); RESTORE_REGS();
@ -800,14 +801,15 @@ send
/* Invoke method without block */ /* Invoke method without block */
DEFINE_INSN DEFINE_INSN
opt_send_without_block opt_send_without_block
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(...) (...)
(VALUE val) (VALUE val)
// attr bool handles_sp = true; // attr bool handles_sp = true;
// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); // attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci);
// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
{ {
VALUE bh = VM_BLOCK_HANDLER_NONE; VALUE bh = VM_BLOCK_HANDLER_NONE;
val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap);
if (val == Qundef) { if (val == Qundef) {
RESTORE_REGS(); RESTORE_REGS();
@ -817,7 +819,7 @@ opt_send_without_block
DEFINE_INSN DEFINE_INSN
opt_str_freeze opt_str_freeze
(VALUE str, CALL_INFO ci, CALL_CACHE cc) (VALUE str, CALL_DATA cd)
() ()
(VALUE val) (VALUE val)
{ {
@ -832,11 +834,11 @@ opt_str_freeze
/* optimized nil? */ /* optimized nil? */
DEFINE_INSN DEFINE_INSN
opt_nil_p opt_nil_p
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
val = vm_opt_nil_p(ci, cc, recv); val = vm_opt_nil_p(cd, recv);
if (val == Qundef) { if (val == Qundef) {
CALL_SIMPLE_METHOD(); CALL_SIMPLE_METHOD();
@ -845,7 +847,7 @@ opt_nil_p
DEFINE_INSN DEFINE_INSN
opt_str_uminus opt_str_uminus
(VALUE str, CALL_INFO ci, CALL_CACHE cc) (VALUE str, CALL_DATA cd)
() ()
(VALUE val) (VALUE val)
{ {
@ -887,13 +889,14 @@ opt_newarray_min
/* super(args) # args.size => num */ /* super(args) # args.size => num */
DEFINE_INSN DEFINE_INSN
invokesuper invokesuper
(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) (CALL_DATA cd, ISEQ blockiseq)
(...) (...)
(VALUE val) (VALUE val)
// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); // attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci);
// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
{ {
VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, true); VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, true);
val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_super_method); val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_super_method);
if (val == Qundef) { if (val == Qundef) {
RESTORE_REGS(); RESTORE_REGS();
@ -904,18 +907,19 @@ invokesuper
/* yield(args) */ /* yield(args) */
DEFINE_INSN DEFINE_INSN
invokeblock invokeblock
(CALL_INFO ci) (CALL_DATA cd)
(...) (...)
(VALUE val) (VALUE val)
// attr bool handles_sp = true; // attr bool handles_sp = true;
// attr rb_snum_t sp_inc = sp_inc_of_invokeblock(ci); // attr rb_snum_t sp_inc = sp_inc_of_invokeblock(&cd->ci);
// attr rb_snum_t comptime_sp_inc = sp_inc_of_invokeblock(ci);
{ {
static struct rb_call_cache cc = { if (UNLIKELY(cd->cc.call != vm_invokeblock_i)) {
0, 0, NULL, NULL, vm_invokeblock_i, cd->cc.call = vm_invokeblock_i; // check before setting to avoid CoW
}; }
VALUE bh = VM_BLOCK_HANDLER_NONE; VALUE bh = VM_BLOCK_HANDLER_NONE;
val = vm_sendish(ec, GET_CFP(), ci, &cc, bh, vm_search_invokeblock); val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_invokeblock);
if (val == Qundef) { if (val == Qundef) {
RESTORE_REGS(); RESTORE_REGS();
@ -1098,7 +1102,7 @@ opt_case_dispatch
/* optimized X+Y. */ /* optimized X+Y. */
DEFINE_INSN DEFINE_INSN
opt_plus opt_plus
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1112,7 +1116,7 @@ opt_plus
/* optimized X-Y. */ /* optimized X-Y. */
DEFINE_INSN DEFINE_INSN
opt_minus opt_minus
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1126,7 +1130,7 @@ opt_minus
/* optimized X*Y. */ /* optimized X*Y. */
DEFINE_INSN DEFINE_INSN
opt_mult opt_mult
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1140,7 +1144,7 @@ opt_mult
/* optimized X/Y. */ /* optimized X/Y. */
DEFINE_INSN DEFINE_INSN
opt_div opt_div
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
/* In case of division by zero, it raises. Thus /* In case of division by zero, it raises. Thus
@ -1157,7 +1161,7 @@ opt_div
/* optimized X%Y. */ /* optimized X%Y. */
DEFINE_INSN DEFINE_INSN
opt_mod opt_mod
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
/* Same discussion as opt_mod. */ /* Same discussion as opt_mod. */
@ -1173,11 +1177,11 @@ opt_mod
/* optimized X==Y. */ /* optimized X==Y. */
DEFINE_INSN DEFINE_INSN
opt_eq opt_eq
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
val = opt_eq_func(recv, obj, ci, cc); val = opt_eq_func(recv, obj, cd);
if (val == Qundef) { if (val == Qundef) {
CALL_SIMPLE_METHOD(); CALL_SIMPLE_METHOD();
@ -1187,11 +1191,11 @@ opt_eq
/* optimized X!=Y. */ /* optimized X!=Y. */
DEFINE_INSN DEFINE_INSN
opt_neq opt_neq
(CALL_INFO ci_eq, CALL_CACHE cc_eq, CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd_eq, CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
val = vm_opt_neq(ci, cc, ci_eq, cc_eq, recv, obj); val = vm_opt_neq(cd, cd_eq, recv, obj);
if (val == Qundef) { if (val == Qundef) {
CALL_SIMPLE_METHOD(); CALL_SIMPLE_METHOD();
@ -1201,7 +1205,7 @@ opt_neq
/* optimized X<Y. */ /* optimized X<Y. */
DEFINE_INSN DEFINE_INSN
opt_lt opt_lt
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1215,7 +1219,7 @@ opt_lt
/* optimized X<=Y. */ /* optimized X<=Y. */
DEFINE_INSN DEFINE_INSN
opt_le opt_le
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1229,7 +1233,7 @@ opt_le
/* optimized X>Y. */ /* optimized X>Y. */
DEFINE_INSN DEFINE_INSN
opt_gt opt_gt
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1243,7 +1247,7 @@ opt_gt
/* optimized X>=Y. */ /* optimized X>=Y. */
DEFINE_INSN DEFINE_INSN
opt_ge opt_ge
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1257,7 +1261,7 @@ opt_ge
/* << */ /* << */
DEFINE_INSN DEFINE_INSN
opt_ltlt opt_ltlt
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
/* This instruction can append an integer, as a codepoint, into a /* This instruction can append an integer, as a codepoint, into a
@ -1275,7 +1279,7 @@ opt_ltlt
/* optimized X&Y. */ /* optimized X&Y. */
DEFINE_INSN DEFINE_INSN
opt_and opt_and
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1289,7 +1293,7 @@ opt_and
/* optimized X|Y. */ /* optimized X|Y. */
DEFINE_INSN DEFINE_INSN
opt_or opt_or
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
{ {
@ -1303,7 +1307,7 @@ opt_or
/* [] */ /* [] */
DEFINE_INSN DEFINE_INSN
opt_aref opt_aref
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj) (VALUE recv, VALUE obj)
(VALUE val) (VALUE val)
/* This is complicated. In case of hash, vm_opt_aref() resorts to /* This is complicated. In case of hash, vm_opt_aref() resorts to
@ -1322,7 +1326,7 @@ opt_aref
/* recv[obj] = set */ /* recv[obj] = set */
DEFINE_INSN DEFINE_INSN
opt_aset opt_aset
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv, VALUE obj, VALUE set) (VALUE recv, VALUE obj, VALUE set)
(VALUE val) (VALUE val)
/* This is another story than opt_aref. When vm_opt_aset() resorts /* This is another story than opt_aref. When vm_opt_aset() resorts
@ -1339,7 +1343,7 @@ opt_aset
/* recv[str] = set */ /* recv[str] = set */
DEFINE_INSN DEFINE_INSN
opt_aset_with opt_aset_with
(VALUE key, CALL_INFO ci, CALL_CACHE cc) (VALUE key, CALL_DATA cd)
(VALUE recv, VALUE val) (VALUE recv, VALUE val)
(VALUE val) (VALUE val)
/* Same discussion as opt_aset. */ /* Same discussion as opt_aset. */
@ -1362,7 +1366,7 @@ opt_aset_with
/* recv[str] */ /* recv[str] */
DEFINE_INSN DEFINE_INSN
opt_aref_with opt_aref_with
(VALUE key, CALL_INFO ci, CALL_CACHE cc) (VALUE key, CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
/* Same discussion as opt_aref. */ /* Same discussion as opt_aref. */
@ -1381,7 +1385,7 @@ opt_aref_with
/* optimized length */ /* optimized length */
DEFINE_INSN DEFINE_INSN
opt_length opt_length
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
@ -1395,7 +1399,7 @@ opt_length
/* optimized size */ /* optimized size */
DEFINE_INSN DEFINE_INSN
opt_size opt_size
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
@ -1409,7 +1413,7 @@ opt_size
/* optimized empty? */ /* optimized empty? */
DEFINE_INSN DEFINE_INSN
opt_empty_p opt_empty_p
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
@ -1423,7 +1427,7 @@ opt_empty_p
/* optimized succ */ /* optimized succ */
DEFINE_INSN DEFINE_INSN
opt_succ opt_succ
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
@ -1437,11 +1441,11 @@ opt_succ
/* optimized not */ /* optimized not */
DEFINE_INSN DEFINE_INSN
opt_not opt_not
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE recv) (VALUE recv)
(VALUE val) (VALUE val)
{ {
val = vm_opt_not(ci, cc, recv); val = vm_opt_not(cd, recv);
if (val == Qundef) { if (val == Qundef) {
CALL_SIMPLE_METHOD(); CALL_SIMPLE_METHOD();
@ -1451,7 +1455,7 @@ opt_not
/* optimized regexp match 2 */ /* optimized regexp match 2 */
DEFINE_INSN DEFINE_INSN
opt_regexpmatch2 opt_regexpmatch2
(CALL_INFO ci, CALL_CACHE cc) (CALL_DATA cd)
(VALUE obj2, VALUE obj1) (VALUE obj2, VALUE obj1)
(VALUE val) (VALUE val)
// attr bool leaf = false; /* match_at() has rb_thread_check_ints() */ // attr bool leaf = false; /* match_at() has rb_thread_check_ints() */

45
iseq.c
View File

@ -102,15 +102,14 @@ rb_iseq_free(const rb_iseq_t *iseq)
ruby_xfree((void *)body->local_table); ruby_xfree((void *)body->local_table);
ruby_xfree((void *)body->is_entries); ruby_xfree((void *)body->is_entries);
if (body->ci_entries) { if (body->call_data) {
unsigned int i; unsigned int i;
struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size];
for (i=0; i<body->ci_kw_size; i++) { for (i=0; i<body->ci_kw_size; i++) {
const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg;
ruby_xfree((void *)kw_arg); ruby_xfree((void *)kw_arg);
} }
ruby_xfree(body->ci_entries); ruby_xfree(body->call_data);
ruby_xfree(body->cc_entries);
} }
ruby_xfree((void *)body->catch_table); ruby_xfree((void *)body->catch_table);
ruby_xfree((void *)body->param.opt_table); ruby_xfree((void *)body->param.opt_table);
@ -379,7 +378,7 @@ rb_iseq_memsize(const rb_iseq_t *iseq)
/* TODO: should we count original_iseq? */ /* TODO: should we count original_iseq? */
if (ISEQ_EXECUTABLE_P(iseq) && body) { if (ISEQ_EXECUTABLE_P(iseq) && body) {
struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size];
size += sizeof(struct rb_iseq_constant_body); size += sizeof(struct rb_iseq_constant_body);
size += body->iseq_size * sizeof(VALUE); size += body->iseq_size * sizeof(VALUE);
@ -394,19 +393,15 @@ rb_iseq_memsize(const rb_iseq_t *iseq)
/* body->is_entries */ /* body->is_entries */
size += body->is_size * sizeof(union iseq_inline_storage_entry); size += body->is_size * sizeof(union iseq_inline_storage_entry);
/* body->ci_entries */ /* body->call_data */
size += body->ci_size * sizeof(struct rb_call_info); size += body->ci_size * sizeof(struct rb_call_data);
size += body->ci_kw_size * sizeof(struct rb_call_info_with_kwarg); size += body->ci_kw_size * sizeof(struct rb_kwarg_call_data);
/* body->cc_entries */ if (kw_calls) {
size += body->ci_size * sizeof(struct rb_call_cache);
size += body->ci_kw_size * sizeof(struct rb_call_cache);
if (ci_kw_entries) {
unsigned int i; unsigned int i;
for (i = 0; i < body->ci_kw_size; i++) { for (i = 0; i < body->ci_kw_size; i++) {
const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg;
if (kw_arg) { if (kw_arg) {
size += rb_call_info_kw_arg_bytes(kw_arg->keyword_len); size += rb_call_info_kw_arg_bytes(kw_arg->keyword_len);
@ -1916,9 +1911,10 @@ rb_insn_operand_intern(const rb_iseq_t *iseq,
ret = rb_sprintf("<is:%"PRIdPTRDIFF">", (union iseq_inline_storage_entry *)op - iseq->body->is_entries); ret = rb_sprintf("<is:%"PRIdPTRDIFF">", (union iseq_inline_storage_entry *)op - iseq->body->is_entries);
break; break;
case TS_CALLINFO: case TS_CALLDATA:
{ {
struct rb_call_info *ci = (struct rb_call_info *)op; struct rb_call_data *cd = (struct rb_call_data *)op;
struct rb_call_info *ci = &cd->ci;
VALUE ary = rb_ary_new(); VALUE ary = rb_ary_new();
if (ci->mid) { if (ci->mid) {
@ -1950,12 +1946,9 @@ rb_insn_operand_intern(const rb_iseq_t *iseq,
CALL_FLAG(OPT_SEND); /* maybe not reachable */ CALL_FLAG(OPT_SEND); /* maybe not reachable */
rb_ary_push(ary, rb_ary_join(flags, rb_str_new2("|"))); rb_ary_push(ary, rb_ary_join(flags, rb_str_new2("|")));
} }
ret = rb_sprintf("<callinfo!%"PRIsVALUE">", rb_ary_join(ary, rb_str_new2(", ")));
}
break;
case TS_CALLCACHE: ret = rb_sprintf("<calldata!%"PRIsVALUE">", rb_ary_join(ary, rb_str_new2(", ")));
ret = rb_str_new2("<callcache>"); }
break; break;
case TS_CDHASH: case TS_CDHASH:
@ -2723,9 +2716,10 @@ iseq_data_to_ary(const rb_iseq_t *iseq)
rb_ary_push(ary, INT2FIX(is - iseq_body->is_entries)); rb_ary_push(ary, INT2FIX(is - iseq_body->is_entries));
} }
break; break;
case TS_CALLINFO: case TS_CALLDATA:
{ {
struct rb_call_info *ci = (struct rb_call_info *)*seq; struct rb_call_data *cd = (struct rb_call_data *)*seq;
struct rb_call_info *ci = &cd->ci;
VALUE e = rb_hash_new(); VALUE e = rb_hash_new();
int orig_argc = ci->orig_argc; int orig_argc = ci->orig_argc;
@ -2749,9 +2743,6 @@ iseq_data_to_ary(const rb_iseq_t *iseq)
rb_ary_push(ary, e); rb_ary_push(ary, e);
} }
break; break;
case TS_CALLCACHE:
rb_ary_push(ary, Qfalse);
break;
case TS_ID: case TS_ID:
rb_ary_push(ary, ID2SYM(*seq)); rb_ary_push(ary, ID2SYM(*seq));
break; break;

11
mjit.c
View File

@ -46,7 +46,16 @@ mjit_copy_job_handler(void *data)
const struct rb_iseq_constant_body *body = job->iseq->body; const struct rb_iseq_constant_body *body = job->iseq->body;
if (job->cc_entries) { if (job->cc_entries) {
memcpy(job->cc_entries, body->cc_entries, sizeof(struct rb_call_cache) * (body->ci_size + body->ci_kw_size)); unsigned int i;
struct rb_call_cache *sink = job->cc_entries;
const struct rb_call_data *calls = body->call_data;
const struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size];
for (i = 0; i < body->ci_size; i++) {
*sink++ = calls[i].cc;
}
for (i = 0; i < body->ci_kw_size; i++) {
*sink++ = kw_calls[i].cc;
}
} }
if (job->is_entries) { if (job->is_entries) {
memcpy(job->is_entries, body->is_entries, sizeof(union iseq_inline_storage_entry) * body->is_size); memcpy(job->is_entries, body->is_entries, sizeof(union iseq_inline_storage_entry) * body->is_size);

20
mjit_compile.c
View File

@ -25,6 +25,21 @@
#define NOT_COMPILED_STACK_SIZE -1 #define NOT_COMPILED_STACK_SIZE -1
#define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE) #define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE)
static size_t
call_data_index(CALL_DATA cd, const struct rb_iseq_constant_body *body)
{
const struct rb_kwarg_call_data *kw_calls = (const struct rb_kwarg_call_data *)&body->call_data[body->ci_size];
const struct rb_kwarg_call_data *kw_cd = (const struct rb_kwarg_call_data *)cd;
VM_ASSERT(cd >= body->call_data && kw_cd < (kw_calls + body->ci_kw_size));
if (kw_cd < kw_calls) {
return cd - body->call_data;
}
else {
return kw_cd - kw_calls + body->ci_size;
}
}
// For propagating information needed for lazily pushing a frame. // For propagating information needed for lazily pushing a frame.
struct inlined_call_context { struct inlined_call_context {
int orig_argc; // ci->orig_argc int orig_argc; // ci->orig_argc
@ -383,8 +398,9 @@ precompile_inlinable_iseqs(FILE *f, const rb_iseq_t *iseq, struct compile_status
#endif #endif
if (insn == BIN(opt_send_without_block)) { // `compile_inlined_cancel_handler` supports only `opt_send_without_block` if (insn == BIN(opt_send_without_block)) { // `compile_inlined_cancel_handler` supports only `opt_send_without_block`
CALL_INFO ci = (CALL_INFO)body->iseq_encoded[pos + 1]; CALL_DATA cd = (CALL_DATA)body->iseq_encoded[pos + 1];
CALL_CACHE cc_copy = status->cc_entries + ((CALL_CACHE)body->iseq_encoded[pos + 2] - body->cc_entries); // use copy to avoid race condition CALL_INFO ci = &cd->ci;
CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); // use copy to avoid race condition
const rb_iseq_t *child_iseq; const rb_iseq_t *child_iseq;
if (has_valid_method_type(cc_copy) && if (has_valid_method_type(cc_copy) &&

19
tool/ruby_vm/models/attribute.rb
View File

@ -21,6 +21,13 @@ class RubyVM::Attribute
@key = opts[:name] @key = opts[:name]
@expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr]
@type = opts[:type] @type = opts[:type]
@ope_decls = @insn.opes.map do |operand|
decl = operand[:decl]
if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA'
decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci')
end
decl
end
end end
def name def name
@ -32,22 +39,20 @@ class RubyVM::Attribute
end end
def declaration def declaration
opes = @insn.opes if @ope_decls.empty?
if opes.empty?
argv = "void" argv = "void"
else else
argv = opes.map {|o| o[:decl] }.join(', ') argv = @ope_decls.join(', ')
end end
sprintf '%s %s(%s)', @type, name, argv sprintf '%s %s(%s)', @type, name, argv
end end
def definition def definition
opes = @insn.opes if @ope_decls.empty?
if opes.empty?
argv = "void" argv = "void"
else else
argv = opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n ") argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ")
argv = "\n #{argv}\n" if opes.size > 1 argv = "\n #{argv}\n" if @ope_decls.size > 1
end end
sprintf "%s\n%s(%s)", @type, name, argv sprintf "%s\n%s(%s)", @type, name, argv
end end

21
tool/ruby_vm/models/bare_instructions.rb
View File

@ -33,6 +33,7 @@ class RubyVM::BareInstructions
h[a.key] = a h[a.key] = a
} }
@attrs_orig = @attrs.dup @attrs_orig = @attrs.dup
check_attribute_consistency
predefine_attributes predefine_attributes
end end
@ -139,8 +140,28 @@ class RubyVM::BareInstructions
return @pops.any? {|i| i[:name] == var[:name] } return @pops.any? {|i| i[:name] == var[:name] }
end end
def use_call_data?
@use_call_data ||=
@variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' }
end
private private
def check_attribute_consistency
if has_attribute?('sp_inc') \
&& use_call_data? \
&& !has_attribute?('comptime_sp_inc')
# As the call cache caches information that can only be obtained at
# runtime, we do not need it when compiling from AST to bytecode. This
# attribute defines an expression that computes the stack pointer
# increase based on just the call info to avoid reserving space for the
# call cache at compile time. In the expression, all call data operands
# are mapped to their call info counterpart. Additionally, all mentions
# of `cd` in the operand name are replaced with `ci`.
raise "Please define attribute `comptime_sp_inc` for `#{@name}`"
end
end
def generate_attribute t, k, v def generate_attribute t, k, v
@attrs[k] ||= RubyVM::Attribute.new \ @attrs[k] ||= RubyVM::Attribute.new \
insn: self, \ insn: self, \

3
tool/ruby_vm/models/typemap.rb
View File

@ -12,8 +12,7 @@
RubyVM::Typemap = { RubyVM::Typemap = {
"..." => %w[. TS_VARIABLE], "..." => %w[. TS_VARIABLE],
"CALL_CACHE" => %w[E TS_CALLCACHE], "CALL_DATA" => %w[C TS_CALLDATA],
"CALL_INFO" => %w[C TS_CALLINFO],
"CDHASH" => %w[H TS_CDHASH], "CDHASH" => %w[H TS_CDHASH],
"GENTRY" => %w[G TS_GENTRY], "GENTRY" => %w[G TS_GENTRY],
"IC" => %w[K TS_IC], "IC" => %w[K TS_IC],

22
tool/ruby_vm/views/_insn_stack_increase.erb → tool/ruby_vm/views/_comptime_insn_stack_increase.erb
View File

@ -6,11 +6,11 @@
%# conditions mentioned in the file COPYING are met. Consult the file for %# conditions mentioned in the file COPYING are met. Consult the file for
%# details. %# details.
%# %#
PUREFUNC(MAYBE_UNUSED(static int insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes)));
PUREFUNC(static rb_snum_t insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes));
rb_snum_t rb_snum_t
insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
{ {
static const signed char t[] = { static const signed char t[] = {
% RubyVM::Instructions.each_slice 8 do |a| % RubyVM::Instructions.each_slice 8 do |a|
@ -34,9 +34,19 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
UNREACHABLE; UNREACHABLE;
% RubyVM::Instructions.each do |i| % RubyVM::Instructions.each do |i|
% next unless i.has_attribute?('sp_inc') % next unless i.has_attribute?('sp_inc')
% attr_function =
% if i.has_attribute?('comptime_sp_inc')
% "attr_comptime_sp_inc_#{i.name}"
% else
% "attr_sp_inc_#{i.name}"
% end
case <%= i.bin %>: case <%= i.bin %>:
return attr_sp_inc_<%= i.name %>(<%= return <%= attr_function %>(<%=
i.opes.map.with_index do |v, j| i.opes.map.with_index do |v, j|
if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc')
v = v.dup
v[:type] = 'CALL_INFO'
end
i.cast_from_VALUE v, "opes[#{j}]" i.cast_from_VALUE v, "opes[#{j}]"
end.join(", ") end.join(", ")
%>); %>);
@ -45,8 +55,8 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
} }
int int
insn_stack_increase(int depth, int insn, const VALUE *opes) comptime_insn_stack_increase(int depth, int insn, const VALUE *opes)
{ {
enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn;
return depth + (int)insn_stack_increase_dispatch(itype, opes); return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes);
} }

4
tool/ruby_vm/views/_mjit_compile_insn.erb
View File

@ -34,8 +34,8 @@
% case ope.fetch(:type) % case ope.fetch(:type)
% when 'ID' % when 'ID'
comment_id(f, (ID)operands[<%= i %>]); comment_id(f, (ID)operands[<%= i %>]);
% when 'CALL_INFO' % when 'CALL_DATA'
comment_id(f, ((CALL_INFO)operands[<%= i %>])->mid); comment_id(f, ((CALL_DATA)operands[<%= i %>])->ci.mid);
% when 'VALUE' % when 'VALUE'
if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>]));
% end % end

5
tool/ruby_vm/views/_mjit_compile_send.erb
View File

@ -14,10 +14,11 @@
MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>];
% end % end
% # compiler: Use copied cc to avoid race condition % # compiler: Use copied cc to avoid race condition
CALL_CACHE cc_copy = status->cc_entries + (cc - body->cc_entries); CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body);
% %
if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) {
const rb_iseq_t *iseq; const rb_iseq_t *iseq;
const CALL_INFO ci = &cd->ci;
unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`.
% if insn.name == 'send' % if insn.name == 'send'
argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp` argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp`
@ -58,7 +59,7 @@
fprintf(f, " {\n"); fprintf(f, " {\n");
fprintf(f, " struct rb_calling_info calling;\n"); fprintf(f, " struct rb_calling_info calling;\n");
% if insn.name == 'send' % if insn.name == 'send'
fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", operands[0], operands[2]); fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq);
% else % else
fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n");
% end % end

2
tool/ruby_vm/views/insns_info.inc.erb
View File

@ -18,5 +18,5 @@
<%= render 'leaf_helpers' %> <%= render 'leaf_helpers' %>
<%= render 'sp_inc_helpers' %> <%= render 'sp_inc_helpers' %>
<%= render 'attributes' %> <%= render 'attributes' %>
<%= render 'insn_stack_increase' %> <%= render 'comptime_insn_stack_increase' %>
<%= render 'insn_sp_pc_dependency' %> <%= render 'insn_sp_pc_dependency' %>

4
tool/ruby_vm/views/mjit_compile.inc.erb
View File

@ -56,8 +56,8 @@ switch (insn) {
<%= render 'mjit_compile_send', locals: { insn: insn } -%> <%= render 'mjit_compile_send', locals: { insn: insn } -%>
% when *send_compatible_opt_insns % when *send_compatible_opt_insns
% # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. % # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated.
% cc_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_CACHE' } % cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' }
if (has_valid_method_type(status->cc_entries + ((CALL_CACHE)operands[<%= cc_index %>] - body->cc_entries))) { if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[<%= cd_index %>], body))) {
<%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> <%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%>
<%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> <%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%>
break; break;

23
vm_core.h
View File

@ -258,6 +258,16 @@ struct rb_calling_info {
int kw_splat; int kw_splat;
}; };
struct rb_call_data {
struct rb_call_cache cc;
struct rb_call_info ci;
};
struct rb_kwarg_call_data {
struct rb_call_cache cc;
struct rb_call_info_with_kwarg ci_kw;
};
struct rb_execution_context_struct; struct rb_execution_context_struct;
typedef VALUE (*vm_call_handler)(struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc); typedef VALUE (*vm_call_handler)(struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc);
@ -417,12 +427,12 @@ struct rb_iseq_constant_body {
struct rb_iseq_struct *local_iseq; /* local_iseq->flip_cnt can be modified */ struct rb_iseq_struct *local_iseq; /* local_iseq->flip_cnt can be modified */
union iseq_inline_storage_entry *is_entries; union iseq_inline_storage_entry *is_entries;
struct rb_call_info *ci_entries; /* struct rb_call_info ci_entries[ci_size]; struct rb_call_data *call_data; /* A buffer for two arrays:
* struct rb_call_info_with_kwarg cikw_entries[ci_kw_size]; * struct rb_call_data calls[ci_size];
* So that: * struct rb_kwarg_call_data kw_calls[ci_kw_size];
* struct rb_call_info_with_kwarg *cikw_entries = &body->ci_entries[ci_size]; * Such that:
*/ * struct rb_kwarg_call_data *kw_calls = &body->call_data[ci_size];
struct rb_call_cache *cc_entries; /* size is ci_size + ci_kw_size */ */
struct { struct {
rb_snum_t flip_count; rb_snum_t flip_count;
@ -1121,6 +1131,7 @@ typedef struct iseq_inline_cache_entry *IC;
typedef union iseq_inline_storage_entry *ISE; typedef union iseq_inline_storage_entry *ISE;
typedef struct rb_call_info *CALL_INFO; typedef struct rb_call_info *CALL_INFO;
typedef struct rb_call_cache *CALL_CACHE; typedef struct rb_call_cache *CALL_CACHE;
typedef struct rb_call_data *CALL_DATA;
void rb_vm_change_state(void); void rb_vm_change_state(void);

66
vm_insnhelper.c
View File

@ -1463,17 +1463,16 @@ check_cfunc(const rb_callable_method_entry_t *me, VALUE (*func)())
} }
static inline int static inline int
vm_method_cfunc_is(CALL_INFO ci, CALL_CACHE cc, vm_method_cfunc_is(CALL_DATA cd, VALUE recv, VALUE (*func)())
VALUE recv, VALUE (*func)())
{ {
vm_search_method(ci, cc, recv); vm_search_method(&cd->ci, &cd->cc, recv);
return check_cfunc(cc->me, func); return check_cfunc(cd->cc.me, func);
} }
static VALUE static VALUE
opt_equal_fallback(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) opt_equal_fallback(VALUE recv, VALUE obj, CALL_DATA cd)
{ {
if (vm_method_cfunc_is(ci, cc, recv, rb_obj_equal)) { if (vm_method_cfunc_is(cd, recv, rb_obj_equal)) {
return recv == obj ? Qtrue : Qfalse; return recv == obj ? Qtrue : Qfalse;
} }
@ -1533,7 +1532,7 @@ static
inline inline
#endif #endif
VALUE VALUE
opt_eq_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) opt_eq_func(VALUE recv, VALUE obj, CALL_DATA cd)
{ {
switch (comparable_by_identity(recv, obj)) { switch (comparable_by_identity(recv, obj)) {
case 1: case 1:
@ -1556,7 +1555,7 @@ opt_eq_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc)
} }
fallback: fallback:
return opt_equal_fallback(recv, obj, ci, cc); return opt_equal_fallback(recv, obj, cd);
} }
static static
@ -1564,7 +1563,7 @@ static
inline inline
#endif #endif
VALUE VALUE
opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) opt_eql_func(VALUE recv, VALUE obj, CALL_DATA cd)
{ {
switch (comparable_by_identity(recv, obj)) { switch (comparable_by_identity(recv, obj)) {
case 1: case 1:
@ -1586,7 +1585,7 @@ opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc)
} }
fallback: fallback:
return opt_equal_fallback(recv, obj, ci, cc); return opt_equal_fallback(recv, obj, cd);
} }
#undef BUILTIN_CLASS_P #undef BUILTIN_CLASS_P
#undef EQ_UNREDEFINED_P #undef EQ_UNREDEFINED_P
@ -1594,27 +1593,25 @@ opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc)
VALUE VALUE
rb_equal_opt(VALUE obj1, VALUE obj2) rb_equal_opt(VALUE obj1, VALUE obj2)
{ {
struct rb_call_info ci; struct rb_call_data cd;
struct rb_call_cache cc;
ci.mid = idEq; cd.ci.mid = idEq;
cc.method_state = 0; cd.cc.method_state = 0;
cc.class_serial = 0; cd.cc.class_serial = 0;
cc.me = NULL; cd.cc.me = NULL;
return opt_eq_func(obj1, obj2, &ci, &cc); return opt_eq_func(obj1, obj2, &cd);
} }
VALUE VALUE
rb_eql_opt(VALUE obj1, VALUE obj2) rb_eql_opt(VALUE obj1, VALUE obj2)
{ {
struct rb_call_info ci; struct rb_call_data cd;
struct rb_call_cache cc;
ci.mid = idEqlP; cd.ci.mid = idEqlP;
cc.method_state = 0; cd.cc.method_state = 0;
cc.class_serial = 0; cd.cc.class_serial = 0;
cc.me = NULL; cd.cc.me = NULL;
return opt_eql_func(obj1, obj2, &ci, &cc); return opt_eql_func(obj1, obj2, &cd);
} }
extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat); extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
@ -3752,8 +3749,7 @@ static VALUE
vm_sendish( vm_sendish(
struct rb_execution_context_struct *ec, struct rb_execution_context_struct *ec,
struct rb_control_frame_struct *reg_cfp, struct rb_control_frame_struct *reg_cfp,
struct rb_call_info *ci, struct rb_call_data *cd,
struct rb_call_cache *cc,
VALUE block_handler, VALUE block_handler,
void (*method_explorer)( void (*method_explorer)(
const struct rb_control_frame_struct *reg_cfp, const struct rb_control_frame_struct *reg_cfp,
@ -3761,6 +3757,8 @@ vm_sendish(
struct rb_call_cache *cc, struct rb_call_cache *cc,
VALUE recv)) VALUE recv))
{ {
CALL_INFO ci = &cd->ci;
CALL_CACHE cc = &cd->cc;
VALUE val; VALUE val;
int argc = ci->orig_argc; int argc = ci->orig_argc;
VALUE recv = TOPN(argc); VALUE recv = TOPN(argc);
@ -4116,12 +4114,10 @@ vm_opt_mod(VALUE recv, VALUE obj)
} }
static VALUE static VALUE
vm_opt_neq(CALL_INFO ci, CALL_CACHE cc, vm_opt_neq(CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
CALL_INFO ci_eq, CALL_CACHE cc_eq,
VALUE recv, VALUE obj)
{ {
if (vm_method_cfunc_is(ci, cc, recv, rb_obj_not_equal)) { if (vm_method_cfunc_is(cd, recv, rb_obj_not_equal)) {
VALUE val = opt_eq_func(recv, obj, ci_eq, cc_eq); VALUE val = opt_eq_func(recv, obj, cd_eq);
if (val != Qundef) { if (val != Qundef) {
return RTEST(val) ? Qfalse : Qtrue; return RTEST(val) ? Qfalse : Qtrue;
@ -4392,7 +4388,7 @@ vm_opt_empty_p(VALUE recv)
VALUE rb_false(VALUE obj); VALUE rb_false(VALUE obj);
static VALUE static VALUE
vm_opt_nil_p(CALL_INFO ci, CALL_CACHE cc, VALUE recv) vm_opt_nil_p(CALL_DATA cd, VALUE recv)
{ {
if (recv == Qnil) { if (recv == Qnil) {
if (BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) { if (BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) {
@ -4403,7 +4399,7 @@ vm_opt_nil_p(CALL_INFO ci, CALL_CACHE cc, VALUE recv)
} }
} }
else { else {
if (vm_method_cfunc_is(ci, cc, recv, rb_false)) { if (vm_method_cfunc_is(cd, recv, rb_false)) {
return Qfalse; return Qfalse;
} }
else { else {
@ -4460,9 +4456,9 @@ vm_opt_succ(VALUE recv)
} }
static VALUE static VALUE
vm_opt_not(CALL_INFO ci, CALL_CACHE cc, VALUE recv) vm_opt_not(CALL_DATA cd, VALUE recv)
{ {
if (vm_method_cfunc_is(ci, cc, recv, rb_obj_not)) { if (vm_method_cfunc_is(cd, recv, rb_obj_not)) {
return RTEST(recv) ? Qfalse : Qtrue; return RTEST(recv) ? Qfalse : Qtrue;
} }
else { else {

2
vm_insnhelper.h
View File

@ -166,7 +166,7 @@ enum vm_regan_acttype {
#ifndef MJIT_HEADER #ifndef MJIT_HEADER
#define CALL_SIMPLE_METHOD() do { \ #define CALL_SIMPLE_METHOD() do { \
rb_snum_t x = leaf ? INSN_ATTR(width) : 0; \ rb_snum_t x = leaf ? INSN_ATTR(width) : 0; \
rb_snum_t y = attr_width_opt_send_without_block(0, 0); \ rb_snum_t y = attr_width_opt_send_without_block(0); \
rb_snum_t z = x - y; \ rb_snum_t z = x - y; \
ADD_PC(z); \ ADD_PC(z); \
DISPATCH_ORIGINAL_INSN(opt_send_without_block); \ DISPATCH_ORIGINAL_INSN(opt_send_without_block); \