YJIT: Refactor into gen_push_frame (#6412)
This refactors the "push frame" operation common to both gen_send_iseq and gen_send_cfunc into its own method. This allows that logic to live in one place.
This commit is contained in:
parent
0c9dc01a2a
commit
4b97f1e525
Notes:
git
2022-09-23 06:47:48 +09:00
Merged-By: maximecb <maximecb@ruby-lang.org>
@ -3998,6 +3998,135 @@ unsafe extern "C" fn build_kwhash(ci: *const rb_callinfo, sp: *const VALUE) -> V
|
|||||||
hash
|
hash
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum BlockHandler {
|
||||||
|
None,
|
||||||
|
CurrentFrame,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ControlFrame {
|
||||||
|
recv: Opnd,
|
||||||
|
sp: Opnd,
|
||||||
|
iseq: Option<IseqPtr>,
|
||||||
|
pc: Option<u64>,
|
||||||
|
frame_type: u32,
|
||||||
|
block_handler: BlockHandler,
|
||||||
|
cme: *const rb_callable_method_entry_t,
|
||||||
|
local_size: i32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Codegen performing a similar (but not identical) function to vm_push_frame
|
||||||
|
//
|
||||||
|
// This will generate the code to:
|
||||||
|
// * initialize locals to Qnil
|
||||||
|
// * push the environment (cme, block handler, frame type)
|
||||||
|
// * push a new CFP
|
||||||
|
// * save the new CFP to ec->cfp
|
||||||
|
//
|
||||||
|
// Notes:
|
||||||
|
// * Provided sp should point to the new frame's sp, immediately following locals and the environment
|
||||||
|
// * At entry, CFP points to the caller (not callee) frame
|
||||||
|
// * At exit, ec->cfp is updated to the pushed CFP
|
||||||
|
// * CFP and SP registers are updated only if switch_in_jit is set
|
||||||
|
// * Stack overflow is not checked (should be done by the caller)
|
||||||
|
// * Interrupts are not checked (should be done by the caller)
|
||||||
|
fn gen_push_frame(
|
||||||
|
jit: &mut JITState,
|
||||||
|
ctx: &mut Context,
|
||||||
|
asm: &mut Assembler,
|
||||||
|
set_pc_cfp: bool, // if true CFP and SP will be switched to the callee
|
||||||
|
frame: ControlFrame,
|
||||||
|
) {
|
||||||
|
assert!(frame.local_size >= 0);
|
||||||
|
|
||||||
|
let sp = frame.sp;
|
||||||
|
|
||||||
|
let num_locals = frame.local_size;
|
||||||
|
if num_locals > 0 {
|
||||||
|
asm.comment("initialize locals");
|
||||||
|
|
||||||
|
// Initialize local variables to Qnil
|
||||||
|
for i in 0..num_locals {
|
||||||
|
let offs = (SIZEOF_VALUE as i32) * (i - num_locals - 3);
|
||||||
|
asm.store(Opnd::mem(64, sp, offs), Qnil.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
asm.comment("push cme, block handler, frame type");
|
||||||
|
|
||||||
|
// Write method entry at sp[-3]
|
||||||
|
// sp[-3] = me;
|
||||||
|
// Use compile time cme. It's assumed to be valid because we are notified when
|
||||||
|
// any cme we depend on become outdated. See yjit_method_lookup_change().
|
||||||
|
asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -3), VALUE::from(frame.cme).into());
|
||||||
|
|
||||||
|
// Write block handler at sp[-2]
|
||||||
|
// sp[-2] = block_handler;
|
||||||
|
let block_handler: Opnd = match frame.block_handler {
|
||||||
|
BlockHandler::None => {
|
||||||
|
VM_BLOCK_HANDLER_NONE.into()
|
||||||
|
},
|
||||||
|
BlockHandler::CurrentFrame => {
|
||||||
|
let cfp_self = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF));
|
||||||
|
asm.or(cfp_self, Opnd::Imm(1))
|
||||||
|
},
|
||||||
|
};
|
||||||
|
asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2), block_handler);
|
||||||
|
|
||||||
|
// Write env flags at sp[-1]
|
||||||
|
// sp[-1] = frame_type;
|
||||||
|
asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -1), frame.frame_type.into());
|
||||||
|
|
||||||
|
// Allocate a new CFP (ec->cfp--)
|
||||||
|
fn cfp_opnd(offset: i32) -> Opnd {
|
||||||
|
Opnd::mem(64, CFP, offset - (RUBY_SIZEOF_CONTROL_FRAME as i32))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup the new frame
|
||||||
|
// *cfp = (const struct rb_control_frame_struct) {
|
||||||
|
// .pc = <unset for iseq, 0 for cfunc>,
|
||||||
|
// .sp = sp,
|
||||||
|
// .iseq = <iseq for iseq, 0 for cfunc>,
|
||||||
|
// .self = recv,
|
||||||
|
// .ep = <sp - 1>,
|
||||||
|
// .block_code = 0,
|
||||||
|
// .__bp__ = sp,
|
||||||
|
// };
|
||||||
|
asm.comment("push callee control frame");
|
||||||
|
|
||||||
|
// For an iseq call PC may be None, in which case we will not set PC and will allow jitted code
|
||||||
|
// to set it as necessary.
|
||||||
|
let pc = if let Some(pc) = frame.pc {
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), pc.into());
|
||||||
|
};
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BP), sp);
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), sp);
|
||||||
|
let iseq: Opnd = if let Some(iseq) = frame.iseq {
|
||||||
|
VALUE::from(iseq).into()
|
||||||
|
} else {
|
||||||
|
0.into()
|
||||||
|
};
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), iseq);
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv);
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into());
|
||||||
|
|
||||||
|
if set_pc_cfp {
|
||||||
|
// Saving SP before calculating ep avoids a dependency on a register
|
||||||
|
// However this must be done after referencing frame.recv, which may be SP-relative
|
||||||
|
asm.mov(SP, sp);
|
||||||
|
}
|
||||||
|
let ep = asm.sub(sp, SIZEOF_VALUE.into());
|
||||||
|
asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep);
|
||||||
|
|
||||||
|
asm.comment("switch to new CFP");
|
||||||
|
let new_cfp = asm.lea(cfp_opnd(0));
|
||||||
|
if set_pc_cfp {
|
||||||
|
asm.mov(CFP, new_cfp);
|
||||||
|
asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP);
|
||||||
|
} else {
|
||||||
|
asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), new_cfp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn gen_send_cfunc(
|
fn gen_send_cfunc(
|
||||||
jit: &mut JITState,
|
jit: &mut JITState,
|
||||||
ctx: &mut Context,
|
ctx: &mut Context,
|
||||||
@ -4103,60 +4232,27 @@ fn gen_send_cfunc(
|
|||||||
// sp += 3
|
// sp += 3
|
||||||
let sp = asm.lea(ctx.sp_opnd((SIZEOF_VALUE as isize) * 3));
|
let sp = asm.lea(ctx.sp_opnd((SIZEOF_VALUE as isize) * 3));
|
||||||
|
|
||||||
// Write method entry at sp[-3]
|
let frame_block_handler = if let Some(_block_iseq) = block {
|
||||||
// sp[-3] = me;
|
BlockHandler::CurrentFrame
|
||||||
// Put compile time cme into REG1. It's assumed to be valid because we are notified when
|
|
||||||
// any cme we depend on become outdated. See yjit_method_lookup_change().
|
|
||||||
asm.comment("push cme, block handler, frame type");
|
|
||||||
asm.mov(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -3), Opnd::UImm(cme as u64));
|
|
||||||
|
|
||||||
// Write block handler at sp[-2]
|
|
||||||
// sp[-2] = block_handler;
|
|
||||||
if let Some(_block_iseq) = block {
|
|
||||||
// reg1 = VM_BH_FROM_ISEQ_BLOCK(VM_CFP_TO_CAPTURED_BLOCK(reg_cfp));
|
|
||||||
let cfp_self = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF));
|
|
||||||
let block_handler = asm.or(cfp_self, Opnd::Imm(1));
|
|
||||||
asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2), block_handler);
|
|
||||||
} else {
|
} else {
|
||||||
let dst_opnd = Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2);
|
BlockHandler::None
|
||||||
asm.store(dst_opnd, Opnd::UImm(VM_BLOCK_HANDLER_NONE.into()));
|
};
|
||||||
}
|
|
||||||
|
|
||||||
// Write env flags at sp[-1]
|
|
||||||
// sp[-1] = frame_type;
|
|
||||||
let mut frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
|
let mut frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
|
||||||
if !kw_arg.is_null() {
|
if !kw_arg.is_null() {
|
||||||
frame_type |= VM_FRAME_FLAG_CFRAME_KW
|
frame_type |= VM_FRAME_FLAG_CFRAME_KW
|
||||||
}
|
}
|
||||||
asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -1), Opnd::UImm(frame_type.into()));
|
|
||||||
|
|
||||||
// Allocate a new CFP (ec->cfp--)
|
gen_push_frame(jit, ctx, asm, false, ControlFrame {
|
||||||
asm.comment("push callee control frame");
|
frame_type,
|
||||||
let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP);
|
block_handler: frame_block_handler,
|
||||||
let new_cfp = asm.sub(ec_cfp_opnd, Opnd::UImm(RUBY_SIZEOF_CONTROL_FRAME as u64));
|
cme,
|
||||||
asm.mov(ec_cfp_opnd, new_cfp);
|
recv,
|
||||||
|
sp,
|
||||||
// Setup the new frame
|
pc: Some(0),
|
||||||
// *cfp = (const struct rb_control_frame_struct) {
|
iseq: None,
|
||||||
// .pc = 0,
|
local_size: 0,
|
||||||
// .sp = sp,
|
});
|
||||||
// .iseq = 0,
|
|
||||||
// .self = recv,
|
|
||||||
// .ep = sp - 1,
|
|
||||||
// .block_code = 0,
|
|
||||||
// .__bp__ = sp,
|
|
||||||
// };
|
|
||||||
|
|
||||||
// Can we re-use ec_cfp_opnd from above?
|
|
||||||
let ec_cfp_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP));
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_PC), Opnd::Imm(0));
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_SP), sp);
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_ISEQ), Opnd::Imm(0));
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_BLOCK_CODE), Opnd::Imm(0));
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_BP), sp);
|
|
||||||
let ep = asm.sub(sp, Opnd::UImm(SIZEOF_VALUE as u64));
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_EP), ep);
|
|
||||||
asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_SELF), recv);
|
|
||||||
|
|
||||||
if !kw_arg.is_null() {
|
if !kw_arg.is_null() {
|
||||||
// Build a hash from all kwargs passed
|
// Build a hash from all kwargs passed
|
||||||
@ -4219,10 +4315,10 @@ fn gen_send_cfunc(
|
|||||||
asm.mov(stack_ret, ret);
|
asm.mov(stack_ret, ret);
|
||||||
|
|
||||||
// Pop the stack frame (ec->cfp++)
|
// Pop the stack frame (ec->cfp++)
|
||||||
// Can we reuse ec_cfp_opnd from above?
|
// Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved
|
||||||
|
// register
|
||||||
let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP);
|
let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP);
|
||||||
let new_cfp = asm.add(ec_cfp_opnd, Opnd::UImm(RUBY_SIZEOF_CONTROL_FRAME as u64));
|
asm.store(ec_cfp_opnd, CFP);
|
||||||
asm.store(ec_cfp_opnd, new_cfp);
|
|
||||||
|
|
||||||
// cfunc calls may corrupt types
|
// cfunc calls may corrupt types
|
||||||
ctx.clear_local_types();
|
ctx.clear_local_types();
|
||||||
@ -4766,62 +4862,25 @@ fn gen_send_iseq(
|
|||||||
(SIZEOF_VALUE as isize) * (3 + (num_locals as isize) + if doing_kw_call { 1 } else { 0 });
|
(SIZEOF_VALUE as isize) * (3 + (num_locals as isize) + if doing_kw_call { 1 } else { 0 });
|
||||||
let callee_sp = asm.lea(ctx.sp_opnd(offs));
|
let callee_sp = asm.lea(ctx.sp_opnd(offs));
|
||||||
|
|
||||||
// Initialize local variables to Qnil
|
let frame_block_handler = if let Some(_) = block {
|
||||||
for i in 0..num_locals {
|
BlockHandler::CurrentFrame
|
||||||
let offs = (SIZEOF_VALUE as i32) * (i - num_locals - 3);
|
} else {
|
||||||
asm.store(Opnd::mem(64, callee_sp, offs), Qnil.into());
|
BlockHandler::None
|
||||||
}
|
};
|
||||||
|
|
||||||
// Write the callee CME on the stack. It's assumed to be valid because we are notified when
|
|
||||||
// any cme we depend on become outdated. See yjit_method_lookup_change().
|
|
||||||
// Write method entry at sp[-3]
|
|
||||||
// sp[-3] = me;
|
|
||||||
asm.comment("push cme, block handler, frame type");
|
|
||||||
asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -3), VALUE(cme as usize).into());
|
|
||||||
|
|
||||||
// Write block handler at sp[-2]
|
|
||||||
// sp[-2] = block_handler;
|
|
||||||
match block {
|
|
||||||
Some(_) => {
|
|
||||||
// reg1 = VM_BH_FROM_ISEQ_BLOCK(VM_CFP_TO_CAPTURED_BLOCK(reg_cfp));
|
|
||||||
let block_handler = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF));
|
|
||||||
let block_handler = asm.or(block_handler, 1.into());
|
|
||||||
asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -2), block_handler);
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -2), VM_BLOCK_HANDLER_NONE.into());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write env flags at sp[-1]
|
|
||||||
// sp[-1] = frame_type;
|
|
||||||
let frame_type = VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL;
|
let frame_type = VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL;
|
||||||
asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -1), frame_type.into());
|
|
||||||
|
|
||||||
asm.comment("push callee control frame");
|
|
||||||
// Allocate a new CFP (ec->cfp--)
|
|
||||||
let new_cfp = asm.sub(CFP, (RUBY_SIZEOF_CONTROL_FRAME as u64).into());
|
|
||||||
asm.mov(CFP, new_cfp);
|
|
||||||
asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP);
|
|
||||||
|
|
||||||
// Setup the new frame
|
// Setup the new frame
|
||||||
// *cfp = (const struct rb_control_frame_struct) {
|
gen_push_frame(jit, ctx, asm, true, ControlFrame {
|
||||||
// .pc = pc,
|
frame_type,
|
||||||
// .sp = sp,
|
block_handler: frame_block_handler,
|
||||||
// .iseq = iseq,
|
cme,
|
||||||
// .self = recv,
|
recv,
|
||||||
// .ep = sp - 1,
|
sp: callee_sp,
|
||||||
// .block_code = 0,
|
iseq: Some(iseq),
|
||||||
// .__bp__ = sp,
|
pc: None, // We are calling into jitted code, which will set the PC as necessary
|
||||||
// };
|
local_size: num_locals
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), recv);
|
});
|
||||||
asm.mov(SP, callee_sp);
|
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), callee_sp);
|
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BP), callee_sp);
|
|
||||||
let callee_ep = asm.sub(callee_sp, (SIZEOF_VALUE as u64).into());
|
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP), callee_ep);
|
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE(iseq as usize).into());
|
|
||||||
asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), 0.into());
|
|
||||||
|
|
||||||
// No need to set cfp->pc since the callee sets it whenever calling into routines
|
// No need to set cfp->pc since the callee sets it whenever calling into routines
|
||||||
// that could look at it through jit_save_pc().
|
// that could look at it through jit_save_pc().
|
||||||
|
Loading…
x
Reference in New Issue
Block a user