1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

YJIT: Merge csel and mov on arm64 (#7747)

Browse Source 
* YJIT: Refactor arm64_split with &mut insn

* YJIT: Merge csel and mov on arm64
This commit is contained in:
Takashi Kokubun 2023-04-20 13:08:42 -07:00 committed by GitHub
parent 995b960c70
commit 64a25977ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
Notes: git 2023-04-20 20:09:12 +00:00 
Merged-By: k0kubun <takashikkbn@gmail.com>
1 changed files with 91 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
yjit/src/backend/arm64/mod.rs
View File

@ -376,6 +376,7 @@ impl Assembler
} }
} }
let live_ranges: Vec<usize> = take(&mut self.live_ranges);
let mut asm_local = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits)); let mut asm_local = Assembler::new_with_label_names(take(&mut self.label_names), take(&mut self.side_exits));
let asm = &mut asm_local; let asm = &mut asm_local;
let mut iterator = self.into_draining_iter(); let mut iterator = self.into_draining_iter();
@ -405,11 +406,11 @@ impl Assembler
// We are replacing instructions here so we know they are already // We are replacing instructions here so we know they are already
// being used. It is okay not to use their output here. // being used. It is okay not to use their output here.
#[allow(unused_must_use)] #[allow(unused_must_use)]
match insn { match &mut insn {
Insn::Add { left, right, .. } => { Insn::Add { left, right, .. } => {
match (left, right) { match (*left, *right) {
(Opnd::Reg(_) | Opnd::InsnOut { .. }, Opnd::Reg(_) | Opnd::InsnOut { .. }) => { (Opnd::Reg(_) | Opnd::InsnOut { .. }, Opnd::Reg(_) | Opnd::InsnOut { .. }) => {
asm.add(left, right); asm.add(*left, *right);
}, },
(reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. }), other_opnd) | (reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. }), other_opnd) |
(other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. })) => { (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::InsnOut { .. })) => {
@ -417,23 +418,19 @@ impl Assembler
asm.add(reg_opnd, opnd1); asm.add(reg_opnd, opnd1);
}, },
_ => { _ => {
let opnd0 = split_load_operand(asm, left); let opnd0 = split_load_operand(asm, *left);
let opnd1 = split_shifted_immediate(asm, right); let opnd1 = split_shifted_immediate(asm, *right);
asm.add(opnd0, opnd1); asm.add(opnd0, opnd1);
} }
} }
}, },
Insn::And { left, right, .. } => { Insn::And { left, right, .. } |
let (opnd0, opnd1) = split_boolean_operands(asm, left, right); Insn::Or { left, right, .. } |
asm.and(opnd0, opnd1);
},
Insn::Or { left, right, .. } => {
let (opnd0, opnd1) = split_boolean_operands(asm, left, right);
asm.or(opnd0, opnd1);
},
Insn::Xor { left, right, .. } => { Insn::Xor { left, right, .. } => {
let (opnd0, opnd1) = split_boolean_operands(asm, left, right); let (opnd0, opnd1) = split_boolean_operands(asm, *left, *right);
asm.xor(opnd0, opnd1); *left = opnd0;
*right = opnd1;
asm.push_insn(insn);
}, },
Insn::CCall { opnds, fptr, .. } => { Insn::CCall { opnds, fptr, .. } => {
assert!(opnds.len() <= C_ARG_OPNDS.len()); assert!(opnds.len() <= C_ARG_OPNDS.len());
@ -448,8 +445,8 @@ impl Assembler
// a UImm of 0 along as the argument to the move. // a UImm of 0 along as the argument to the move.
let value = match opnd { let value = match opnd {
Opnd::UImm(0) | Opnd::Imm(0) => Opnd::UImm(0), Opnd::UImm(0) | Opnd::Imm(0) => Opnd::UImm(0),
Opnd::Mem(_) => split_memory_address(asm, opnd), Opnd::Mem(_) => split_memory_address(asm, *opnd),
_ => opnd _ => *opnd
}; };
asm.load_into(C_ARG_OPNDS[idx], value); asm.load_into(C_ARG_OPNDS[idx], value);
@ -457,12 +454,12 @@ impl Assembler
// Now we push the CCall without any arguments so that it // Now we push the CCall without any arguments so that it
// just performs the call. // just performs the call.
asm.ccall(fptr, vec![]); asm.ccall(*fptr, vec![]);
}, },
Insn::Cmp { left, right } => { Insn::Cmp { left, right } => {
let opnd0 = split_load_operand(asm, left); let opnd0 = split_load_operand(asm, *left);
let opnd0 = split_less_than_32_cmp(asm, opnd0); let opnd0 = split_less_than_32_cmp(asm, opnd0);
let split_right = split_shifted_immediate(asm, right); let split_right = split_shifted_immediate(asm, *right);
let opnd1 = match split_right { let opnd1 = match split_right {
Opnd::InsnOut { .. } if opnd0.num_bits() != split_right.num_bits() => { Opnd::InsnOut { .. } if opnd0.num_bits() != split_right.num_bits() => {
split_right.with_num_bits(opnd0.num_bits().unwrap()).unwrap() split_right.with_num_bits(opnd0.num_bits().unwrap()).unwrap()
@ -482,81 +479,66 @@ impl Assembler
// make sure the displacement isn't too large and then // make sure the displacement isn't too large and then
// load it into the return register. // load it into the return register.
Opnd::Mem(_) => { Opnd::Mem(_) => {
let split = split_memory_address(asm, opnd); let split = split_memory_address(asm, *opnd);
asm.load_into(C_RET_OPND, split); asm.load_into(C_RET_OPND, split);
}, },
// Otherwise we just need to load the value into the // Otherwise we just need to load the value into the
// return register. // return register.
_ => { _ => {
asm.load_into(C_RET_OPND, opnd); asm.load_into(C_RET_OPND, *opnd);
} }
} }
asm.cret(C_RET_OPND); asm.cret(C_RET_OPND);
}, },
Insn::CSelZ { truthy, falsy, .. } => { Insn::CSelZ { truthy, falsy, out } |
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); Insn::CSelNZ { truthy, falsy, out } |
asm.csel_z(opnd0, opnd1); Insn::CSelE { truthy, falsy, out } |
}, Insn::CSelNE { truthy, falsy, out } |
Insn::CSelNZ { truthy, falsy, .. } => { Insn::CSelL { truthy, falsy, out } |
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); Insn::CSelLE { truthy, falsy, out } |
asm.csel_nz(opnd0, opnd1); Insn::CSelG { truthy, falsy, out } |
}, Insn::CSelGE { truthy, falsy, out } => {
Insn::CSelE { truthy, falsy, .. } => { let (opnd0, opnd1) = split_csel_operands(asm, *truthy, *falsy);
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); *truthy = opnd0;
asm.csel_e(opnd0, opnd1); *falsy = opnd1;
}, // Merge `csel` and `mov` into a single `csel` when possible
Insn::CSelNE { truthy, falsy, .. } => { match iterator.peek() {
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); Some(Insn::Mov { dest: Opnd::Reg(reg), src })
asm.csel_ne(opnd0, opnd1); if matches!(out, Opnd::InsnOut { .. }) && *out == *src && live_ranges[index] == index + 1 => {
}, *out = Opnd::Reg(*reg);
Insn::CSelL { truthy, falsy, .. } => { asm.push_insn(insn);
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); iterator.map_insn_index(asm);
asm.csel_l(opnd0, opnd1); iterator.next_unmapped(); // Pop merged Insn::Mov
}, }
Insn::CSelLE { truthy, falsy, .. } => { _ => {
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy); asm.push_insn(insn);
asm.csel_le(opnd0, opnd1); }
}, }
Insn::CSelG { truthy, falsy, .. } => {
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy);
asm.csel_g(opnd0, opnd1);
},
Insn::CSelGE { truthy, falsy, .. } => {
let (opnd0, opnd1) = split_csel_operands(asm, truthy, falsy);
asm.csel_ge(opnd0, opnd1);
}, },
Insn::IncrCounter { mem, value } => { Insn::IncrCounter { mem, value } => {
let counter_addr = match mem { let counter_addr = match mem {
Opnd::Mem(_) => split_lea_operand(asm, mem), Opnd::Mem(_) => split_lea_operand(asm, *mem),
_ => mem _ => *mem
}; };
asm.incr_counter(counter_addr, value); asm.incr_counter(counter_addr, *value);
}, },
Insn::JmpOpnd(opnd) => { Insn::JmpOpnd(opnd) => {
if let Opnd::Mem(_) = opnd { if let Opnd::Mem(_) = opnd {
let opnd0 = split_load_operand(asm, opnd); let opnd0 = split_load_operand(asm, *opnd);
asm.jmp_opnd(opnd0); asm.jmp_opnd(opnd0);
} else { } else {
asm.jmp_opnd(opnd); asm.jmp_opnd(*opnd);
} }
}, },
Insn::Load { opnd, .. } => { Insn::Load { opnd, .. } |
let value = match opnd { Insn::LoadInto { opnd, .. } => {
Opnd::Mem(_) => split_memory_address(asm, opnd), *opnd = match opnd {
_ => opnd Opnd::Mem(_) => split_memory_address(asm, *opnd),
_ => *opnd
}; };
asm.push_insn(insn);
asm.load(value);
},
Insn::LoadInto { dest, opnd } => {
let value = match opnd {
Opnd::Mem(_) => split_memory_address(asm, opnd),
_ => opnd
};
asm.load_into(dest, value);
}, },
Insn::LoadSExt { opnd, .. } => { Insn::LoadSExt { opnd, .. } => {
match opnd { match opnd {
@ -567,50 +549,50 @@ impl Assembler
Opnd::Reg(Reg { num_bits: 32, .. }) | Opnd::Reg(Reg { num_bits: 32, .. }) |
Opnd::InsnOut { num_bits: 32, .. } | Opnd::InsnOut { num_bits: 32, .. } |
Opnd::Mem(Mem { num_bits: 32, .. }) => { Opnd::Mem(Mem { num_bits: 32, .. }) => {
asm.load_sext(opnd); asm.load_sext(*opnd);
}, },
_ => { _ => {
asm.load(opnd); asm.load(*opnd);
} }
}; };
}, },
Insn::Mov { dest, src } => { Insn::Mov { dest, src } => {
match (dest, src) { match (&dest, &src) {
// If we're attempting to load into a memory operand, then // If we're attempting to load into a memory operand, then
// we'll switch over to the store instruction. // we'll switch over to the store instruction.
(Opnd::Mem(_), _) => { (Opnd::Mem(_), _) => {
let value = match src { let value = match *src {
// If the first operand is zero, then we can just use // If the first operand is zero, then we can just use
// the zero register. // the zero register.
Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG), Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG),
// If the first operand is a memory operand, we're going // If the first operand is a memory operand, we're going
// to transform this into a store instruction, so we'll // to transform this into a store instruction, so we'll
// need to load this anyway. // need to load this anyway.
Opnd::UImm(_) => asm.load(src), Opnd::UImm(_) => asm.load(*src),
// The value that is being moved must be either a // The value that is being moved must be either a
// register or an immediate that can be encoded as a // register or an immediate that can be encoded as a
// bitmask immediate. Otherwise, we'll need to split the // bitmask immediate. Otherwise, we'll need to split the
// move into multiple instructions. // move into multiple instructions.
_ => split_bitmask_immediate(asm, src, dest.rm_num_bits()) _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits())
}; };
let opnd0 = split_memory_address(asm, dest); let opnd0 = split_memory_address(asm, *dest);
asm.store(opnd0, value); asm.store(opnd0, value);
}, },
// If we're loading a memory operand into a register, then // If we're loading a memory operand into a register, then
// we'll switch over to the load instruction. // we'll switch over to the load instruction.
(Opnd::Reg(_), Opnd::Mem(_)) => { (Opnd::Reg(_), Opnd::Mem(_)) => {
let value = split_memory_address(asm, src); let value = split_memory_address(asm, *src);
asm.load_into(dest, value); asm.load_into(*dest, value);
}, },
// Otherwise we'll use the normal mov instruction. // Otherwise we'll use the normal mov instruction.
(Opnd::Reg(_), _) => { (Opnd::Reg(_), _) => {
let value = match src { let value = match *src {
// Unlike other instructions, we can avoid splitting this case, using movz. // Unlike other instructions, we can avoid splitting this case, using movz.
Opnd::UImm(uimm) if uimm <= 0xffff => src, Opnd::UImm(uimm) if uimm <= 0xffff => *src,
_ => split_bitmask_immediate(asm, src, dest.rm_num_bits()), _ => split_bitmask_immediate(asm, *src, dest.rm_num_bits()),
}; };
asm.mov(dest, value); asm.mov(*dest, value);
}, },
_ => unreachable!() _ => unreachable!()
}; };
@ -619,8 +601,8 @@ impl Assembler
// The value that is being negated must be in a register, so // The value that is being negated must be in a register, so
// if we get anything else we need to load it first. // if we get anything else we need to load it first.
let opnd0 = match opnd { let opnd0 = match opnd {
Opnd::Mem(_) => split_load_operand(asm, opnd), Opnd::Mem(_) => split_load_operand(asm, *opnd),
_ => opnd _ => *opnd
}; };
asm.not(opnd0); asm.not(opnd0);
@ -633,38 +615,38 @@ impl Assembler
// the zero register. // the zero register.
Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG), Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG),
// Otherwise we'll check if we need to load it first. // Otherwise we'll check if we need to load it first.
_ => split_load_operand(asm, src) _ => split_load_operand(asm, *src)
}; };
match dest { match dest {
Opnd::Reg(_) => { Opnd::Reg(_) => {
// Store does not support a register as a dest operand. // Store does not support a register as a dest operand.
asm.mov(dest, opnd1); asm.mov(*dest, opnd1);
} }
_ => { _ => {
// The displacement for the STUR instruction can't be more // The displacement for the STUR instruction can't be more
// than 9 bits long. If it's longer, we need to load the // than 9 bits long. If it's longer, we need to load the
// memory address into a register first. // memory address into a register first.
let opnd0 = split_memory_address(asm, dest); let opnd0 = split_memory_address(asm, *dest);
asm.store(opnd0, opnd1); asm.store(opnd0, opnd1);
} }
} }
}, },
Insn::Sub { left, right, .. } => { Insn::Sub { left, right, .. } => {
let opnd0 = split_load_operand(asm, left); let opnd0 = split_load_operand(asm, *left);
let opnd1 = split_shifted_immediate(asm, right); let opnd1 = split_shifted_immediate(asm, *right);
asm.sub(opnd0, opnd1); asm.sub(opnd0, opnd1);
}, },
Insn::Test { left, right } => { Insn::Test { left, right } => {
// The value being tested must be in a register, so if it's // The value being tested must be in a register, so if it's
// not already one we'll load it first. // not already one we'll load it first.
let opnd0 = split_load_operand(asm, left); let opnd0 = split_load_operand(asm, *left);
// The second value must be either a register or an // The second value must be either a register or an
// unsigned immediate that can be encoded as a bitmask // unsigned immediate that can be encoded as a bitmask
// immediate. If it's not one of those, we'll need to load // immediate. If it's not one of those, we'll need to load
// it first. // it first.
let opnd1 = split_bitmask_immediate(asm, right, opnd0.rm_num_bits()); let opnd1 = split_bitmask_immediate(asm, *right, opnd0.rm_num_bits());
asm.test(opnd0, opnd1); asm.test(opnd0, opnd1);
}, },
_ => { _ => {
@ -1617,4 +1599,19 @@ mod tests {
0x4: orr x1, xzr, #0x10000 0x4: orr x1, xzr, #0x10000
"}); "});
} }
#[test]
fn test_merge_csel_mov() {
let (mut asm, mut cb) = setup_asm();
let out = asm.csel_l(Qtrue.into(), Qfalse.into());
asm.mov(Opnd::Reg(Assembler::TEMP_REGS[0]), out);
asm.compile_with_num_regs(&mut cb, 2);
assert_disasm!(cb, "8b0280d20c0080d261b18c9a", {"
0x0: mov x11, #0x14
0x4: mov x12, #0
0x8: csel x1, x11, x12, lt
"});
}
} }