From 9ad8dd00b1be782b7a053315bc4c00ff642c5fba Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 20 Jan 2026 20:34:49 +0100 Subject: [PATCH 1/6] [ruby/prism] Fix `on_*` return value of ripper translator You're supposed to return the first argument. ```rb # Before [[:stmts_new], [:rescue_mod, nil, nil], [:stmts_add, nil, nil], [:program, nil]] # After [[:stmts_new], [:rescue_mod, "1", "2"], [:stmts_add, nil, "1"], [:program, nil]] ``` The correct result would be: `[[:rescue_mod, "1", "2"], [:stmts_new], [:stmts_add, nil, "1"], [:program, nil]]` But the order depends on the prism AST so it seems very difficult to match. https://github.com/ruby/prism/commit/94e0107729 --- lib/prism/translation/ripper.rb | 12 ++++++------ test/prism/ruby/ripper_test.rb | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index c8f9fa7731a539..afe30bb5db4131 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -3446,12 +3446,12 @@ def bounds(location) # :stopdoc: def _dispatch_0; end - def _dispatch_1(_); end - def _dispatch_2(_, _); end - def _dispatch_3(_, _, _); end - def _dispatch_4(_, _, _, _); end - def _dispatch_5(_, _, _, _, _); end - def _dispatch_7(_, _, _, _, _, _, _); end + def _dispatch_1(arg); arg end + def _dispatch_2(arg, _); arg end + def _dispatch_3(arg, _, _); arg end + def _dispatch_4(arg, _, _, _); arg end + def _dispatch_5(arg, _, _, _, _); arg end + def _dispatch_7(arg, _, _, _, _, _, _); arg end # :startdoc: # diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 280abd94ea3e64..174c90cbcabcc7 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -86,6 +86,40 @@ class RipperTest < TestCase define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) } end + module Events + attr_reader :events + + def initialize(...) + super + @events = [] + end + + Prism::Translation::Ripper::PARSER_EVENTS.each do |event| + define_method(:"on_#{event}") do |*args| + @events << [event, *args] + super(*args) + end + end + end + + class RipperEvents < Ripper + include Events + end + + class PrismEvents < Translation::Ripper + include Events + end + + def test_events + source = "1 rescue 2" + ripper = RipperEvents.new(source) + prism = PrismEvents.new(source) + ripper.parse + prism.parse + # This makes sure that the content is the same. Ordering is not correct for now. + assert_equal(ripper.events.sort, prism.events.sort) + end + def test_lexer lexer = Translation::Ripper::Lexer.new("foo") expected = [[1, 0], :on_ident, "foo", Translation::Ripper::EXPR_CMDARG] From 1bc51114114aae12f533e60e02a0dc606fb5d793 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Tue, 20 Jan 2026 15:35:50 -0500 Subject: [PATCH 2/6] ZJIT: Add a smoke test for --zjit-trace-exits Better than nothing! --- test/ruby/test_zjit.rb | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index 2cfe9dd7e3a530..1cc43be971a49b 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -4471,6 +4471,28 @@ def test(x) } end + def test_exit_tracing + # This is a very basic smoke test. The StackProf format + # this option generates is external to us. + Dir.mktmpdir("zjit_test_exit_tracing") do |tmp_dir| + assert_compiles('true', <<~RUBY, extra_args: ['-C', tmp_dir, '--zjit-trace-exits']) + def test(object) = object.itself + + # induce an exit just for good measure + array = [] + test(array) + test(array) + def array.itself = :not_itself + test(array) + + RubyVM::ZJIT.exit_locations.is_a?(Hash) + RUBY + dump_files = Dir.glob('zjit_exits_*.dump', base: tmp_dir) + assert_equal(1, dump_files.length) + refute(File.empty?(File.join(tmp_dir, dump_files.first))) + end + end + private # Assert that every method call in `test_script` can be compiled by ZJIT @@ -4531,10 +4553,11 @@ def eval_with_jit( stats: false, debug: true, allowed_iseqs: nil, + extra_args: nil, timeout: 1000, pipe_fd: nil ) - args = ["--disable-gems"] + args = ["--disable-gems", *extra_args] if zjit args << "--zjit-call-threshold=#{call_threshold}" args << "--zjit-num-profiles=#{num_profiles}" From 631a5076dabcb846157a4410bcccf202fce76127 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Thu, 8 Jan 2026 23:12:24 -0500 Subject: [PATCH 3/6] ZJIT: Delete Insn::CPushAll and Insn::CPopAll Since we automatically preserve registers across calls, it's never necessary to manually and imprecisely do it with `C{Push,Pop}All`. Delete them to remove the maintenance burden and reduce confusion. --- zjit/src/backend/arm64/mod.rs | 66 ---------------------------------- zjit/src/backend/lir.rs | 23 ------------ zjit/src/backend/x86_64/mod.rs | 19 ---------- 3 files changed, 108 deletions(-) diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index ea5d86659f513a..a019e2037d2e55 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1426,28 +1426,6 @@ impl Assembler { Insn::CPopInto(opnd) => { emit_pop(cb, opnd.into()); }, - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - emit_push(cb, A64Opnd::Reg(reg)); - } - - // Push the flags/state register - mrs(cb, Self::EMIT_OPND, SystemRegister::NZCV); - emit_push(cb, Self::EMIT_OPND); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - // Pop the state/flags register - msr(cb, SystemRegister::NZCV, Self::EMIT_OPND); - emit_pop(cb, Self::EMIT_OPND); - - for reg in regs.into_iter().rev() { - emit_pop(cb, A64Opnd::Reg(reg)); - } - }, Insn::CCall { fptr, .. } => { match fptr { Opnd::UImm(fptr) => { @@ -1881,50 +1859,6 @@ mod tests { assert_snapshot!(cb.hexdump(), @"48656c6c6f2c20776f726c6421000000"); } - #[test] - fn test_emit_cpush_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpush_all(); - asm.compile_with_num_regs(&mut cb, 0); - - assert_disasm_snapshot!(cb.disasm(), @r" - 0x0: str x1, [sp, #-0x10]! - 0x4: str x9, [sp, #-0x10]! - 0x8: str x10, [sp, #-0x10]! - 0xc: str x11, [sp, #-0x10]! - 0x10: str x12, [sp, #-0x10]! - 0x14: str x13, [sp, #-0x10]! - 0x18: str x14, [sp, #-0x10]! - 0x1c: str x15, [sp, #-0x10]! - 0x20: mrs x16, nzcv - 0x24: str x16, [sp, #-0x10]! - "); - assert_snapshot!(cb.hexdump(), @"e10f1ff8e90f1ff8ea0f1ff8eb0f1ff8ec0f1ff8ed0f1ff8ee0f1ff8ef0f1ff810423bd5f00f1ff8"); - } - - #[test] - fn test_emit_cpop_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpop_all(); - asm.compile_with_num_regs(&mut cb, 0); - - assert_disasm_snapshot!(cb.disasm(), @r" - 0x0: msr nzcv, x16 - 0x4: ldr x16, [sp], #0x10 - 0x8: ldr x15, [sp], #0x10 - 0xc: ldr x14, [sp], #0x10 - 0x10: ldr x13, [sp], #0x10 - 0x14: ldr x12, [sp], #0x10 - 0x18: ldr x11, [sp], #0x10 - 0x1c: ldr x10, [sp], #0x10 - 0x20: ldr x9, [sp], #0x10 - 0x24: ldr x1, [sp], #0x10 - "); - assert_snapshot!(cb.hexdump(), @"10421bd5f00741f8ef0741f8ee0741f8ed0741f8ec0741f8eb0741f8ea0741f8e90741f8e10741f8"); - } - #[test] fn test_emit_frame() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index d8d82a09cabb00..69ddbf24717dd6 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -377,18 +377,12 @@ pub enum Insn { /// Pop a register from the C stack CPop { out: Opnd }, - /// Pop all of the caller-save registers and the flags from the C stack - CPopAll, - /// Pop a register from the C stack and store it into another register CPopInto(Opnd), /// Push a register onto the C stack CPush(Opnd), - /// Push all of the caller-save registers and the flags to the C stack - CPushAll, - // C function call with N arguments (variadic) CCall { opnds: Vec, @@ -614,10 +608,8 @@ impl Insn { Insn::Comment(_) => "Comment", Insn::Cmp { .. } => "Cmp", Insn::CPop { .. } => "CPop", - Insn::CPopAll => "CPopAll", Insn::CPopInto(_) => "CPopInto", Insn::CPush(_) => "CPush", - Insn::CPushAll => "CPushAll", Insn::CCall { .. } => "CCall", Insn::CRet(_) => "CRet", Insn::CSelE { .. } => "CSelE", @@ -851,8 +843,6 @@ impl<'a> Iterator for InsnOpndIterator<'a> { Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | Insn::PadPatchPoint | Insn::PosMarker(_) => None, @@ -1020,8 +1010,6 @@ impl<'a> InsnOpndMutIterator<'a> { Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | Insn::FrameSetup { .. } | Insn::FrameTeardown { .. } | Insn::PadPatchPoint | @@ -1867,10 +1855,7 @@ impl Assembler } if should_record_exit { - // Preserve caller-saved registers that may be used in the shared exit. - self.cpush_all(); asm_ccall!(self, rb_zjit_record_exit_stack, pc); - self.cpop_all(); } // If the side exit has already been compiled, jump to it. @@ -2135,10 +2120,6 @@ impl Assembler { out } - pub fn cpop_all(&mut self) { - self.push_insn(Insn::CPopAll); - } - pub fn cpop_into(&mut self, opnd: Opnd) { assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); self.push_insn(Insn::CPopInto(opnd)); @@ -2148,10 +2129,6 @@ impl Assembler { self.push_insn(Insn::CPush(opnd)); } - pub fn cpush_all(&mut self) { - self.push_insn(Insn::CPushAll); - } - pub fn cret(&mut self, opnd: Opnd) { self.push_insn(Insn::CRet(opnd)); } diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index b493a99929b7e3..c1b9b2da13d366 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -858,25 +858,6 @@ impl Assembler { pop(cb, opnd.into()); }, - // Push and pop to the C stack all caller-save registers and the - // flags - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - push(cb, X86Opnd::Reg(reg)); - } - pushfq(cb); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - popfq(cb); - for reg in regs.into_iter().rev() { - pop(cb, X86Opnd::Reg(reg)); - } - }, - // C function call Insn::CCall { fptr, .. } => { match fptr { From e24b52885feaa87cdb5796c2a08e5995274e83cb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 19 Jan 2026 19:26:47 -0500 Subject: [PATCH 4/6] Allow objects on Ruby stack to be GC movable Objects on the Ruby stack can be GC movable and there is corresponding code in rb_execution_context_update to update references for moved objects. --- vm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm.c b/vm.c index 7c86c0ff54d1b6..2cae6779d9cbc1 100644 --- a/vm.c +++ b/vm.c @@ -3682,8 +3682,9 @@ rb_execution_context_mark(const rb_execution_context_t *ec) rb_control_frame_t *cfp = ec->cfp; rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size); - VM_ASSERT(sp == ec->cfp->sp); - rb_gc_mark_vm_stack_values((long)(sp - p), p); + for (long i = 0; i < (long)(sp - p); i++) { + rb_gc_mark_movable(p[i]); + } while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; From f7e73ba3bf9ced61ac9cca5cf042fd0efe398f69 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 20 Jan 2026 18:31:26 -0500 Subject: [PATCH 5/6] ZJIT: A64: Avoid gaps in the stack when preserving registers for calls Previously, we used a `str x, [sp, #-0x10]!` for each value, which left an 8-byte gap. Use STP to store a pair at a time instead. --- zjit/src/backend/arm64/mod.rs | 110 ++++++++++++++++++++++++++++++ zjit/src/backend/lir.rs | 54 +++++++++++++-- zjit/src/backend/x86_64/mod.rs | 121 +++++++++++++++++++++++++++++++++ zjit/src/codegen.rs | 26 +++++-- 4 files changed, 301 insertions(+), 10 deletions(-) diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index a019e2037d2e55..574249dabda81c 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1420,12 +1420,20 @@ impl Assembler { Insn::CPush(opnd) => { emit_push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + // Second operand ends up at the lower stack address + stp_pre(cb, opnd1.into(), opnd0.into(), A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + }, Insn::CPop { out } => { emit_pop(cb, out.into()); }, Insn::CPopInto(opnd) => { emit_pop(cb, opnd.into()); }, + Insn::CPopPairInto(opnd0, opnd1) => { + // First operand is popped from the lower stack address + ldp_post(cb, opnd0.into(), opnd1.into(), A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); + }, Insn::CCall { fptr, .. } => { match fptr { Opnd::UImm(fptr) => { @@ -2662,6 +2670,76 @@ mod tests { assert_snapshot!(cb.hexdump(), @"ef0302aae20303aae3030faaef0300aae00301aae1030faa100080d200023fd6"); } + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, x0 + 0x14: stp x2, x1, [sp, #-0x10]! + 0x18: stp x4, x3, [sp, #-0x10]! + 0x1c: mov x16, #0 + 0x20: blr x16 + 0x24: ldp x4, x3, [sp], #0x10 + 0x28: ldp x2, x1, [sp], #0x10 + 0x2c: adds x4, x4, x1 + 0x30: adds x2, x2, x3 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2e40300aae207bfa9e40fbfa9100080d200023fd6e40fc1a8e207c1a8840001ab420003ab"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, #5 + 0x14: mov x5, x0 + 0x18: stp x2, x1, [sp, #-0x10]! + 0x1c: stp x4, x3, [sp, #-0x10]! + 0x20: str x5, [sp, #-0x10]! + 0x24: mov x16, #0 + 0x28: blr x16 + 0x2c: ldr x5, [sp], #0x10 + 0x30: ldp x4, x3, [sp], #0x10 + 0x34: ldp x2, x1, [sp], #0x10 + 0x38: adds x5, x5, x1 + 0x3c: adds x0, x2, x3 + 0x40: adds x2, x2, x4 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2a40080d2e50300aae207bfa9e40fbfa9e50f1ff8100080d200023fd6e50741f8e40fc1a8e207c1a8a50001ab400003ab420004ab"); + } + #[test] fn test_ccall_resolve_parallel_moves_large_cycle() { let (mut asm, mut cb) = setup_asm(); @@ -2685,6 +2763,38 @@ mod tests { assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae10302aae2030faa100080d200023fd6"); } + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: stp x1, x0, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e103bfa9"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: ldp x0, x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e007c1a8"); + } + #[test] fn test_split_spilled_lshift() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 69ddbf24717dd6..bb39d85cc8ff5c 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -380,9 +380,17 @@ pub enum Insn { /// Pop a register from the C stack and store it into another register CPopInto(Opnd), + /// Pop a pair of registers from the C stack and store it into a pair of registers. + /// The registers are popped from left to right. + CPopPairInto(Opnd, Opnd), + /// Push a register onto the C stack CPush(Opnd), + /// Push a pair of registers onto the C stack. + /// The registers are pushed from left to right. + CPushPair(Opnd, Opnd), + // C function call with N arguments (variadic) CCall { opnds: Vec, @@ -609,7 +617,9 @@ impl Insn { Insn::Cmp { .. } => "Cmp", Insn::CPop { .. } => "CPop", Insn::CPopInto(_) => "CPopInto", + Insn::CPopPairInto(_, _) => "CPopPairInto", Insn::CPush(_) => "CPush", + Insn::CPushPair(_, _) => "CPushPair", Insn::CCall { .. } => "CCall", Insn::CRet(_) => "CRet", Insn::CSelE { .. } => "CSelE", @@ -865,6 +875,8 @@ impl<'a> Iterator for InsnOpndIterator<'a> { }, Insn::Add { left: opnd0, right: opnd1, .. } | Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | Insn::Cmp { left: opnd0, right: opnd1 } | Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | @@ -1034,6 +1046,8 @@ impl<'a> InsnOpndMutIterator<'a> { }, Insn::Add { left: opnd0, right: opnd1, .. } | Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | Insn::Cmp { left: opnd0, right: opnd1 } | Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | @@ -1592,9 +1606,19 @@ impl Assembler saved_regs = pool.live_regs(); // Save live registers - for &(reg, _) in saved_regs.iter() { - asm.cpush(Opnd::Reg(reg)); - pool.dealloc_opnd(&Opnd::Reg(reg)); + for pair in saved_regs.chunks(2) { + match *pair { + [(reg0, _), (reg1, _)] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + pool.dealloc_opnd(&Opnd::Reg(reg0)); + pool.dealloc_opnd(&Opnd::Reg(reg1)); + } + [(reg, _)] => { + asm.cpush(Opnd::Reg(reg)); + pool.dealloc_opnd(&Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } } // On x86_64, maintain 16-byte stack alignment if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { @@ -1725,9 +1749,19 @@ impl Assembler asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0)); } // Restore saved registers - for &(reg, vreg_idx) in saved_regs.iter().rev() { - asm.cpop_into(Opnd::Reg(reg)); - pool.take_reg(®, vreg_idx); + for pair in saved_regs.chunks(2).rev() { + match *pair { + [(reg, vreg_idx)] => { + asm.cpop_into(Opnd::Reg(reg)); + pool.take_reg(®, vreg_idx); + } + [(reg0, vreg_idx0), (reg1, vreg_idx1)] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + pool.take_reg(®1, vreg_idx1); + pool.take_reg(®0, vreg_idx0); + } + _ => unreachable!("chunks(2)") + } } saved_regs.clear(); } @@ -2125,10 +2159,18 @@ impl Assembler { self.push_insn(Insn::CPopInto(opnd)); } + pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); + } + pub fn cpush(&mut self, opnd: Opnd) { self.push_insn(Insn::CPush(opnd)); } + pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + self.push_insn(Insn::CPushPair(opnd0, opnd1)); + } + pub fn cret(&mut self, opnd: Opnd) { self.push_insn(Insn::CRet(opnd)); } diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index c1b9b2da13d366..38b9f2791b44f1 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -851,12 +851,20 @@ impl Assembler { Insn::CPush(opnd) => { push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + push(cb, opnd0.into()); + push(cb, opnd1.into()); + }, Insn::CPop { out } => { pop(cb, out.into()); }, Insn::CPopInto(opnd) => { pop(cb, opnd.into()); }, + Insn::CPopPairInto(opnd0, opnd1) => { + pop(cb, opnd0.into()); + pop(cb, opnd1.into()); + }, // C function call Insn::CCall { fptr, .. } => { @@ -1648,6 +1656,119 @@ mod tests { assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0"); } + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: push rdi + 0x15: push rsi + 0x16: push rdx + 0x17: push rcx + 0x18: mov eax, 0 + 0x1d: call rax + 0x1f: pop rcx + 0x20: pop rdx + 0x21: pop rsi + 0x22: pop rdi + 0x23: add rdi, rsi + 0x26: add rdx, rcx + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74801ca"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: mov r8d, 5 + 0x1a: push rdi + 0x1b: push rsi + 0x1c: push rdx + 0x1d: push rcx + 0x1e: push r8 + 0x20: push r8 + 0x22: mov eax, 0 + 0x27: call rax + 0x29: pop r8 + 0x2b: pop r8 + 0x2d: pop rcx + 0x2e: pop rdx + 0x2f: pop rsi + 0x30: pop rdi + 0x31: add rdi, rsi + 0x34: mov rdi, rdx + 0x37: add rdi, rcx + 0x3a: add rdx, r8 + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141504150b800000000ffd041584158595a5e5f4801f74889d74801cf4c01c2"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: push rdi + 0xb: push rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: pop rdi + 0xb: pop rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e"); + } + #[test] fn test_cmov_mem() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 0ae85c24a2f1bf..4dae41bf02388f 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2638,8 +2638,17 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + } + [reg] => { + asm.cpush(Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } } if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 @@ -2653,8 +2662,17 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result { + asm.cpop_into(Opnd::Reg(reg)); + } + [reg0, reg1] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + } + _ => unreachable!("chunks(2)") + } } // Discard the current frame since the JIT function will set it up again From 36809a8d0c7ab67ff0919b331db926529a3e98a9 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Tue, 20 Jan 2026 19:06:54 -0500 Subject: [PATCH 6/6] ZJIT: Add fail-fast assert for non-register {cpush,cpop}_pair There is no splitting for these so let's add a assert to try and catch misuse. VRegs are not necessarily registers in the end, so this is best effort. In those situations they'll get a less proximate panic message. --- zjit/src/backend/lir.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index bb39d85cc8ff5c..06127b5c1a40e4 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -2159,7 +2159,10 @@ impl Assembler { self.push_insn(Insn::CPopInto(opnd)); } + #[track_caller] pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd1:?}"); self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); } @@ -2167,7 +2170,10 @@ impl Assembler { self.push_insn(Insn::CPush(opnd)); } + #[track_caller] pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd1:?}"); self.push_insn(Insn::CPushPair(opnd0, opnd1)); }