Skip to content

Commit f092edb

Browse files
sulixsulix
committed
x86: Correctly pass large integers and structs in registers with -Zregparm
The -Zregparm option for 32-bit x86 modifies the calling convention to pass a number of arguments in registers instead of on the stack. (This is primarily used by the Linux kernel.) Currently, rustc will only pass integers of size <= 32 bits in registers, but gcc (and clang) will pass larger integers (e.g., 64-bit integers) across two registers if possible. This is not the case with fastcall/vectorcall. Equally, struct arguments will be split into registers and passed where possible. Supporting this in rustc required moving the 'inreg' determination into compute_abi_info, as this is the stage where we determine if structs are passed directly or indirectly. This involves updating the x86_win32 ABI implementation as well, as it previously shared the inreg handling, but has a different compute_abi_info implementation. Also add some assembly-llvm and codegen-llvm tests. Note that the LLVM IR generated doesn't exactly match clang, as rustc treats struct arguments as one argument in three registers, whereas clang seems to decompose it into three registers. It seems to work regardless...
1 parent 0e72e32 commit f092edb

4 files changed

Lines changed: 180 additions & 77 deletions

File tree

‎compiler/rustc_target/src/callconv/x86.rs‎

Lines changed: 50 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use rustc_abi::{
22
AddressSpace, Align, BackendRepr, HasDataLayout, Primitive, Reg, RegKind, TyAndLayout,
33
};
44

5-
use crate::callconv::{ArgAttribute, FnAbi, PassMode, TyAbiInterface};
5+
use crate::callconv::{ArgAttribute, FnAbi, PassMode, TyAbiInterface, Uniform};
66
use crate::spec::{HasTargetSpec, RustcAbi};
77

88
#[derive(PartialEq)]
@@ -58,6 +58,13 @@ where
5858
}
5959
}
6060

61+
// Some calling conversions pass arguments in registers.
62+
// This is set with -Zregparm, or the fastcall/vectorcall (always 2 registers)
63+
let mut free_regs = opts.regparm.unwrap_or(0).into();
64+
if opts.flavor == Flavor::FastcallOrVectorcall {
65+
free_regs = 2;
66+
}
67+
6168
for arg in fn_abi.args.iter_mut() {
6269
if arg.is_ignore() || !arg.layout.is_sized() {
6370
continue;
@@ -72,7 +79,21 @@ where
7279
let align_4 = Align::from_bytes(4).unwrap();
7380
let align_16 = Align::from_bytes(16).unwrap();
7481

82+
let size_in_regs = arg.layout.size.bits().div_ceil(32);
83+
let mut pass_in_reg = free_regs >= size_in_regs;
84+
85+
// In fastcall/vectorcall only 32-bit integers can be passed in registers.
86+
if opts.flavor == Flavor::FastcallOrVectorcall && size_in_regs > 1 {
87+
pass_in_reg = false;
88+
// Once we've had an argument which doesn't fit, don't try to fit any more.
89+
free_regs = 0;
90+
}
91+
7592
if arg.layout.is_aggregate() {
93+
if opts.flavor == Flavor::FastcallOrVectorcall && size_in_regs > 1 {
94+
pass_in_reg = false;
95+
}
96+
7697
// We need to compute the alignment of the `byval` argument. The rules can be found in
7798
// `X86_32ABIInfo::getTypeStackAlignInBytes` in Clang's `TargetInfo.cpp`. Summarized
7899
// here, they are:
@@ -119,83 +140,40 @@ where
119140
align_4
120141
};
121142

122-
arg.pass_by_stack_offset(Some(byval_align));
143+
if pass_in_reg {
144+
arg.cast_to(Uniform::new(Reg::i32(), arg.layout.size));
145+
} else {
146+
arg.pass_by_stack_offset(Some(byval_align))
147+
}
123148
} else {
124-
arg.extend_integer_width_to(32);
125-
}
126-
}
127-
128-
fill_inregs(cx, fn_abi, opts, false);
129-
}
130-
131-
pub(crate) fn fill_inregs<'a, Ty, C>(
132-
cx: &C,
133-
fn_abi: &mut FnAbi<'a, Ty>,
134-
opts: X86Options,
135-
rust_abi: bool,
136-
) where
137-
Ty: TyAbiInterface<'a, C> + Copy,
138-
{
139-
if opts.flavor != Flavor::FastcallOrVectorcall && opts.regparm.is_none_or(|x| x == 0) {
140-
return;
141-
}
142-
// Mark arguments as InReg like clang does it,
143-
// so our fastcall/vectorcall is compatible with C/C++ fastcall/vectorcall.
144-
145-
// Clang reference: lib/CodeGen/TargetInfo.cpp
146-
// See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()
147-
148-
// IsSoftFloatABI is only set to true on ARM platforms,
149-
// which in turn can't be x86?
150-
151-
// 2 for fastcall/vectorcall, regparm limited by 3 otherwise
152-
let mut free_regs = opts.regparm.unwrap_or(2).into();
149+
let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap();
153150

154-
// For types generating PassMode::Cast, InRegs will not be set.
155-
// Maybe, this is a FIXME
156-
let has_casts = fn_abi.args.iter().any(|arg| matches!(arg.mode, PassMode::Cast { .. }));
157-
if has_casts && rust_abi {
158-
return;
159-
}
160-
161-
for arg in fn_abi.args.iter_mut() {
162-
let attrs = match arg.mode {
163-
PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
164-
continue;
151+
// Fastcall/regparm won't pass floats in registers
152+
// (though regparm _will_ pass f]oat-containing structs in registers)
153+
if unit.kind != RegKind::Integer {
154+
pass_in_reg = false;
165155
}
166-
PassMode::Direct(ref mut attrs) => attrs,
167-
PassMode::Pair(..)
168-
| PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }
169-
| PassMode::Cast { .. } => {
170-
unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
171-
}
172-
};
173156

174-
// At this point we know this must be a primitive of sorts.
175-
let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap();
176-
assert_eq!(unit.size, arg.layout.size);
177-
if matches!(unit.kind, RegKind::Float | RegKind::Vector { .. }) {
178-
continue;
179-
}
180-
181-
let size_in_regs = arg.layout.size.bits().div_ceil(32);
182-
183-
if size_in_regs == 0 {
184-
continue;
185-
}
186-
187-
if size_in_regs > free_regs {
188-
break;
189-
}
190-
191-
free_regs -= size_in_regs;
192-
193-
if arg.layout.size.bits() <= 32 && unit.kind == RegKind::Integer {
194-
attrs.set(ArgAttribute::InReg);
157+
arg.extend_integer_width_to(32);
195158
}
196159

197-
if free_regs == 0 {
198-
break;
160+
// Set the InReg annotation if we're passing by register
161+
if pass_in_reg {
162+
match arg.mode {
163+
PassMode::Ignore
164+
| PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {}
165+
PassMode::Cast { pad_i32: _, ref mut cast } => {
166+
cast.attrs.set(ArgAttribute::InReg);
167+
}
168+
PassMode::Direct(ref mut attrs) => {
169+
attrs.set(ArgAttribute::InReg);
170+
}
171+
PassMode::Pair(..)
172+
| PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
173+
unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
174+
}
175+
};
176+
free_regs -= size_in_regs;
199177
}
200178
}
201179
}

‎compiler/rustc_target/src/callconv/x86_win32.rs‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
use rustc_abi::{Align, HasDataLayout, Reg, TyAbiInterface};
1+
use rustc_abi::{Align, HasDataLayout, Reg, RegKind};
22

3-
use crate::callconv::FnAbi;
3+
use crate::callconv::x86::Flavor;
4+
use crate::callconv::{ArgAttribute, FnAbi, PassMode, TyAbiInterface, Uniform};
45
use crate::spec::HasTargetSpec;
56

67
pub(crate) fn compute_abi_info<'a, Ty, C>(
@@ -41,6 +42,13 @@ pub(crate) fn compute_abi_info<'a, Ty, C>(
4142
}
4243
}
4344

45+
// Some calling conversions pass arguments in registers.
46+
// This is set with -Zregparm, or the fastcall/vectorcall (always 2 registers)
47+
let mut free_regs = opts.regparm.unwrap_or(0).into();
48+
if opts.flavor == Flavor::FastcallOrVectorcall {
49+
free_regs = 2;
50+
}
51+
4452
for arg in fn_abi.args.iter_mut() {
4553
if arg.is_ignore() || !arg.layout.is_sized() {
4654
continue;
@@ -51,6 +59,16 @@ pub(crate) fn compute_abi_info<'a, Ty, C>(
5159
continue;
5260
}
5361

62+
let size_in_regs = arg.layout.size.bits().div_ceil(32);
63+
let mut pass_in_reg = free_regs >= size_in_regs;
64+
65+
// In fastcall/vectorcall only 32-bit integers can be passed in registers.
66+
if opts.flavor == Flavor::FastcallOrVectorcall && size_in_regs > 1 {
67+
pass_in_reg = false;
68+
// Once we've had an argument which doesn't fit, don't try to fit any more.
69+
free_regs = 0;
70+
}
71+
5472
// FIXME: MSVC 2015+ will pass the first 3 vector arguments in [XYZ]MM0-2
5573
// See https://reviews.llvm.org/D72114 for Clang behavior
5674

@@ -73,14 +91,45 @@ pub(crate) fn compute_abi_info<'a, Ty, C>(
7391
);
7492
arg.make_indirect();
7593
} else if arg.layout.is_aggregate() {
94+
if opts.flavor == Flavor::FastcallOrVectorcall && size_in_regs > 1 {
95+
pass_in_reg = false;
96+
}
7697
// Alignment of the `byval` argument.
7798
// The rules can be found in `X86_32ABIInfo::getTypeStackAlignInBytes` in Clang's `TargetInfo.cpp`.
7899
let byval_align = align_4;
79-
arg.pass_by_stack_offset(Some(byval_align));
100+
if pass_in_reg {
101+
arg.cast_to(Uniform::new(Reg::i32(), arg.layout.size));
102+
} else {
103+
arg.pass_by_stack_offset(Some(byval_align))
104+
}
80105
} else {
106+
let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap();
107+
108+
// Fastcall/regparm won't pass floats in registers
109+
// (though regparm _will_ pass f]oat-containing structs in registers)
110+
if unit.kind != RegKind::Integer {
111+
pass_in_reg = false;
112+
}
113+
81114
arg.extend_integer_width_to(32);
82115
}
116+
// Set the InReg annotation if we're passing by register
117+
if pass_in_reg {
118+
match arg.mode {
119+
PassMode::Ignore
120+
| PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {}
121+
PassMode::Cast { pad_i32: _, ref mut cast } => {
122+
cast.attrs.set(ArgAttribute::InReg);
123+
}
124+
PassMode::Direct(ref mut attrs) => {
125+
attrs.set(ArgAttribute::InReg);
126+
}
127+
PassMode::Pair(..)
128+
| PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
129+
unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
130+
}
131+
};
132+
free_regs -= size_in_regs;
133+
}
83134
}
84-
85-
super::x86::fill_inregs(cx, fn_abi, opts, false);
86135
}

‎tests/assembly-llvm/regparm-module-flag.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,58 @@
1616
extern crate minicore;
1717
use minicore::*;
1818

19+
#[repr(C)]
20+
struct ThreeRegStruct {
21+
a: i64,
22+
b: i32,
23+
}
24+
1925
unsafe extern "C" {
2026
fn memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void;
2127
fn non_builtin_memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void;
28+
fn test_i64_arg(s: i64) -> i64;
29+
fn test_struct_arg(s: ThreeRegStruct);
30+
}
31+
32+
#[unsafe(no_mangle)]
33+
pub unsafe extern "C" fn test_i64() -> i64 {
34+
// REGPARM1-LABEL: test_i64
35+
// REGPARM1: pushl
36+
// REGPARM1: pushl
37+
// REGPARM1: calll test_i64_arg
38+
39+
// REGPARM2-LABEL: test_i64
40+
// REGPARM2: movl $42, %eax
41+
// REGPARM2: xorl %edx, %edx
42+
// REGPARM2: jmp test_i64_arg
43+
44+
// REGPARM3-LABEL: test_i64
45+
// REGPARM3: movl $42, %eax
46+
// REGPARM3: xorl %edx, %edx
47+
// REGPARM3: jmp test_i64_arg
48+
unsafe { test_i64_arg(42) }
49+
}
50+
51+
#[unsafe(no_mangle)]
52+
pub unsafe extern "C" fn test_struct() {
53+
// REGPARM1-LABEL: test_struct
54+
// REGPARM1: movl $0, {{.*}}(%esp)
55+
// REGPARM1: movl $42, {{.*}}(%esp)
56+
// REGPARM1: movl $1, {{.*}}(%esp)
57+
// REGPARM1: calll test_struct_arg
58+
59+
// REGPARM2-LABEL: test_struct
60+
// REGPARM2: movl $0, {{.*}}(%esp)
61+
// REGPARM2: movl $42, {{.*}}(%esp)
62+
// REGPARM2: movl $1, {{.*}}(%esp)
63+
// REGPARM2: calll test_struct_arg
64+
65+
// REGPARM3-LABEL: test_struct
66+
// REGPARM3: movl $42, %eax
67+
// REGPARM3: xorl %edx, %edx
68+
// REGPARM3: movl $1, %ecx
69+
// REGPARM3: jmp test_struct_arg
70+
unsafe { test_struct_arg(ThreeRegStruct { a: 42, b: 1 }) }
2271
}
2372

2473
#[unsafe(no_mangle)]

‎tests/codegen-llvm/regparm-inreg.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,31 @@ pub mod tests {
121121
// regparm3-SAME: i32 inreg noundef %_4)
122122
#[no_mangle]
123123
pub extern "C" fn f12(_: i32, _: __m256, _: i32, _: i32) {}
124+
125+
// regparm0: @f13(i64 noundef %_1)
126+
// regparm1: @f13(i64 noundef %_1)
127+
// regparm2: @f13(i64 inreg noundef %_1)
128+
// regparm3: @f13(i64 inreg noundef %_1)
129+
#[no_mangle]
130+
pub extern "C" fn f13(_: i64) {}
131+
132+
// regparm0: @f14(ptr noalias nofree noundef byval([8 x i8]) align 4 captures(address) dereferenceable(8) %_1)
133+
// regparm1: @f14(ptr noalias nofree noundef byval([8 x i8]) align 4 captures(address) dereferenceable(8) %_1)
134+
// regparm2: @f14([2 x i32] inreg %0)
135+
// regparm3: @f14([2 x i32] inreg %0)
136+
#[no_mangle]
137+
pub extern "C" fn f14(_: S2) {}
138+
139+
#[repr(C)]
140+
struct S3 {
141+
x1: i32,
142+
x2: i32,
143+
x3: i32,
144+
}
145+
// regparm0: @f15(ptr noalias nofree noundef byval([12 x i8]) align 4 captures(address) dereferenceable(12) %_1)
146+
// regparm1: @f15(ptr noalias nofree noundef byval([12 x i8]) align 4 captures(address) dereferenceable(12) %_1)
147+
// regparm2: @f15(ptr noalias nofree noundef byval([12 x i8]) align 4 captures(address) dereferenceable(12) %_1)
148+
// regparm3: @f15([3 x i32] inreg %0)
149+
#[no_mangle]
150+
pub extern "C" fn f15(_: S3) {}
124151
}

0 commit comments

Comments
 (0)