Skip to content

Commit dba68d9

Browse files
borsbors
committed
Auto merge of #128969 - DianQK:cast-target, r=<try>
Use `insertvalue` and `extractvalue` instead of `memcpy` in CastTarget r? `@nikic`
2 parents c9bd03c + 448885c commit dba68d9

File tree

9 files changed

+633
-357
lines changed

9 files changed

+633
-357
lines changed

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
1-
use std::cmp;
2-
31
use libc::c_uint;
42
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
53
use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
64
use rustc_codegen_ssa::traits::*;
7-
use rustc_codegen_ssa::MemFlags;
85
use rustc_middle::ty::layout::LayoutOf;
96
pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
107
use rustc_middle::ty::Ty;
@@ -215,35 +212,8 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
215212
PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
216213
bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
217214
}
218-
PassMode::Cast { cast, pad_i32: _ } => {
219-
// The ABI mandates that the value is passed as a different struct representation.
220-
// Spill and reload it from the stack to convert from the ABI representation to
221-
// the Rust representation.
222-
let scratch_size = cast.size(bx);
223-
let scratch_align = cast.align(bx);
224-
// Note that the ABI type may be either larger or smaller than the Rust type,
225-
// due to the presence or absence of trailing padding. For example:
226-
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
227-
// when passed by value, making it smaller.
228-
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
229-
// when passed by value, making it larger.
230-
let copy_bytes =
231-
cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes());
232-
// Allocate some scratch space...
233-
let llscratch = bx.alloca(scratch_size, scratch_align);
234-
bx.lifetime_start(llscratch, scratch_size);
235-
// ...store the value...
236-
bx.store(val, llscratch, scratch_align);
237-
// ... and then memcpy it to the intended destination.
238-
bx.memcpy(
239-
dst.val.llval,
240-
self.layout.align.abi,
241-
llscratch,
242-
scratch_align,
243-
bx.const_usize(copy_bytes),
244-
MemFlags::empty(),
245-
);
246-
bx.lifetime_end(llscratch, scratch_size);
215+
PassMode::Cast { cast, .. } => {
216+
cast.cast_other_abi_to_rust(bx, val, dst.val.llval, self.layout);
247217
}
248218
_ => {
249219
OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ use super::{CachedLlbb, FunctionCx, LocalRef};
2323
use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
2424
use crate::common::{self, IntPredicate};
2525
use crate::errors::CompilerBuiltinsCannotCall;
26+
use crate::meth;
2627
use crate::traits::*;
27-
use crate::{meth, MemFlags};
2828

2929
// Indicates if we are in the middle of merging a BB's successor into it. This
3030
// can happen when BB jumps directly to its successor and the successor has no
@@ -462,7 +462,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
462462
}
463463
}
464464

465-
PassMode::Cast { cast: cast_ty, pad_i32: _ } => {
465+
PassMode::Cast { cast, pad_i32: _ } => {
466466
let op = match self.locals[mir::RETURN_PLACE] {
467467
LocalRef::Operand(op) => op,
468468
LocalRef::PendingOperand => bug!("use of return before def"),
@@ -471,23 +471,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
471471
}
472472
LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
473473
};
474-
let llslot = match op.val {
474+
let (llslot, align) = match op.val {
475475
Immediate(_) | Pair(..) => {
476476
let scratch = PlaceRef::alloca(bx, self.fn_abi.ret.layout);
477477
op.val.store(bx, scratch);
478-
scratch.val.llval
478+
(scratch.val.llval, scratch.val.align)
479479
}
480480
Ref(place_val) => {
481481
assert_eq!(
482482
place_val.align, op.layout.align.abi,
483483
"return place is unaligned!"
484484
);
485-
place_val.llval
485+
(place_val.llval, place_val.align)
486486
}
487487
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
488488
};
489-
let ty = bx.cast_backend_type(cast_ty);
490-
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
489+
cast.cast_rust_abi_to_other(bx, llslot, align)
491490
}
492491
};
493492
bx.ret(llval);
@@ -1515,35 +1514,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
15151514

15161515
if by_ref && !arg.is_indirect() {
15171516
// Have to load the argument, maybe while casting it.
1518-
if let PassMode::Cast { cast, pad_i32: _ } = &arg.mode {
1519-
// The ABI mandates that the value is passed as a different struct representation.
1520-
// Spill and reload it from the stack to convert from the Rust representation to
1521-
// the ABI representation.
1522-
let scratch_size = cast.size(bx);
1523-
let scratch_align = cast.align(bx);
1524-
// Note that the ABI type may be either larger or smaller than the Rust type,
1525-
// due to the presence or absence of trailing padding. For example:
1526-
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
1527-
// when passed by value, making it smaller.
1528-
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
1529-
// when passed by value, making it larger.
1530-
let copy_bytes = cmp::min(cast.unaligned_size(bx).bytes(), arg.layout.size.bytes());
1531-
// Allocate some scratch space...
1532-
let llscratch = bx.alloca(scratch_size, scratch_align);
1533-
bx.lifetime_start(llscratch, scratch_size);
1534-
// ...memcpy the value...
1535-
bx.memcpy(
1536-
llscratch,
1537-
scratch_align,
1538-
llval,
1539-
align,
1540-
bx.const_usize(copy_bytes),
1541-
MemFlags::empty(),
1542-
);
1543-
// ...and then load it with the ABI type.
1544-
let cast_ty = bx.cast_backend_type(cast);
1545-
llval = bx.load(cast_ty, llscratch, scratch_align);
1546-
bx.lifetime_end(llscratch, scratch_size);
1517+
if let PassMode::Cast { cast, .. } = &arg.mode {
1518+
llval = cast.cast_rust_abi_to_other(bx, llval, align);
15471519
} else {
15481520
// We can't use `PlaceRef::load` here because the argument
15491521
// may have a type we don't treat as immediate, but the ABI

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
236236
let llretptr = start_bx.get_param(0);
237237
return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
238238
}
239-
PassMode::Cast { ref cast, .. } => {
239+
PassMode::Cast { ref cast, .. }
240+
if start_bx.cast_backend_type(cast)
241+
== start_bx.reg_backend_type(&cast.rest.unit)
242+
&& cast.size(&start_bx) > layout.size =>
243+
{
244+
// When using just a single register, we directly use load or store instructions,
245+
// so we need to ensure that the allocated space is sufficiently large.
240246
debug!("alloc: {:?} (return place) -> place", local);
241247
let size = cast.size(&start_bx);
242248
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));

‎compiler/rustc_codegen_ssa/src/traits/abi.rs

Lines changed: 144 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,148 @@
1-
use super::BackendTypes;
1+
use rustc_middle::bug;
2+
use rustc_middle::ty::layout::TyAndLayout;
3+
use rustc_target::abi::call::CastTarget;
4+
use rustc_target::abi::Align;
5+
6+
use super::consts::ConstMethods;
7+
use super::type_::BaseTypeMethods;
8+
use super::{BackendTypes, BuilderMethods, LayoutTypeMethods};
29

310
pub trait AbiBuilderMethods<'tcx>: BackendTypes {
411
fn get_param(&mut self, index: usize) -> Self::Value;
512
}
13+
14+
/// The ABI mandates that the value is passed as a different struct representation.
15+
pub trait CastTargetAbiExt<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
16+
/// Spill and reload it from the stack to convert from the Rust representation to the ABI representation.
17+
fn cast_rust_abi_to_other(&self, bx: &mut Bx, src: Bx::Value, align: Align) -> Bx::Value;
18+
/// Spill and reload it from the stack to convert from the ABI representation to the Rust representation.
19+
fn cast_other_abi_to_rust(
20+
&self,
21+
bx: &mut Bx,
22+
src: Bx::Value,
23+
dst: Bx::Value,
24+
layout: TyAndLayout<'tcx>,
25+
);
26+
}
27+
28+
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> CastTargetAbiExt<'a, 'tcx, Bx> for CastTarget {
29+
fn cast_rust_abi_to_other(&self, bx: &mut Bx, src: Bx::Value, align: Align) -> Bx::Value {
30+
let cast_ty = bx.cast_backend_type(self);
31+
match bx.type_kind(cast_ty) {
32+
crate::common::TypeKind::Struct | crate::common::TypeKind::Array => {
33+
let mut index = 0;
34+
let mut offset = 0;
35+
let mut target = bx.const_poison(cast_ty);
36+
for reg in self.prefix.iter().filter_map(|&x| x) {
37+
let ptr = if offset == 0 {
38+
src
39+
} else {
40+
bx.inbounds_ptradd(src, bx.const_usize(offset))
41+
};
42+
let load = bx.load(bx.reg_backend_type(&reg), ptr, align);
43+
target = bx.insert_value(target, load, index);
44+
index += 1;
45+
offset += reg.size.bytes();
46+
}
47+
let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 {
48+
(0, 0)
49+
} else {
50+
(
51+
self.rest.total.bytes() / self.rest.unit.size.bytes(),
52+
self.rest.total.bytes() % self.rest.unit.size.bytes(),
53+
)
54+
};
55+
for _ in 0..rest_count {
56+
let ptr = if offset == 0 {
57+
src
58+
} else {
59+
bx.inbounds_ptradd(src, bx.const_usize(offset))
60+
};
61+
let load = bx.load(bx.reg_backend_type(&self.rest.unit), ptr, align);
62+
target = bx.insert_value(target, load, index);
63+
index += 1;
64+
offset += self.rest.unit.size.bytes();
65+
}
66+
if rem_bytes != 0 {
67+
let ptr = bx.inbounds_ptradd(src, bx.const_usize(offset));
68+
let load = bx.load(bx.reg_backend_type(&self.rest.unit), ptr, align);
69+
target = bx.insert_value(target, load, index);
70+
}
71+
target
72+
}
73+
ty_kind if bx.type_kind(bx.reg_backend_type(&self.rest.unit)) == ty_kind => {
74+
bx.load(cast_ty, src, align)
75+
}
76+
ty_kind => bug!("cannot cast {ty_kind:?} to the ABI representation in CastTarget"),
77+
}
78+
}
79+
80+
fn cast_other_abi_to_rust(
81+
&self,
82+
bx: &mut Bx,
83+
src: Bx::Value,
84+
dst: Bx::Value,
85+
layout: TyAndLayout<'tcx>,
86+
) {
87+
let align = layout.align.abi;
88+
match bx.type_kind(bx.val_ty(src)) {
89+
crate::common::TypeKind::Struct | crate::common::TypeKind::Array => {
90+
let mut index = 0;
91+
let mut offset = 0;
92+
for reg in self.prefix.iter().filter_map(|&x| x) {
93+
let from = bx.extract_value(src, index);
94+
let ptr = if index == 0 {
95+
dst
96+
} else {
97+
bx.inbounds_ptradd(dst, bx.const_usize(offset))
98+
};
99+
bx.store(from, ptr, align);
100+
index += 1;
101+
offset += reg.size.bytes();
102+
}
103+
let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 {
104+
(0, 0)
105+
} else {
106+
(
107+
self.rest.total.bytes() / self.rest.unit.size.bytes(),
108+
self.rest.total.bytes() % self.rest.unit.size.bytes(),
109+
)
110+
};
111+
for _ in 0..rest_count {
112+
let from = bx.extract_value(src, index);
113+
let ptr = if offset == 0 {
114+
dst
115+
} else {
116+
bx.inbounds_ptradd(dst, bx.const_usize(offset))
117+
};
118+
bx.store(from, ptr, align);
119+
index += 1;
120+
offset += self.rest.unit.size.bytes();
121+
}
122+
if rem_bytes != 0 {
123+
let from = bx.extract_value(src, index);
124+
let ptr = bx.inbounds_ptradd(dst, bx.const_usize(offset));
125+
bx.store(from, ptr, align);
126+
}
127+
}
128+
ty_kind if bx.type_kind(bx.reg_backend_type(&self.rest.unit)) == ty_kind => {
129+
let scratch_size = self.size(bx);
130+
let src = if scratch_size > layout.size {
131+
// When using just a single register, we directly use load or store instructions,
132+
// so we must allocate sufficient space.
133+
let scratch_align = self.align(bx);
134+
let llscratch = bx.alloca(scratch_size, scratch_align);
135+
bx.lifetime_start(llscratch, scratch_size);
136+
bx.store(src, llscratch, scratch_align);
137+
let tmp = bx.load(bx.backend_type(layout), llscratch, scratch_align);
138+
bx.lifetime_end(llscratch, scratch_size);
139+
tmp
140+
} else {
141+
src
142+
};
143+
bx.store(src, dst, align);
144+
}
145+
ty_kind => bug!("cannot cast {ty_kind:?} to the Rust representation in CastTarget"),
146+
};
147+
}
148+
}

‎compiler/rustc_codegen_ssa/src/traits/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use std::fmt;
3333
use rustc_middle::ty::layout::{HasParamEnv, HasTyCtxt};
3434
use rustc_target::spec::HasTargetSpec;
3535

36-
pub use self::abi::AbiBuilderMethods;
36+
pub use self::abi::{AbiBuilderMethods, CastTargetAbiExt};
3737
pub use self::asm::{AsmBuilderMethods, AsmMethods, GlobalAsmOperandRef, InlineAsmOperandRef};
3838
pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods};
3939
pub use self::builder::{BuilderMethods, OverflowOp};

‎tests/codegen/array-codegen.rs

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,23 @@
55
// CHECK-LABEL: @array_load
66
#[no_mangle]
77
pub fn array_load(a: &[u8; 4]) -> [u8; 4] {
8-
// CHECK-NOT: alloca
9-
// CHECK: %[[ALLOCA:.+]] = alloca [4 x i8], align 1
10-
// CHECK-NOT: alloca
11-
// CHECK: call void @llvm.memcpy.{{.+}}(ptr align 1 %[[ALLOCA]], ptr align 1 %a, {{.+}} 4, i1 false)
12-
// CHECK: %[[TEMP:.+]] = load i32, ptr %[[ALLOCA]], align 1
13-
// CHECK: ret i32 %[[TEMP]]
8+
// CHECK-NEXT: [[START:.*:]]
9+
// CHECK-NEXT: %[[ALLOCA:.+]] = alloca [4 x i8], align 1
10+
// CHECK-NEXT: call void @llvm.memcpy.{{.+}}(ptr align 1 %[[ALLOCA]], ptr align 1 %a, {{.+}} 4, i1 false)
11+
// CHECK-NEXT: %[[TEMP:.+]] = load i32, ptr %[[ALLOCA]], align 1
12+
// CHECK-NEXT: ret i32 %[[TEMP]]
1413
*a
1514
}
1615

1716
// CHECK-LABEL: @array_store
1817
#[no_mangle]
1918
pub fn array_store(a: [u8; 4], p: &mut [u8; 4]) {
20-
// CHECK-NOT: alloca
21-
// CHECK: %[[TEMP:.+]] = alloca [4 x i8], [[TEMPALIGN:align [0-9]+]]
22-
// CHECK-NOT: alloca
23-
// CHECK: %a = alloca [4 x i8]
24-
// CHECK-NOT: alloca
25-
// store i32 %0, ptr %[[TEMP]]
26-
// CHECK: call void @llvm.memcpy.{{.+}}(ptr align 1 %a, ptr [[TEMPALIGN]] %[[TEMP]], {{.+}} 4, i1 false)
27-
// CHECK: call void @llvm.memcpy.{{.+}}(ptr align 1 %p, ptr align 1 %a, {{.+}} 4, i1 false)
19+
// CHECK-SAME: i32 [[TMP0:%.*]], ptr{{.*}} [[P:%.*]])
20+
// CHECK-NEXT: [[START:.*:]]
21+
// CHECK-NEXT: [[A:%.*]] = alloca [4 x i8], align 1
22+
// CHECK-NEXT: store i32 [[TMP0]], ptr [[A]], align 1
23+
// CHECK-NEXT: call void @llvm.memcpy.{{.+}}(ptr align 1 [[P]], ptr align 1 [[A]], {{.+}} 4, i1 false)
24+
// CHECK-NEXT: ret void
2825
*p = a;
2926
}
3027

0 commit comments

Comments
 (0)