Skip to content

Commit 934001d

Browse files
borsbors
committed
Auto merge of #137271 - nikic:gep-nuw-2, r=<try>
Emit getelementptr inbounds nuw for pointer::add() Lower pointer::add (via intrinsic::offset with unsigned offset) to getelementptr inbounds nuw on LLVM versions that support it. This lets LLVM make use of the pre-condition that the offset addition does not wrap in an unsigned sense. Together with inbounds, this also implies that the offset is non-negative. Fixes #137217.
2 parents 5986ff0 + 511c91a commit 934001d

File tree

9 files changed

+90
-26
lines changed

9 files changed

+90
-26
lines changed

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ use crate::abi::FnAbiLlvmExt;
3232
use crate::attributes;
3333
use crate::common::Funclet;
3434
use crate::context::{CodegenCx, SimpleCx};
35-
use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, Metadata, True};
35+
use crate::llvm::{
36+
self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
37+
};
3638
use crate::type_::Type;
3739
use crate::type_of::LayoutLlvmExt;
3840
use crate::value::Value;
@@ -908,13 +910,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
908910

909911
fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
910912
unsafe {
911-
llvm::LLVMBuildGEP2(
913+
llvm::LLVMBuildGEPWithNoWrapFlags(
912914
self.llbuilder,
913915
ty,
914916
ptr,
915917
indices.as_ptr(),
916918
indices.len() as c_uint,
917919
UNNAMED,
920+
GEPNoWrapFlags::default(),
918921
)
919922
}
920923
}
@@ -926,13 +929,33 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
926929
indices: &[&'ll Value],
927930
) -> &'ll Value {
928931
unsafe {
929-
llvm::LLVMBuildInBoundsGEP2(
932+
llvm::LLVMBuildGEPWithNoWrapFlags(
933+
self.llbuilder,
934+
ty,
935+
ptr,
936+
indices.as_ptr(),
937+
indices.len() as c_uint,
938+
UNNAMED,
939+
GEPNoWrapFlags::InBounds,
940+
)
941+
}
942+
}
943+
944+
fn inbounds_nuw_gep(
945+
&mut self,
946+
ty: &'ll Type,
947+
ptr: &'ll Value,
948+
indices: &[&'ll Value],
949+
) -> &'ll Value {
950+
unsafe {
951+
llvm::LLVMBuildGEPWithNoWrapFlags(
930952
self.llbuilder,
931953
ty,
932954
ptr,
933955
indices.as_ptr(),
934956
indices.len() as c_uint,
935957
UNNAMED,
958+
GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
936959
)
937960
}
938961
}

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -954,6 +954,17 @@ bitflags! {
954954
}
955955
}
956956

957+
// These values **must** match with LLVMGEPNoWrapFlags
958+
bitflags! {
959+
#[repr(transparent)]
960+
#[derive(Default)]
961+
pub struct GEPNoWrapFlags : c_uint {
962+
const InBounds = 1 << 0;
963+
const NUSW = 1 << 1;
964+
const NUW = 1 << 2;
965+
}
966+
}
967+
957968
unsafe extern "C" {
958969
pub type ModuleBuffer;
959970
}
@@ -1454,21 +1465,14 @@ unsafe extern "C" {
14541465

14551466
pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value;
14561467

1457-
pub(crate) fn LLVMBuildGEP2<'a>(
1458-
B: &Builder<'a>,
1459-
Ty: &'a Type,
1460-
Pointer: &'a Value,
1461-
Indices: *const &'a Value,
1462-
NumIndices: c_uint,
1463-
Name: *const c_char,
1464-
) -> &'a Value;
1465-
pub(crate) fn LLVMBuildInBoundsGEP2<'a>(
1468+
pub(crate) fn LLVMBuildGEPWithNoWrapFlags<'a>(
14661469
B: &Builder<'a>,
14671470
Ty: &'a Type,
14681471
Pointer: &'a Value,
14691472
Indices: *const &'a Value,
14701473
NumIndices: c_uint,
14711474
Name: *const c_char,
1475+
Flags: GEPNoWrapFlags,
14721476
) -> &'a Value;
14731477

14741478
// Casts

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -664,9 +664,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
664664
lhs.layout.ty,
665665
),
666666

667-
(OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => {
668-
self.codegen_scalar_binop(bx, op, lhs_val, rhs_val, lhs.layout.ty)
669-
}
667+
(OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => self
668+
.codegen_scalar_binop(
669+
bx,
670+
op,
671+
lhs_val,
672+
rhs_val,
673+
lhs.layout.ty,
674+
rhs.layout.ty,
675+
),
670676

671677
_ => bug!(),
672678
};
@@ -887,10 +893,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
887893
op: mir::BinOp,
888894
lhs: Bx::Value,
889895
rhs: Bx::Value,
890-
input_ty: Ty<'tcx>,
896+
lhs_ty: Ty<'tcx>,
897+
rhs_ty: Ty<'tcx>,
891898
) -> Bx::Value {
892-
let is_float = input_ty.is_floating_point();
893-
let is_signed = input_ty.is_signed();
899+
let is_float = lhs_ty.is_floating_point();
900+
let is_signed = lhs_ty.is_signed();
894901
match op {
895902
mir::BinOp::Add => {
896903
if is_float {
@@ -956,17 +963,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
956963
mir::BinOp::BitAnd => bx.and(lhs, rhs),
957964
mir::BinOp::BitXor => bx.xor(lhs, rhs),
958965
mir::BinOp::Offset => {
959-
let pointee_type = input_ty
966+
let pointee_type = lhs_ty
960967
.builtin_deref(true)
961-
.unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty));
968+
.unwrap_or_else(|| bug!("deref of non-pointer {:?}", lhs_ty));
962969
let pointee_layout = bx.cx().layout_of(pointee_type);
963970
if pointee_layout.is_zst() {
964971
// `Offset` works in terms of the size of pointee,
965972
// so offsetting a pointer to ZST is a noop.
966973
lhs
967974
} else {
968975
let llty = bx.cx().backend_type(pointee_layout);
969-
bx.inbounds_gep(llty, lhs, &[rhs])
976+
if !rhs_ty.is_signed() {
977+
bx.inbounds_nuw_gep(llty, lhs, &[rhs])
978+
} else {
979+
bx.inbounds_gep(llty, lhs, &[rhs])
980+
}
970981
}
971982
}
972983
mir::BinOp::Shl | mir::BinOp::ShlUnchecked => {

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,14 @@ pub trait BuilderMethods<'a, 'tcx>:
332332
ptr: Self::Value,
333333
indices: &[Self::Value],
334334
) -> Self::Value;
335+
fn inbounds_nuw_gep(
336+
&mut self,
337+
ty: Self::Type,
338+
ptr: Self::Value,
339+
indices: &[Self::Value],
340+
) -> Self::Value {
341+
self.inbounds_gep(ty, ptr, indices)
342+
}
335343
fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
336344
self.gep(self.cx().type_i8(), ptr, &[offset])
337345
}

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,6 +1767,24 @@ extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS,
17671767
return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS), unwrap(RHS)));
17681768
}
17691769

1770+
#if LLVM_VERSION_LT(19, 0)
1771+
enum {
1772+
LLVMGEPFlagInBounds = (1 << 0),
1773+
LLVMGEPFlagNUSW = (1 << 1),
1774+
LLVMGEPFlagNUW = (1 << 2),
1775+
};
1776+
LLVMValueRef LLVMBuildGEPWithNoWrapFlags(LLVMBuilderRef B, LLVMTypeRef Ty,
1777+
LLVMValueRef Pointer,
1778+
LLVMValueRef *Indices,
1779+
unsigned NumIndices, const char *Name,
1780+
unsigned NoWrapFlags) {
1781+
if (NoWrapFlags & LLVMGEPFlagInBounds)
1782+
return LLVMBuildInBoundsGEP2(B, Ty, Pointer, Indices, NumIndices, Name);
1783+
else
1784+
return LLVMBuildGEP2(B, Ty, Pointer, Indices, NumIndices, Name);
1785+
}
1786+
#endif
1787+
17701788
// Transfers ownership of DiagnosticHandler unique_ptr to the caller.
17711789
extern "C" DiagnosticHandler *
17721790
LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) {

‎tests/codegen/gep-index.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ fn index_on_struct(a: &[Foo], index: usize) -> &Foo {
1818
// CHECK-LABEL: @offset_on_struct(
1919
#[no_mangle]
2020
fn offset_on_struct(a: *const Foo, index: usize) -> *const Foo {
21-
// CHECK: getelementptr inbounds %Foo, ptr %a, {{i64|i32}} %index
21+
// CHECK: getelementptr inbounds{{( nuw)?}} %Foo, ptr %a, {{i64|i32}} %index
2222
unsafe { a.add(index) }
2323
}
2424

@@ -32,6 +32,6 @@ fn index_on_i32(a: &[i32], index: usize) -> &i32 {
3232
// CHECK-LABEL: @offset_on_i32(
3333
#[no_mangle]
3434
fn offset_on_i32(a: *const i32, index: usize) -> *const i32 {
35-
// CHECK: getelementptr inbounds i32, ptr %a, {{i64|i32}} %index
35+
// CHECK: getelementptr inbounds{{( nuw)?}} i32, ptr %a, {{i64|i32}} %index
3636
unsafe { a.add(index) }
3737
}

‎tests/codegen/intrinsics/offset.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
2727
// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
2828
#[no_mangle]
2929
pub unsafe fn offset_usize(p: *const u64, d: usize) -> *const u64 {
30-
// CHECK: %[[R:.*]] = getelementptr inbounds i64, ptr %p, [[SIZE]] %d
30+
// CHECK: %[[R:.*]] = getelementptr inbounds{{( nuw)?}} i64, ptr %p, [[SIZE]] %d
3131
// CHECK-NEXT: ret ptr %[[R]]
3232
offset(p, d)
3333
}

‎tests/codegen/intrinsics/ptr_metadata.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ pub unsafe fn dyn_byte_offset(
2828
p: *const dyn std::fmt::Debug,
2929
n: usize,
3030
) -> *const dyn std::fmt::Debug {
31-
// CHECK: %[[Q:.+]] = getelementptr inbounds i8, ptr %p.0, i64 %n
31+
// CHECK: %[[Q:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %p.0, i64 %n
3232
// CHECK: %[[TEMP1:.+]] = insertvalue { ptr, ptr } poison, ptr %[[Q]], 0
3333
// CHECK: %[[TEMP2:.+]] = insertvalue { ptr, ptr } %[[TEMP1]], ptr %p.1, 1
3434
// CHECK: ret { ptr, ptr } %[[TEMP2]]

‎tests/codegen/ptr-arithmetic.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
// CHECK-SAME: [[WORD:i[0-9]+]] noundef %n)
77
#[no_mangle]
88
pub unsafe fn i32_add(p: *const i32, n: usize) -> *const i32 {
9-
// CHECK: %[[TEMP:.+]] = getelementptr inbounds i32, ptr %p, [[WORD]] %n
9+
// CHECK: %[[TEMP:.+]] = getelementptr inbounds{{( nuw)?}} i32, ptr %p, [[WORD]] %n
1010
// CHECK: ret ptr %[[TEMP]]
1111
p.add(n)
1212
}

0 commit comments

Comments
 (0)