Skip to content

Commit 0ee10e9

Browse files
asbasb
authored
[RISCV] Add additional fence for amocas when required by recent ABI change (#101023)
A recent atomics ABI change / fix requires that for the "A6C" and A6S" atomics ABIs (i.e. both of those supported by LLVM currently), an additional fence is inserted for an atomic_compare_exchange with seq_cst failure ordering. <riscv-non-isa/riscv-elf-psabi-doc#445> This isn't trivial to support through the hooks used by AtomicExpandPass because that pass assumes that when fences are inserted, the original atomics ordering information can be removed from the instruction. Rather than try to change and complicate that API, this patch implements the needed fence insertion through a small special purpose pass.
1 parent a6ab19b commit 0ee10e9

11 files changed

+141
-0
lines changed

‎llvm/lib/Target/RISCV/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ add_llvm_target(RISCVCodeGen
6161
RISCVTargetObjectFile.cpp
6262
RISCVTargetTransformInfo.cpp
6363
RISCVVectorPeephole.cpp
64+
RISCVZacasABIFix.cpp
6465
GISel/RISCVCallLowering.cpp
6566
GISel/RISCVInstructionSelector.cpp
6667
GISel/RISCVLegalizerInfo.cpp

‎llvm/lib/Target/RISCV/RISCV.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ void initializeRISCVMoveMergePass(PassRegistry &);
8585
FunctionPass *createRISCVPushPopOptimizationPass();
8686
void initializeRISCVPushPopOptPass(PassRegistry &);
8787

88+
FunctionPass *createRISCVZacasABIFixPass();
89+
void initializeRISCVZacasABIFixPass(PassRegistry &);
90+
8891
InstructionSelector *
8992
createRISCVInstructionSelector(const RISCVTargetMachine &,
9093
const RISCVSubtarget &,

‎llvm/lib/Target/RISCV/RISCVISelLowering.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,9 @@ class RISCVTargetLowering : public TargetLowering {
684684

685685
bool preferZeroCompareBranch() const override { return true; }
686686

687+
// Note that one specific case requires fence insertion for an
688+
// AtomicCmpXchgInst but is handled via the RISCVZacasABIFix pass rather
689+
// than this hook due to limitations in the interface here.
687690
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
688691
return isa<LoadInst>(I) || isa<StoreInst>(I);
689692
}

‎llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@ bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
420420

421421
void RISCVPassConfig::addIRPasses() {
422422
addPass(createAtomicExpandLegacyPass());
423+
addPass(createRISCVZacasABIFixPass());
423424

424425
if (getOptLevel() != CodeGenOptLevel::None) {
425426
if (EnableLoopDataPrefetch)

‎llvm/lib/Target/RISCV/RISCVZacasABIFix.cpp

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
//===----- RISCVZacasABIFix.cpp -------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// This pass implements a fence insertion for an atomic cmpxchg in a case that
10+
// isn't easy to do with the current AtomicExpandPass hooks API.
11+
//
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "RISCV.h"
15+
#include "RISCVTargetMachine.h"
16+
#include "llvm/ADT/Statistic.h"
17+
#include "llvm/Analysis/ValueTracking.h"
18+
#include "llvm/CodeGen/TargetPassConfig.h"
19+
#include "llvm/IR/Dominators.h"
20+
#include "llvm/IR/IRBuilder.h"
21+
#include "llvm/IR/InstVisitor.h"
22+
#include "llvm/IR/Intrinsics.h"
23+
#include "llvm/IR/IntrinsicsRISCV.h"
24+
#include "llvm/IR/PatternMatch.h"
25+
#include "llvm/InitializePasses.h"
26+
#include "llvm/Pass.h"
27+
28+
using namespace llvm;
29+
30+
#define DEBUG_TYPE "riscv-zacas-abi-fix"
31+
#define PASS_NAME "RISC-V Zacas ABI fix"
32+
33+
namespace {
34+
35+
class RISCVZacasABIFix : public FunctionPass,
36+
public InstVisitor<RISCVZacasABIFix, bool> {
37+
const RISCVSubtarget *ST;
38+
39+
public:
40+
static char ID;
41+
42+
RISCVZacasABIFix() : FunctionPass(ID) {}
43+
44+
bool runOnFunction(Function &F) override;
45+
46+
StringRef getPassName() const override { return PASS_NAME; }
47+
48+
void getAnalysisUsage(AnalysisUsage &AU) const override {
49+
AU.setPreservesCFG();
50+
AU.addRequired<TargetPassConfig>();
51+
}
52+
53+
bool visitInstruction(Instruction &I) { return false; }
54+
bool visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
55+
};
56+
57+
} // end anonymous namespace
58+
59+
// Insert a leading fence (needed for broadest atomics ABI compatibility)
60+
// only if the Zacas extension is enabled and the AtomicCmpXchgInst has a
61+
// SequentiallyConsistent failure ordering.
62+
bool RISCVZacasABIFix::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
63+
assert(ST->hasStdExtZacas() && "only necessary to run in presence of zacas");
64+
IRBuilder<> Builder(&I);
65+
if (I.getFailureOrdering() != AtomicOrdering::SequentiallyConsistent)
66+
return false;
67+
68+
Builder.CreateFence(AtomicOrdering::SequentiallyConsistent);
69+
return true;
70+
}
71+
72+
bool RISCVZacasABIFix::runOnFunction(Function &F) {
73+
auto &TPC = getAnalysis<TargetPassConfig>();
74+
auto &TM = TPC.getTM<RISCVTargetMachine>();
75+
ST = &TM.getSubtarget<RISCVSubtarget>(F);
76+
77+
if (skipFunction(F) || !ST->hasStdExtZacas())
78+
return false;
79+
80+
bool MadeChange = false;
81+
for (auto &BB : F)
82+
for (Instruction &I : llvm::make_early_inc_range(BB))
83+
MadeChange |= visit(I);
84+
85+
return MadeChange;
86+
}
87+
88+
INITIALIZE_PASS_BEGIN(RISCVZacasABIFix, DEBUG_TYPE, PASS_NAME, false, false)
89+
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
90+
INITIALIZE_PASS_END(RISCVZacasABIFix, DEBUG_TYPE, PASS_NAME, false, false)
91+
92+
char RISCVZacasABIFix::ID = 0;
93+
94+
FunctionPass *llvm::createRISCVZacasABIFixPass() {
95+
return new RISCVZacasABIFix();
96+
}

‎llvm/test/CodeGen/RISCV/O0-pipeline.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
; CHECK-NEXT: Expand large div/rem
2323
; CHECK-NEXT: Expand large fp convert
2424
; CHECK-NEXT: Expand Atomic instructions
25+
; CHECK-NEXT: RISC-V Zacas ABI fix
2526
; CHECK-NEXT: Module Verifier
2627
; CHECK-NEXT: Lower Garbage Collection Instructions
2728
; CHECK-NEXT: Shadow Stack GC Lowering

‎llvm/test/CodeGen/RISCV/O3-pipeline.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
; CHECK-NEXT: Expand large div/rem
2727
; CHECK-NEXT: Expand large fp convert
2828
; CHECK-NEXT: Expand Atomic instructions
29+
; CHECK-NEXT: RISC-V Zacas ABI fix
2930
; CHECK-NEXT: Dominator Tree Construction
3031
; CHECK-NEXT: Natural Loop Information
3132
; CHECK-NEXT: Canonicalize natural loops

‎llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ define void @cmpxchg_and_branch1(ptr %ptr, i32 signext %cmp, i32 signext %val) n
3636
; ZACAS: # %bb.0: # %entry
3737
; ZACAS-NEXT: .LBB0_1: # %do_cmpxchg
3838
; ZACAS-NEXT: # =>This Inner Loop Header: Depth=1
39+
; ZACAS-NEXT: fence rw, rw
3940
; ZACAS-NEXT: mv a3, a1
4041
; ZACAS-NEXT: amocas.w.aqrl a3, a2, (a0)
4142
; ZACAS-NEXT: bne a3, a1, .LBB0_1
@@ -76,6 +77,7 @@ define void @cmpxchg_and_branch2(ptr %ptr, i32 signext %cmp, i32 signext %val) n
7677
; ZACAS: # %bb.0: # %entry
7778
; ZACAS-NEXT: .LBB1_1: # %do_cmpxchg
7879
; ZACAS-NEXT: # =>This Inner Loop Header: Depth=1
80+
; ZACAS-NEXT: fence rw, rw
7981
; ZACAS-NEXT: mv a3, a1
8082
; ZACAS-NEXT: amocas.w.aqrl a3, a2, (a0)
8183
; ZACAS-NEXT: beq a3, a1, .LBB1_1
@@ -216,6 +218,7 @@ define void @cmpxchg_masked_and_branch1(ptr %ptr, i8 signext %cmp, i8 signext %v
216218
; RV64IA-ZABHA: # %bb.0: # %entry
217219
; RV64IA-ZABHA-NEXT: .LBB2_1: # %do_cmpxchg
218220
; RV64IA-ZABHA-NEXT: # =>This Inner Loop Header: Depth=1
221+
; RV64IA-ZABHA-NEXT: fence rw, rw
219222
; RV64IA-ZABHA-NEXT: mv a3, a1
220223
; RV64IA-ZABHA-NEXT: amocas.b.aqrl a3, a2, (a0)
221224
; RV64IA-ZABHA-NEXT: bne a3, a1, .LBB2_1
@@ -368,6 +371,7 @@ define void @cmpxchg_masked_and_branch2(ptr %ptr, i8 signext %cmp, i8 signext %v
368371
; RV64IA-ZABHA: # %bb.0: # %entry
369372
; RV64IA-ZABHA-NEXT: .LBB3_1: # %do_cmpxchg
370373
; RV64IA-ZABHA-NEXT: # =>This Inner Loop Header: Depth=1
374+
; RV64IA-ZABHA-NEXT: fence rw, rw
371375
; RV64IA-ZABHA-NEXT: mv a3, a1
372376
; RV64IA-ZABHA-NEXT: amocas.b.aqrl a3, a2, (a0)
373377
; RV64IA-ZABHA-NEXT: beq a3, a1, .LBB3_1
@@ -408,6 +412,7 @@ define void @cmpxchg_and_irrelevant_branch(ptr %ptr, i32 signext %cmp, i32 signe
408412
; ZACAS: # %bb.0: # %entry
409413
; ZACAS-NEXT: .LBB4_1: # %do_cmpxchg
410414
; ZACAS-NEXT: # =>This Inner Loop Header: Depth=1
415+
; ZACAS-NEXT: fence rw, rw
411416
; ZACAS-NEXT: mv a4, a1
412417
; ZACAS-NEXT: amocas.w.aqrl a4, a2, (a0)
413418
; ZACAS-NEXT: beqz a3, .LBB4_1

‎llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1857,6 +1857,7 @@ define void @cmpxchg_i8_seq_cst_seq_cst(ptr %ptr, i8 %cmp, i8 %val) nounwind {
18571857
;
18581858
; RV64IA-WMO-ZABHA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
18591859
; RV64IA-WMO-ZABHA: # %bb.0:
1860+
; RV64IA-WMO-ZABHA-NEXT: fence rw, rw
18601861
; RV64IA-WMO-ZABHA-NEXT: amocas.b.aqrl a1, a2, (a0)
18611862
; RV64IA-WMO-ZABHA-NEXT: ret
18621863
;
@@ -1885,6 +1886,7 @@ define void @cmpxchg_i8_seq_cst_seq_cst(ptr %ptr, i8 %cmp, i8 %val) nounwind {
18851886
;
18861887
; RV64IA-TSO-ZABHA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
18871888
; RV64IA-TSO-ZABHA: # %bb.0:
1889+
; RV64IA-TSO-ZABHA-NEXT: fence rw, rw
18881890
; RV64IA-TSO-ZABHA-NEXT: amocas.b a1, a2, (a0)
18891891
; RV64IA-TSO-ZABHA-NEXT: ret
18901892
%res = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst seq_cst
@@ -3787,6 +3789,7 @@ define void @cmpxchg_i16_seq_cst_seq_cst(ptr %ptr, i16 %cmp, i16 %val) nounwind
37873789
;
37883790
; RV64IA-WMO-ZABHA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
37893791
; RV64IA-WMO-ZABHA: # %bb.0:
3792+
; RV64IA-WMO-ZABHA-NEXT: fence rw, rw
37903793
; RV64IA-WMO-ZABHA-NEXT: amocas.h.aqrl a1, a2, (a0)
37913794
; RV64IA-WMO-ZABHA-NEXT: ret
37923795
;
@@ -3816,6 +3819,7 @@ define void @cmpxchg_i16_seq_cst_seq_cst(ptr %ptr, i16 %cmp, i16 %val) nounwind
38163819
;
38173820
; RV64IA-TSO-ZABHA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
38183821
; RV64IA-TSO-ZABHA: # %bb.0:
3822+
; RV64IA-TSO-ZABHA-NEXT: fence rw, rw
38193823
; RV64IA-TSO-ZABHA-NEXT: amocas.h a1, a2, (a0)
38203824
; RV64IA-TSO-ZABHA-NEXT: ret
38213825
%res = cmpxchg ptr %ptr, i16 %cmp, i16 %val seq_cst seq_cst
@@ -4788,6 +4792,7 @@ define void @cmpxchg_i32_seq_cst_seq_cst(ptr %ptr, i32 %cmp, i32 %val) nounwind
47884792
;
47894793
; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
47904794
; RV32IA-WMO-ZACAS: # %bb.0:
4795+
; RV32IA-WMO-ZACAS-NEXT: fence rw, rw
47914796
; RV32IA-WMO-ZACAS-NEXT: amocas.w.aqrl a1, a2, (a0)
47924797
; RV32IA-WMO-ZACAS-NEXT: ret
47934798
;
@@ -4804,6 +4809,7 @@ define void @cmpxchg_i32_seq_cst_seq_cst(ptr %ptr, i32 %cmp, i32 %val) nounwind
48044809
;
48054810
; RV32IA-TSO-ZACAS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
48064811
; RV32IA-TSO-ZACAS: # %bb.0:
4812+
; RV32IA-TSO-ZACAS-NEXT: fence rw, rw
48074813
; RV32IA-TSO-ZACAS-NEXT: amocas.w a1, a2, (a0)
48084814
; RV32IA-TSO-ZACAS-NEXT: ret
48094815
;
@@ -4834,11 +4840,13 @@ define void @cmpxchg_i32_seq_cst_seq_cst(ptr %ptr, i32 %cmp, i32 %val) nounwind
48344840
;
48354841
; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
48364842
; RV64IA-WMO-ZACAS: # %bb.0:
4843+
; RV64IA-WMO-ZACAS-NEXT: fence rw, rw
48374844
; RV64IA-WMO-ZACAS-NEXT: amocas.w.aqrl a1, a2, (a0)
48384845
; RV64IA-WMO-ZACAS-NEXT: ret
48394846
;
48404847
; RV64IA-WMO-ZABHA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
48414848
; RV64IA-WMO-ZABHA: # %bb.0:
4849+
; RV64IA-WMO-ZABHA-NEXT: fence rw, rw
48424850
; RV64IA-WMO-ZABHA-NEXT: amocas.w.aqrl a1, a2, (a0)
48434851
; RV64IA-WMO-ZABHA-NEXT: ret
48444852
;
@@ -4856,11 +4864,13 @@ define void @cmpxchg_i32_seq_cst_seq_cst(ptr %ptr, i32 %cmp, i32 %val) nounwind
48564864
;
48574865
; RV64IA-TSO-ZACAS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
48584866
; RV64IA-TSO-ZACAS: # %bb.0:
4867+
; RV64IA-TSO-ZACAS-NEXT: fence rw, rw
48594868
; RV64IA-TSO-ZACAS-NEXT: amocas.w a1, a2, (a0)
48604869
; RV64IA-TSO-ZACAS-NEXT: ret
48614870
;
48624871
; RV64IA-TSO-ZABHA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
48634872
; RV64IA-TSO-ZABHA: # %bb.0:
4873+
; RV64IA-TSO-ZABHA-NEXT: fence rw, rw
48644874
; RV64IA-TSO-ZABHA-NEXT: amocas.w a1, a2, (a0)
48654875
; RV64IA-TSO-ZABHA-NEXT: ret
48664876
%res = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
@@ -5753,11 +5763,13 @@ define void @cmpxchg_i64_seq_cst_seq_cst(ptr %ptr, i64 %cmp, i64 %val) nounwind
57535763
;
57545764
; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i64_seq_cst_seq_cst:
57555765
; RV64IA-WMO-ZACAS: # %bb.0:
5766+
; RV64IA-WMO-ZACAS-NEXT: fence rw, rw
57565767
; RV64IA-WMO-ZACAS-NEXT: amocas.d.aqrl a1, a2, (a0)
57575768
; RV64IA-WMO-ZACAS-NEXT: ret
57585769
;
57595770
; RV64IA-WMO-ZABHA-LABEL: cmpxchg_i64_seq_cst_seq_cst:
57605771
; RV64IA-WMO-ZABHA: # %bb.0:
5772+
; RV64IA-WMO-ZABHA-NEXT: fence rw, rw
57615773
; RV64IA-WMO-ZABHA-NEXT: amocas.d.aqrl a1, a2, (a0)
57625774
; RV64IA-WMO-ZABHA-NEXT: ret
57635775
;
@@ -5774,11 +5786,13 @@ define void @cmpxchg_i64_seq_cst_seq_cst(ptr %ptr, i64 %cmp, i64 %val) nounwind
57745786
;
57755787
; RV64IA-TSO-ZACAS-LABEL: cmpxchg_i64_seq_cst_seq_cst:
57765788
; RV64IA-TSO-ZACAS: # %bb.0:
5789+
; RV64IA-TSO-ZACAS-NEXT: fence rw, rw
57775790
; RV64IA-TSO-ZACAS-NEXT: amocas.d a1, a2, (a0)
57785791
; RV64IA-TSO-ZACAS-NEXT: ret
57795792
;
57805793
; RV64IA-TSO-ZABHA-LABEL: cmpxchg_i64_seq_cst_seq_cst:
57815794
; RV64IA-TSO-ZABHA: # %bb.0:
5795+
; RV64IA-TSO-ZABHA-NEXT: fence rw, rw
57825796
; RV64IA-TSO-ZABHA-NEXT: amocas.d a1, a2, (a0)
57835797
; RV64IA-TSO-ZABHA-NEXT: ret
57845798
%res = cmpxchg ptr %ptr, i64 %cmp, i64 %val seq_cst seq_cst

0 commit comments

Comments
 (0)