-
Notifications
You must be signed in to change notification settings - Fork 13.7k
[VPlan] Use VPWidenIntrinsicRecipe to vp.select. #110489
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
8ae59c7
d7fc1fd
78b59ac
efab51f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -79,7 +79,6 @@ bool VPRecipeBase::mayWriteToMemory() const { | |
return !cast<VPWidenCallRecipe>(this) | ||
->getCalledScalarFunction() | ||
->onlyReadsMemory(); | ||
case VPWidenIntrinsicSC: | ||
return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory(); | ||
case VPBranchOnMaskSC: | ||
case VPScalarIVStepsSC: | ||
|
@@ -1042,6 +1041,14 @@ StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const { | |
return Intrinsic::getBaseName(VectorIntrinsicID); | ||
} | ||
|
||
bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const { | ||
assert(is_contained(operands(), Op) && "Op must be an operand of the recipe"); | ||
// Vector predication intrinsics only demand the the first lane the last | ||
// operand (the EVL operand). | ||
return VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) && | ||
Op == getOperand(getNumOperands() - 1); | ||
Comment on lines
+1048
to
+1049
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Meet an implementation divergence when rebasing #101641.
Maybe we need to consider isVectorIntrinsicWithScalarOpAtArg? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't see the changes in the current version of the PR I think? Would probably be good to improve (I assume scalar arg here means we only request the first lane). If there are unrelated tests that improve, would be good to prepare a separate patch There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Dropped the related changes while resolving conflicts, as the patch can still function even without considering |
||
} | ||
|
||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||
void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent, | ||
VPSlotTracker &SlotTracker) const { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
; REQUIRES: asserts | ||
|
||
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ | ||
; RUN: -force-tail-folding-style=data-with-evl \ | ||
; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ | ||
; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s | ||
|
||
define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { | ||
; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { | ||
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF | ||
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count | ||
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count | ||
|
||
; IF-EVL: vector.ph: | ||
; IF-EVL-NEXT: Successor(s): vector loop | ||
|
||
; IF-EVL: <x1> vector loop: { | ||
; IF-EVL-NEXT: vector.body: | ||
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION | ||
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%[0-9]+]]> | ||
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> | ||
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> | ||
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> | ||
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> | ||
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> | ||
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> | ||
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]> | ||
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> | ||
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]> | ||
; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]> | ||
; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]> | ||
; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>) | ||
; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]> | ||
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> | ||
; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]> | ||
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> | ||
; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 | ||
; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> | ||
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%[0-9]+]]> = add vp<[[IV]]>, vp<[[VFUF]]> | ||
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> | ||
; IF-EVL-NEXT: No successors | ||
; IF-EVL-NEXT: } | ||
|
||
entry: | ||
br label %for.body | ||
|
||
for.body: | ||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] | ||
%arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv | ||
%0 = load i32, ptr %arrayidx, align 4 | ||
%arrayidx3 = getelementptr inbounds i32, ptr %c, i64 %indvars.iv | ||
%1 = load i32, ptr %arrayidx3, align 4 | ||
%cmp4 = icmp sgt i32 %0, %1 | ||
%2 = sub i32 0, %1 | ||
%cond.p = select i1 %cmp4, i32 %1, i32 %2 | ||
%cond = add i32 %cond.p, %0 | ||
%arrayidx15 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv | ||
store i32 %cond, ptr %arrayidx15, align 4 | ||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 | ||
%exitcond.not = icmp eq i64 %indvars.iv.next, %N | ||
br i1 %exitcond.not, label %exit, label %for.body | ||
|
||
exit: | ||
ret void | ||
} |
Uh oh!
There was an error while loading. Please reload this page.