diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp index cd5813a5338ea..41cf0fc2cef4f 100644 --- a/llvm/lib/Target/X86/X86LowerAMXType.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp @@ -248,7 +248,11 @@ std::pair ShapeCalculator::getShape(IntrinsicInst *II, case Intrinsic::x86_tdpbuud_internal: case Intrinsic::x86_tdpbf16ps_internal: case Intrinsic::x86_tdpfp16ps_internal: - case Intrinsic::x86_tmmultf32ps_internal: { + case Intrinsic::x86_tmmultf32ps_internal: + case Intrinsic::x86_tdpbf8ps_internal: + case Intrinsic::x86_tdpbhf8ps_internal: + case Intrinsic::x86_tdphbf8ps_internal: + case Intrinsic::x86_tdphf8ps_internal: { switch (OpNo) { case 3: Row = II->getArgOperand(0); diff --git a/llvm/test/CodeGen/X86/amx-fp8-internal.ll b/llvm/test/CodeGen/X86/amx-fp8-internal.ll index ade71499b361a..87d755f2a69f6 100644 --- a/llvm/test/CodeGen/X86/amx-fp8-internal.ll +++ b/llvm/test/CodeGen/X86/amx-fp8-internal.ll @@ -59,6 +59,91 @@ define void @test_amx(i8* %pointer, i8* %base, i64 %stride) { ret void } +; Function Attrs: nounwind +define dso_local void @__tile_dpbf8ps(ptr %dst, ptr %src1, ptr %src2) #0 { +; CHECK-LABEL: __tile_dpbf8ps: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: subq $4976, %rsp # imm = 0x1370 +; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movb $1, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movzwl (%rsi), %eax +; CHECK-NEXT: movb %al, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movb %al, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movb %al, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movswq 2(%rdx), %rcx +; CHECK-NEXT: movw %cx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movw %cx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movw %cx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movswq 2(%rsi), %r8 +; CHECK-NEXT: movw %r8w, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movzwl %r8w, %r9d +; CHECK-NEXT: movb %r9b, {{[0-9]+}}(%rsp) +; CHECK-NEXT: shrl $2, %r9d +; CHECK-NEXT: movb %r9b, {{[0-9]+}}(%rsp) +; CHECK-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; CHECK-NEXT: addq $64, %rdi +; CHECK-NEXT: tileloadd (%rdi,%rcx), %tmm0 +; CHECK-NEXT: addq $64, %rsi +; CHECK-NEXT: tileloadd (%rsi,%r8), %tmm1 +; CHECK-NEXT: addq $64, %rdx +; CHECK-NEXT: tileloadd (%rdx,%rcx), %tmm2 +; CHECK-NEXT: movabsq $64, %rbp +; CHECK-NEXT: tilestored %tmm0, 896(%rsp,%rbp) # 1024-byte Folded Spill +; CHECK-NEXT: tileloadd 896(%rsp,%rbp), %tmm3 # 1024-byte Folded Reload +; CHECK-NEXT: tdpbf8ps %tmm2, %tmm1, %tmm3 +; CHECK-NEXT: tilestored %tmm3, (%rdi,%rcx) +; CHECK-NEXT: tilestored %tmm0, 1920(%rsp,%rbp) # 1024-byte Folded Spill +; CHECK-NEXT: tileloadd 1920(%rsp,%rbp), %tmm3 # 1024-byte Folded Reload +; CHECK-NEXT: tdpbhf8ps %tmm2, %tmm1, %tmm3 +; CHECK-NEXT: tilestored %tmm3, (%rdi,%rcx) +; CHECK-NEXT: tilestored %tmm0, 2944(%rsp,%rbp) # 1024-byte Folded Spill +; CHECK-NEXT: tileloadd 2944(%rsp,%rbp), %tmm3 # 1024-byte Folded Reload +; CHECK-NEXT: tdphbf8ps %tmm2, %tmm1, %tmm3 +; CHECK-NEXT: tilestored %tmm3, (%rdi,%rcx) +; CHECK-NEXT: tdphf8ps %tmm2, %tmm1, %tmm0 +; CHECK-NEXT: tilestored %tmm0, (%rdi,%rcx) +; CHECK-NEXT: addq $4976, %rsp # imm = 0x1370 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: tilerelease +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq +entry: + %0 = load i16, ptr %src1, align 64 + %col = getelementptr inbounds nuw i8, ptr %src2, i64 2 + %1 = load i16, ptr %col, align 2 + %col1 = getelementptr inbounds nuw i8, ptr %src1, i64 2 + %2 = load i16, ptr %col1, align 2 + %tile = getelementptr inbounds nuw i8, ptr %dst, i64 64 + %3 = load <256 x i32>, ptr %tile, align 64 + %tile2 = getelementptr inbounds nuw i8, ptr %src1, i64 64 + %4 = load <256 x i32>, ptr %tile2, align 64 + %tile3 = getelementptr inbounds nuw i8, ptr %src2, i64 64 + %5 = load <256 x i32>, ptr %tile3, align 64 + %6 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %3) + %7 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %4) + %8 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %5) + + %9 = tail call x86_amx @llvm.x86.tdpbf8ps.internal(i16 %0, i16 %1, i16 %2, x86_amx %6, x86_amx %7, x86_amx %8) + %10 = tail call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %9) + store <256 x i32> %10, ptr %tile, align 64 + + %11 = tail call x86_amx @llvm.x86.tdpbhf8ps.internal(i16 %0, i16 %1, i16 %2, x86_amx %6, x86_amx %7, x86_amx %8) + %12 = tail call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %11) + store <256 x i32> %12, ptr %tile, align 64 + + %13 = tail call x86_amx @llvm.x86.tdphbf8ps.internal(i16 %0, i16 %1, i16 %2, x86_amx %6, x86_amx %7, x86_amx %8) + %14 = tail call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %13) + store <256 x i32> %14, ptr %tile, align 64 + + %15 = tail call x86_amx @llvm.x86.tdphf8ps.internal(i16 %0, i16 %1, i16 %2, x86_amx %6, x86_amx %7, x86_amx %8) + %16 = tail call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %15) + store <256 x i32> %16, ptr %tile, align 64 + + ret void +} + declare x86_amx @llvm.x86.tilezero.internal(i16, i16) declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx) @@ -67,3 +152,6 @@ declare x86_amx @llvm.x86.tdpbf8ps.internal(i16, i16, i16, x86_amx, x86_amx, x86 declare x86_amx @llvm.x86.tdpbhf8ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) declare x86_amx @llvm.x86.tdphbf8ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) declare x86_amx @llvm.x86.tdphf8ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) + +attributes #0 = { nounwind } +