From 99fcfbda5a732d4562cfbf6f5d25c11841f4c716 Mon Sep 17 00:00:00 2001 From: Jin Huang Date: Fri, 19 Dec 2025 22:49:22 +0000 Subject: [PATCH] [profcheck] Fix profile metadata missing in ExpandLargeDivRem --- llvm/lib/Transforms/Utils/IntegerDivision.cpp | 75 +++++++++++++++++-- .../Transforms/ExpandIRInsts/X86/sdiv129.ll | 20 +++-- .../Transforms/ExpandIRInsts/X86/srem129.ll | 20 +++-- .../Transforms/ExpandIRInsts/X86/udiv129.ll | 20 +++-- .../Transforms/ExpandIRInsts/X86/urem129.ll | 20 +++-- .../Transforms/ExpandIRInsts/X86/vector.ll | 71 ++++++++++-------- llvm/utils/profcheck-xfail.txt | 5 -- 7 files changed, 161 insertions(+), 70 deletions(-) diff --git a/llvm/lib/Transforms/Utils/IntegerDivision.cpp b/llvm/lib/Transforms/Utils/IntegerDivision.cpp index e95a7a9ae525a..115c254df251e 100644 --- a/llvm/lib/Transforms/Utils/IntegerDivision.cpp +++ b/llvm/lib/Transforms/Utils/IntegerDivision.cpp @@ -16,13 +16,22 @@ #include "llvm/Transforms/Utils/IntegerDivision.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/ProfDataUtils.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; #define DEBUG_TYPE "integer-division" +namespace llvm { +extern cl::opt ProfcheckDisableMetadataFixes; +} + /// Generate code to compute the remainder of two signed integers. Returns the /// remainder, which will have the sign of the dividend. Builder's insert point /// should be pointing where the caller wants code generated, e.g. at the srem @@ -235,11 +244,53 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor, Value *Tmp1 = Builder.CreateCall(CTLZ, {Dividend, True}); Value *SR = Builder.CreateSub(Tmp0, Tmp1); Value *Ret0_4 = Builder.CreateICmpUGT(SR, MSB); + + // Add 'unlikely' branch weights to the select instruction ('Ret0') generated + // by 'CreateLogicalOr'. We assume the 'true' path, where either the divisor + // or dividend is zero ('Ret0_3'), is less likely than the 'false' path, + // which corresponds to the |divisor| > |dividend| ('Ret0_4'). Value *Ret0 = Builder.CreateLogicalOr(Ret0_3, Ret0_4); + if (!ProfcheckDisableMetadataFixes) { + if (Instruction *Ret0SelectI = dyn_cast(Ret0)) { + Ret0SelectI->setMetadata( + LLVMContext::MD_prof, + MDBuilder(Ret0SelectI->getContext()).createUnlikelyBranchWeights()); + } + } Value *RetDividend = Builder.CreateICmpEQ(SR, MSB); - Value *RetVal = Builder.CreateSelect(Ret0, Zero, Dividend); - Value *EarlyRet = Builder.CreateLogicalOr(Ret0, RetDividend); - Builder.CreateCondBr(EarlyRet, End, BB1); + + // This select instruction (RetVal) immediately follows the Ret0 check. Since + // Ret0_4 determines if the divisor is greater then the dividend so the + // unknown (50/50) branch weights are fairly accurate for RetVal. + Value *RetVal; + if (!ProfcheckDisableMetadataFixes) { + RetVal = Builder.CreateSelectWithUnknownProfile(Ret0, Zero, Dividend, + DEBUG_TYPE); + } else { + RetVal = Builder.CreateSelect(Ret0, Zero, Dividend); + } + // This select instruction (EarlyRet) is used to check another edge case, and + // it share the same branch weights as RetVal so the unknown (50/50)branch + // weights are also accurate for EarlyRet. + Value *EarlyRet; + if (!ProfcheckDisableMetadataFixes) { + EarlyRet = Builder.CreateSelectWithUnknownProfile( + Ret0, ConstantInt::getAllOnesValue(Ret0->getType()), RetDividend, + DEBUG_TYPE); + } else { + EarlyRet = Builder.CreateLogicalOr(Ret0, RetDividend); + } + + // The condition of this branch is based on `EarlyRet`. `EarlyRet` is true + // only for special cases like dividend or divisor being zero, or the divisor + // being greater than the dividend. Thus, the branch to `End` is unlikely, + // and we expect to more frequently enter `BB1`. + Instruction *ConBrSpecialCases = Builder.CreateCondBr(EarlyRet, End, BB1); + if (!ProfcheckDisableMetadataFixes) { + ConBrSpecialCases->setMetadata(LLVMContext::MD_prof, + MDBuilder(ConBrSpecialCases->getContext()) + .createUnlikelyBranchWeights()); + } // ; bb1: ; preds = %special-cases // ; %sr_1 = add i32 %sr, 1 @@ -252,7 +303,15 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor, Value *Tmp2 = Builder.CreateSub(MSB, SR); Value *Q = Builder.CreateShl(Dividend, Tmp2); Value *SkipLoop = Builder.CreateICmpEQ(SR_1, Zero); - Builder.CreateCondBr(SkipLoop, LoopExit, Preheader); + // This branch is highly unlikely. SR_1 (SR + 1) is expected to be in [1, 129] + // because the case where |divisor| > |dividend| (which would make SR + // negative) has alredy been intercepted in the previous SpecialCases BB. + Instruction *ConBrBB1 = Builder.CreateCondBr(SkipLoop, LoopExit, Preheader); + if (!ProfcheckDisableMetadataFixes) { + ConBrBB1->setMetadata(LLVMContext::MD_prof, + MDBuilder(ConBrBB1->getContext()) + .createUnlikelyBranchWeights()); + } // ; preheader: ; preds = %bb1 // ; %tmp3 = lshr i32 %dividend, %sr_1 @@ -298,7 +357,13 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor, Value *R = Builder.CreateSub(Tmp7, Tmp11); Value *SR_2 = Builder.CreateAdd(SR_3, NegOne); Value *Tmp12 = Builder.CreateICmpEQ(SR_2, Zero); - Builder.CreateCondBr(Tmp12, LoopExit, DoWhile); + // The loop implements the core bit-by-bit binary long division algorithm. + // Since each iteration generates one bit of the quotient until covering all + // significant bits so we would label it as unknown (50/50) branch weights. + Instruction *ConBrDoWhile = Builder.CreateCondBr(Tmp12, LoopExit, DoWhile); + if (!ProfcheckDisableMetadataFixes) { + setExplicitlyUnknownBranchWeightsIfProfiled(*ConBrDoWhile, DEBUG_TYPE, F); + } // ; loop-exit: ; preds = %do-while, %bb1 // ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ] diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll index fc823cd543144..01b50da066d88 100644 --- a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll +++ b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll @@ -2,8 +2,9 @@ ; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 128 < %s | FileCheck %s ; RUN: opt -S -mtriple=x86_64-- -passes='require,expand-ir-insts' -expand-div-rem-bits 128 < %s | FileCheck %s -define void @sdiv129(ptr %ptr, ptr %out) nounwind { +define void @sdiv129(ptr %ptr, ptr %out) nounwind !prof !0 { ; CHECK-LABEL: @sdiv129( +; CHECK: !prof [[PROF_0:![0-9]+]] { ; CHECK-NEXT: _udiv-special-cases: ; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16 ; CHECK-NEXT: [[TMP0:%.*]] = freeze i129 [[A]] @@ -24,11 +25,11 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP15:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP10]], i1 true) ; CHECK-NEXT: [[TMP16:%.*]] = sub i129 [[TMP14]], [[TMP15]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp ugt i129 [[TMP16]], 128 -; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], i1 true, i1 [[TMP17]] +; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], i1 true, i1 [[TMP17]], !prof [[PROF_1:![0-9]+]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i129 [[TMP16]], 128 -; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP18]], i129 0, i129 [[TMP10]] -; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP18]], i1 true, i1 [[TMP19]] -; CHECK-NEXT: br i1 [[TMP21]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP18]], i129 0, i129 [[TMP10]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP18]], i1 true, i1 [[TMP19]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP21]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP22:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP37:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP23:%.*]] = phi i129 [ [[TMP46:%.*]], [[UDIV_BB1]] ], [ [[TMP34:%.*]], [[UDIV_DO_WHILE]] ] @@ -52,7 +53,7 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP39]] = sub i129 [[TMP32]], [[TMP38]] ; CHECK-NEXT: [[TMP40]] = add i129 [[TMP27]], -1 ; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i129 [[TMP40]], 0 -; CHECK-NEXT: br i1 [[TMP41]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]] +; CHECK-NEXT: br i1 [[TMP41]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]], !prof [[PROF_2:![0-9]+]] ; CHECK: udiv-preheader: ; CHECK-NEXT: [[TMP42]] = lshr i129 [[TMP10]], [[TMP44]] ; CHECK-NEXT: [[TMP43]] = add i129 [[TMP9]], -1 @@ -62,7 +63,7 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP45:%.*]] = sub i129 128, [[TMP16]] ; CHECK-NEXT: [[TMP46]] = shl i129 [[TMP10]], [[TMP45]] ; CHECK-NEXT: [[TMP47:%.*]] = icmp eq i129 [[TMP44]], 0 -; CHECK-NEXT: br i1 [[TMP47]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP47]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP48:%.*]] = phi i129 [ [[TMP25]], [[UDIV_LOOP_EXIT]] ], [ [[TMP20]], [[_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP49:%.*]] = xor i129 [[TMP48]], [[TMP8]] @@ -75,3 +76,8 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind { store i129 %res, ptr %out ret void } + +!0 = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_0]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_1]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[PROF_2]] = !{!"unknown", !"integer-division"} \ No newline at end of file diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll index 667152228d258..0485c1ac5607e 100644 --- a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll +++ b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll @@ -2,8 +2,9 @@ ; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 128 < %s | FileCheck %s ; RUN: opt -S -mtriple=x86_64-- -passes='require,expand-ir-insts' -expand-div-rem-bits 128 < %s | FileCheck %s -define void @test(ptr %ptr, ptr %out) nounwind { +define void @test(ptr %ptr, ptr %out) nounwind !prof !0 { ; CHECK-LABEL: @test( +; CHECK: !prof [[PROF_0:![0-9]+]] { ; CHECK-NEXT: _udiv-special-cases: ; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16 ; CHECK-NEXT: [[TMP0:%.*]] = freeze i129 [[A]] @@ -25,11 +26,11 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP16:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP11]], i1 true) ; CHECK-NEXT: [[TMP17:%.*]] = sub i129 [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i129 [[TMP17]], 128 -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], i1 true, i1 [[TMP18]] +; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], i1 true, i1 [[TMP18]], !prof [[PROF_1:![0-9]+]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i129 [[TMP17]], 128 -; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP19]], i129 0, i129 [[TMP11]] -; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP19]], i1 true, i1 [[TMP20]] -; CHECK-NEXT: br i1 [[TMP22]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP19]], i129 0, i129 [[TMP11]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP19]], i1 true, i1 [[TMP20]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP22]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP23:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP38:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP24:%.*]] = phi i129 [ [[TMP47:%.*]], [[UDIV_BB1]] ], [ [[TMP35:%.*]], [[UDIV_DO_WHILE]] ] @@ -53,7 +54,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP40]] = sub i129 [[TMP33]], [[TMP39]] ; CHECK-NEXT: [[TMP41]] = add i129 [[TMP28]], -1 ; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i129 [[TMP41]], 0 -; CHECK-NEXT: br i1 [[TMP42]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]] +; CHECK-NEXT: br i1 [[TMP42]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]], !prof [[PROF_2:![0-9]+]] ; CHECK: udiv-preheader: ; CHECK-NEXT: [[TMP43]] = lshr i129 [[TMP11]], [[TMP45]] ; CHECK-NEXT: [[TMP44]] = add i129 [[TMP10]], -1 @@ -63,7 +64,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP46:%.*]] = sub i129 128, [[TMP17]] ; CHECK-NEXT: [[TMP47]] = shl i129 [[TMP11]], [[TMP46]] ; CHECK-NEXT: [[TMP48:%.*]] = icmp eq i129 [[TMP45]], 0 -; CHECK-NEXT: br i1 [[TMP48]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP48]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP49:%.*]] = phi i129 [ [[TMP26]], [[UDIV_LOOP_EXIT]] ], [ [[TMP21]], [[_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP50:%.*]] = mul i129 [[TMP9]], [[TMP49]] @@ -78,3 +79,8 @@ define void @test(ptr %ptr, ptr %out) nounwind { store i129 %res, ptr %out ret void } + +!0 = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_0]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_1]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[PROF_2]] = !{!"unknown", !"integer-division"} diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll index b2b83815f79b0..c3a21dbd145d8 100644 --- a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll +++ b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll @@ -2,8 +2,9 @@ ; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 128 < %s | FileCheck %s ; RUN: opt -S -mtriple=x86_64-- -passes='require,expand-ir-insts' -expand-div-rem-bits 128 < %s | FileCheck %s -define void @test(ptr %ptr, ptr %out) nounwind { +define void @test(ptr %ptr, ptr %out) nounwind !prof !0 { ; CHECK-LABEL: @test( +; CHECK: !prof [[PROF_0:![0-9]+]] { ; CHECK-NEXT: _udiv-special-cases: ; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16 ; CHECK-NEXT: [[TMP0:%.*]] = freeze i129 3 @@ -15,11 +16,11 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP6:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP1]], i1 true) ; CHECK-NEXT: [[TMP7:%.*]] = sub i129 [[TMP5]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i129 [[TMP7]], 128 -; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP4]], i1 true, i1 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP4]], i1 true, i1 [[TMP8]], !prof [[PROF_1:![0-9]+]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i129 [[TMP7]], 128 -; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i129 0, i129 [[TMP1]] -; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP9]], i1 true, i1 [[TMP10]] -; CHECK-NEXT: br i1 [[TMP12]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i129 0, i129 [[TMP1]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP9]], i1 true, i1 [[TMP10]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP12]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP13:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP28:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP14:%.*]] = phi i129 [ [[TMP37:%.*]], [[UDIV_BB1]] ], [ [[TMP25:%.*]], [[UDIV_DO_WHILE]] ] @@ -43,7 +44,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP30]] = sub i129 [[TMP23]], [[TMP29]] ; CHECK-NEXT: [[TMP31]] = add i129 [[TMP18]], -1 ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i129 [[TMP31]], 0 -; CHECK-NEXT: br i1 [[TMP32]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]] +; CHECK-NEXT: br i1 [[TMP32]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]], !prof [[PROF_2:![0-9]+]] ; CHECK: udiv-preheader: ; CHECK-NEXT: [[TMP33]] = lshr i129 [[TMP1]], [[TMP35]] ; CHECK-NEXT: [[TMP34]] = add i129 [[TMP0]], -1 @@ -53,7 +54,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP36:%.*]] = sub i129 128, [[TMP7]] ; CHECK-NEXT: [[TMP37]] = shl i129 [[TMP1]], [[TMP36]] ; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i129 [[TMP35]], 0 -; CHECK-NEXT: br i1 [[TMP38]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP38]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP39:%.*]] = phi i129 [ [[TMP16]], [[UDIV_LOOP_EXIT]] ], [ [[TMP11]], [[_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: store i129 [[TMP39]], ptr [[OUT:%.*]], align 16 @@ -64,3 +65,8 @@ define void @test(ptr %ptr, ptr %out) nounwind { store i129 %res, ptr %out ret void } + +!0 = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_0]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_1]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[PROF_2]] = !{!"unknown", !"integer-division"} diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll index 46e72001b2c2d..0809b70b55a42 100644 --- a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll +++ b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll @@ -2,8 +2,9 @@ ; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 128 < %s | FileCheck %s ; RUN: opt -S -mtriple=x86_64-- -passes='require,expand-ir-insts' -expand-div-rem-bits 128 < %s | FileCheck %s -define void @test(ptr %ptr, ptr %out) nounwind { +define void @test(ptr %ptr, ptr %out) nounwind !prof !0 { ; CHECK-LABEL: @test( +; CHECK: !prof [[PROF_0:![0-9]+]] { ; CHECK-NEXT: _udiv-special-cases: ; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16 ; CHECK-NEXT: [[TMP0:%.*]] = freeze i129 [[A]] @@ -17,11 +18,11 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP8:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP3]], i1 true) ; CHECK-NEXT: [[TMP9:%.*]] = sub i129 [[TMP7]], [[TMP8]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp ugt i129 [[TMP9]], 128 -; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP10]], !prof [[PROF_1:![0-9]+]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i129 [[TMP9]], 128 -; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP11]], i129 0, i129 [[TMP3]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP11]], i1 true, i1 [[TMP12]] -; CHECK-NEXT: br i1 [[TMP14]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP11]], i129 0, i129 [[TMP3]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP11]], i1 true, i1 [[TMP12]], !prof [[PROF_2:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP14]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP15:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP30:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP16:%.*]] = phi i129 [ [[TMP39:%.*]], [[UDIV_BB1]] ], [ [[TMP27:%.*]], [[UDIV_DO_WHILE]] ] @@ -45,7 +46,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP32]] = sub i129 [[TMP25]], [[TMP31]] ; CHECK-NEXT: [[TMP33]] = add i129 [[TMP20]], -1 ; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i129 [[TMP33]], 0 -; CHECK-NEXT: br i1 [[TMP34]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]] +; CHECK-NEXT: br i1 [[TMP34]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]], !prof [[PROF_2:![0-9]+]] ; CHECK: udiv-preheader: ; CHECK-NEXT: [[TMP35]] = lshr i129 [[TMP3]], [[TMP37]] ; CHECK-NEXT: [[TMP36]] = add i129 [[TMP2]], -1 @@ -55,7 +56,7 @@ define void @test(ptr %ptr, ptr %out) nounwind { ; CHECK-NEXT: [[TMP38:%.*]] = sub i129 128, [[TMP9]] ; CHECK-NEXT: [[TMP39]] = shl i129 [[TMP3]], [[TMP38]] ; CHECK-NEXT: [[TMP40:%.*]] = icmp eq i129 [[TMP37]], 0 -; CHECK-NEXT: br i1 [[TMP40]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP40]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF_1:![0-9]+]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP41:%.*]] = phi i129 [ [[TMP18]], [[UDIV_LOOP_EXIT]] ], [ [[TMP13]], [[_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP42:%.*]] = mul i129 [[TMP1]], [[TMP41]] @@ -68,3 +69,8 @@ define void @test(ptr %ptr, ptr %out) nounwind { store i129 %res, ptr %out ret void } + +!0 = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_0]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF_1]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[PROF_2]] = !{!"unknown", !"integer-division"} \ No newline at end of file diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/vector.ll b/llvm/test/Transforms/ExpandIRInsts/X86/vector.ll index 58e74b8d17b55..8d85d8f347016 100644 --- a/llvm/test/Transforms/ExpandIRInsts/X86/vector.ll +++ b/llvm/test/Transforms/ExpandIRInsts/X86/vector.ll @@ -2,9 +2,9 @@ ; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 128 < %s | FileCheck %s ; RUN: opt -S -mtriple=x86_64-- -passes='require,expand-ir-insts' -expand-div-rem-bits 128 < %s | FileCheck %s -define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { +define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind !prof !0 { ; CHECK-LABEL: define <2 x i129> @sdiv129( -; CHECK-SAME: <2 x i129> [[A:%.*]], <2 x i129> [[B:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-SAME: <2 x i129> [[A:%.*]], <2 x i129> [[B:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0:![0-9]+]] { ; CHECK-NEXT: _udiv-special-cases_udiv-special-cases: ; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x i129> [[A]], i64 0 ; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i129> [[B]], i64 0 @@ -26,11 +26,11 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP17:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP12]], i1 true) ; CHECK-NEXT: [[TMP18:%.*]] = sub i129 [[TMP16]], [[TMP17]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp ugt i129 [[TMP18]], 128 -; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP15]], i1 true, i1 [[TMP19]] +; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP15]], i1 true, i1 [[TMP19]], !prof [[PROF1:![0-9]+]] ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i129 [[TMP18]], 128 -; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP20]], i129 0, i129 [[TMP12]] -; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP20]], i1 true, i1 [[TMP21]] -; CHECK-NEXT: br i1 [[TMP23]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]] +; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP20]], i129 0, i129 [[TMP12]], !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP20]], i1 true, i1 [[TMP21]], !prof [[PROF2]] +; CHECK-NEXT: br i1 [[TMP23]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit2: ; CHECK-NEXT: [[TMP24:%.*]] = phi i129 [ 0, [[UDIV_BB15]] ], [ [[TMP39:%.*]], [[UDIV_DO_WHILE3:%.*]] ] ; CHECK-NEXT: [[TMP25:%.*]] = phi i129 [ [[TMP48:%.*]], [[UDIV_BB15]] ], [ [[TMP36:%.*]], [[UDIV_DO_WHILE3]] ] @@ -54,7 +54,7 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP41]] = sub i129 [[TMP34]], [[TMP40]] ; CHECK-NEXT: [[TMP42]] = add i129 [[TMP29]], -1 ; CHECK-NEXT: [[TMP43:%.*]] = icmp eq i129 [[TMP42]], 0 -; CHECK-NEXT: br i1 [[TMP43]], label [[UDIV_LOOP_EXIT2:%.*]], label [[UDIV_DO_WHILE3]] +; CHECK-NEXT: br i1 [[TMP43]], label [[UDIV_LOOP_EXIT2:%.*]], label [[UDIV_DO_WHILE3]], !prof [[PROF2]] ; CHECK: udiv-preheader4: ; CHECK-NEXT: [[TMP44]] = lshr i129 [[TMP12]], [[TMP46]] ; CHECK-NEXT: [[TMP45]] = add i129 [[TMP11]], -1 @@ -64,7 +64,7 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP47:%.*]] = sub i129 128, [[TMP18]] ; CHECK-NEXT: [[TMP48]] = shl i129 [[TMP12]], [[TMP47]] ; CHECK-NEXT: [[TMP49:%.*]] = icmp eq i129 [[TMP46]], 0 -; CHECK-NEXT: br i1 [[TMP49]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]] +; CHECK-NEXT: br i1 [[TMP49]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]], !prof [[PROF1]] ; CHECK: udiv-end1: ; CHECK-NEXT: [[TMP50:%.*]] = phi i129 [ [[TMP27]], [[UDIV_LOOP_EXIT2]] ], [ [[TMP22]], [[_UDIV_SPECIAL_CASES_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP51:%.*]] = xor i129 [[TMP50]], [[TMP10]] @@ -90,11 +90,11 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP71:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP66]], i1 true) ; CHECK-NEXT: [[TMP72:%.*]] = sub i129 [[TMP70]], [[TMP71]] ; CHECK-NEXT: [[TMP73:%.*]] = icmp ugt i129 [[TMP72]], 128 -; CHECK-NEXT: [[TMP74:%.*]] = select i1 [[TMP69]], i1 true, i1 [[TMP73]] +; CHECK-NEXT: [[TMP74:%.*]] = select i1 [[TMP69]], i1 true, i1 [[TMP73]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP75:%.*]] = icmp eq i129 [[TMP72]], 128 -; CHECK-NEXT: [[TMP76:%.*]] = select i1 [[TMP74]], i129 0, i129 [[TMP66]] -; CHECK-NEXT: [[TMP77:%.*]] = select i1 [[TMP74]], i1 true, i1 [[TMP75]] -; CHECK-NEXT: br i1 [[TMP77]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: [[TMP76:%.*]] = select i1 [[TMP74]], i129 0, i129 [[TMP66]], !prof [[PROF2]] +; CHECK-NEXT: [[TMP77:%.*]] = select i1 [[TMP74]], i1 true, i1 [[TMP75]], !prof [[PROF2]] +; CHECK-NEXT: br i1 [[TMP77]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP78:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP93:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP79:%.*]] = phi i129 [ [[TMP102:%.*]], [[UDIV_BB1]] ], [ [[TMP90:%.*]], [[UDIV_DO_WHILE]] ] @@ -118,7 +118,7 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP95]] = sub i129 [[TMP88]], [[TMP94]] ; CHECK-NEXT: [[TMP96]] = add i129 [[TMP83]], -1 ; CHECK-NEXT: [[TMP97:%.*]] = icmp eq i129 [[TMP96]], 0 -; CHECK-NEXT: br i1 [[TMP97]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]] +; CHECK-NEXT: br i1 [[TMP97]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]], !prof [[PROF2]] ; CHECK: udiv-preheader: ; CHECK-NEXT: [[TMP98]] = lshr i129 [[TMP66]], [[TMP100]] ; CHECK-NEXT: [[TMP99]] = add i129 [[TMP65]], -1 @@ -128,7 +128,7 @@ define <2 x i129> @sdiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP101:%.*]] = sub i129 128, [[TMP72]] ; CHECK-NEXT: [[TMP102]] = shl i129 [[TMP66]], [[TMP101]] ; CHECK-NEXT: [[TMP103:%.*]] = icmp eq i129 [[TMP100]], 0 -; CHECK-NEXT: br i1 [[TMP103]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP103]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF1]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP104:%.*]] = phi i129 [ [[TMP81]], [[UDIV_LOOP_EXIT]] ], [ [[TMP76]], [[UDIV_END1]] ] ; CHECK-NEXT: [[TMP105:%.*]] = xor i129 [[TMP104]], [[TMP64]] @@ -155,11 +155,11 @@ define <2 x i129> @udiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP8:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP3]], i1 true) ; CHECK-NEXT: [[TMP9:%.*]] = sub i129 [[TMP7]], [[TMP8]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp ugt i129 [[TMP9]], 128 -; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP10]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i129 [[TMP9]], 128 ; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP11]], i129 0, i129 [[TMP3]] ; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP11]], i1 true, i1 [[TMP12]] -; CHECK-NEXT: br i1 [[TMP14]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]] +; CHECK-NEXT: br i1 [[TMP14]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit2: ; CHECK-NEXT: [[TMP15:%.*]] = phi i129 [ 0, [[UDIV_BB15]] ], [ [[TMP30:%.*]], [[UDIV_DO_WHILE3:%.*]] ] ; CHECK-NEXT: [[TMP16:%.*]] = phi i129 [ [[TMP39:%.*]], [[UDIV_BB15]] ], [ [[TMP27:%.*]], [[UDIV_DO_WHILE3]] ] @@ -193,7 +193,7 @@ define <2 x i129> @udiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP38:%.*]] = sub i129 128, [[TMP9]] ; CHECK-NEXT: [[TMP39]] = shl i129 [[TMP3]], [[TMP38]] ; CHECK-NEXT: [[TMP40:%.*]] = icmp eq i129 [[TMP37]], 0 -; CHECK-NEXT: br i1 [[TMP40]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]] +; CHECK-NEXT: br i1 [[TMP40]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]], !prof [[PROF1]] ; CHECK: udiv-end1: ; CHECK-NEXT: [[TMP41:%.*]] = phi i129 [ [[TMP18]], [[UDIV_LOOP_EXIT2]] ], [ [[TMP13]], [[_UDIV_SPECIAL_CASES_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP42:%.*]] = insertelement <2 x i129> poison, i129 [[TMP41]], i64 0 @@ -208,11 +208,11 @@ define <2 x i129> @udiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP51:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP46]], i1 true) ; CHECK-NEXT: [[TMP52:%.*]] = sub i129 [[TMP50]], [[TMP51]] ; CHECK-NEXT: [[TMP53:%.*]] = icmp ugt i129 [[TMP52]], 128 -; CHECK-NEXT: [[TMP54:%.*]] = select i1 [[TMP49]], i1 true, i1 [[TMP53]] +; CHECK-NEXT: [[TMP54:%.*]] = select i1 [[TMP49]], i1 true, i1 [[TMP53]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP55:%.*]] = icmp eq i129 [[TMP52]], 128 ; CHECK-NEXT: [[TMP56:%.*]] = select i1 [[TMP54]], i129 0, i129 [[TMP46]] ; CHECK-NEXT: [[TMP57:%.*]] = select i1 [[TMP54]], i1 true, i1 [[TMP55]] -; CHECK-NEXT: br i1 [[TMP57]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: br i1 [[TMP57]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP58:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP73:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP59:%.*]] = phi i129 [ [[TMP82:%.*]], [[UDIV_BB1]] ], [ [[TMP70:%.*]], [[UDIV_DO_WHILE]] ] @@ -246,7 +246,7 @@ define <2 x i129> @udiv129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP81:%.*]] = sub i129 128, [[TMP52]] ; CHECK-NEXT: [[TMP82]] = shl i129 [[TMP46]], [[TMP81]] ; CHECK-NEXT: [[TMP83:%.*]] = icmp eq i129 [[TMP80]], 0 -; CHECK-NEXT: br i1 [[TMP83]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP83]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF1]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP84:%.*]] = phi i129 [ [[TMP61]], [[UDIV_LOOP_EXIT]] ], [ [[TMP56]], [[UDIV_END1]] ] ; CHECK-NEXT: [[TMP85:%.*]] = insertelement <2 x i129> [[TMP42]], i129 [[TMP84]], i64 1 @@ -281,11 +281,11 @@ define <2 x i129> @srem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP18:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP13]], i1 true) ; CHECK-NEXT: [[TMP19:%.*]] = sub i129 [[TMP17]], [[TMP18]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp ugt i129 [[TMP19]], 128 -; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP16]], i1 true, i1 [[TMP20]] +; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP16]], i1 true, i1 [[TMP20]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i129 [[TMP19]], 128 ; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP21]], i129 0, i129 [[TMP13]] ; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP21]], i1 true, i1 [[TMP22]] -; CHECK-NEXT: br i1 [[TMP24]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]] +; CHECK-NEXT: br i1 [[TMP24]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit2: ; CHECK-NEXT: [[TMP25:%.*]] = phi i129 [ 0, [[UDIV_BB15]] ], [ [[TMP40:%.*]], [[UDIV_DO_WHILE3:%.*]] ] ; CHECK-NEXT: [[TMP26:%.*]] = phi i129 [ [[TMP49:%.*]], [[UDIV_BB15]] ], [ [[TMP37:%.*]], [[UDIV_DO_WHILE3]] ] @@ -319,7 +319,7 @@ define <2 x i129> @srem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP48:%.*]] = sub i129 128, [[TMP19]] ; CHECK-NEXT: [[TMP49]] = shl i129 [[TMP13]], [[TMP48]] ; CHECK-NEXT: [[TMP50:%.*]] = icmp eq i129 [[TMP47]], 0 -; CHECK-NEXT: br i1 [[TMP50]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]] +; CHECK-NEXT: br i1 [[TMP50]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]], !prof [[PROF1]] ; CHECK: udiv-end1: ; CHECK-NEXT: [[TMP51:%.*]] = phi i129 [ [[TMP28]], [[UDIV_LOOP_EXIT2]] ], [ [[TMP23]], [[_UDIV_SPECIAL_CASES_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP52:%.*]] = mul i129 [[TMP11]], [[TMP51]] @@ -348,11 +348,11 @@ define <2 x i129> @srem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP75:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP70]], i1 true) ; CHECK-NEXT: [[TMP76:%.*]] = sub i129 [[TMP74]], [[TMP75]] ; CHECK-NEXT: [[TMP77:%.*]] = icmp ugt i129 [[TMP76]], 128 -; CHECK-NEXT: [[TMP78:%.*]] = select i1 [[TMP73]], i1 true, i1 [[TMP77]] +; CHECK-NEXT: [[TMP78:%.*]] = select i1 [[TMP73]], i1 true, i1 [[TMP77]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP79:%.*]] = icmp eq i129 [[TMP76]], 128 ; CHECK-NEXT: [[TMP80:%.*]] = select i1 [[TMP78]], i129 0, i129 [[TMP70]] ; CHECK-NEXT: [[TMP81:%.*]] = select i1 [[TMP78]], i1 true, i1 [[TMP79]] -; CHECK-NEXT: br i1 [[TMP81]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: br i1 [[TMP81]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP82:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP97:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP83:%.*]] = phi i129 [ [[TMP106:%.*]], [[UDIV_BB1]] ], [ [[TMP94:%.*]], [[UDIV_DO_WHILE]] ] @@ -386,7 +386,7 @@ define <2 x i129> @srem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP105:%.*]] = sub i129 128, [[TMP76]] ; CHECK-NEXT: [[TMP106]] = shl i129 [[TMP70]], [[TMP105]] ; CHECK-NEXT: [[TMP107:%.*]] = icmp eq i129 [[TMP104]], 0 -; CHECK-NEXT: br i1 [[TMP107]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP107]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF1]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP108:%.*]] = phi i129 [ [[TMP85]], [[UDIV_LOOP_EXIT]] ], [ [[TMP80]], [[UDIV_END1]] ] ; CHECK-NEXT: [[TMP109:%.*]] = mul i129 [[TMP68]], [[TMP108]] @@ -417,11 +417,11 @@ define <2 x i129> @urem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP10:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP5]], i1 true) ; CHECK-NEXT: [[TMP11:%.*]] = sub i129 [[TMP9]], [[TMP10]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i129 [[TMP11]], 128 -; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP8]], i1 true, i1 [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP8]], i1 true, i1 [[TMP12]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i129 [[TMP11]], 128 ; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP13]], i129 0, i129 [[TMP5]] ; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP13]], i1 true, i1 [[TMP14]] -; CHECK-NEXT: br i1 [[TMP16]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]] +; CHECK-NEXT: br i1 [[TMP16]], label [[UDIV_END1:%.*]], label [[UDIV_BB15:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit2: ; CHECK-NEXT: [[TMP17:%.*]] = phi i129 [ 0, [[UDIV_BB15]] ], [ [[TMP32:%.*]], [[UDIV_DO_WHILE3:%.*]] ] ; CHECK-NEXT: [[TMP18:%.*]] = phi i129 [ [[TMP41:%.*]], [[UDIV_BB15]] ], [ [[TMP29:%.*]], [[UDIV_DO_WHILE3]] ] @@ -455,7 +455,7 @@ define <2 x i129> @urem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP40:%.*]] = sub i129 128, [[TMP11]] ; CHECK-NEXT: [[TMP41]] = shl i129 [[TMP5]], [[TMP40]] ; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i129 [[TMP39]], 0 -; CHECK-NEXT: br i1 [[TMP42]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]] +; CHECK-NEXT: br i1 [[TMP42]], label [[UDIV_LOOP_EXIT2]], label [[UDIV_PREHEADER4]], !prof [[PROF1]] ; CHECK: udiv-end1: ; CHECK-NEXT: [[TMP43:%.*]] = phi i129 [ [[TMP20]], [[UDIV_LOOP_EXIT2]] ], [ [[TMP15]], [[_UDIV_SPECIAL_CASES_UDIV_SPECIAL_CASES:%.*]] ] ; CHECK-NEXT: [[TMP44:%.*]] = mul i129 [[TMP3]], [[TMP43]] @@ -474,11 +474,11 @@ define <2 x i129> @urem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP57:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP52]], i1 true) ; CHECK-NEXT: [[TMP58:%.*]] = sub i129 [[TMP56]], [[TMP57]] ; CHECK-NEXT: [[TMP59:%.*]] = icmp ugt i129 [[TMP58]], 128 -; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP55]], i1 true, i1 [[TMP59]] +; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP55]], i1 true, i1 [[TMP59]], !prof [[PROF1]] ; CHECK-NEXT: [[TMP61:%.*]] = icmp eq i129 [[TMP58]], 128 ; CHECK-NEXT: [[TMP62:%.*]] = select i1 [[TMP60]], i129 0, i129 [[TMP52]] ; CHECK-NEXT: [[TMP63:%.*]] = select i1 [[TMP60]], i1 true, i1 [[TMP61]] -; CHECK-NEXT: br i1 [[TMP63]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]] +; CHECK-NEXT: br i1 [[TMP63]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]], !prof [[PROF1]] ; CHECK: udiv-loop-exit: ; CHECK-NEXT: [[TMP64:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP79:%.*]], [[UDIV_DO_WHILE:%.*]] ] ; CHECK-NEXT: [[TMP65:%.*]] = phi i129 [ [[TMP88:%.*]], [[UDIV_BB1]] ], [ [[TMP76:%.*]], [[UDIV_DO_WHILE]] ] @@ -512,7 +512,7 @@ define <2 x i129> @urem129(<2 x i129> %a, <2 x i129> %b) nounwind { ; CHECK-NEXT: [[TMP87:%.*]] = sub i129 128, [[TMP58]] ; CHECK-NEXT: [[TMP88]] = shl i129 [[TMP52]], [[TMP87]] ; CHECK-NEXT: [[TMP89:%.*]] = icmp eq i129 [[TMP86]], 0 -; CHECK-NEXT: br i1 [[TMP89]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]] +; CHECK-NEXT: br i1 [[TMP89]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]], !prof [[PROF1]] ; CHECK: udiv-end: ; CHECK-NEXT: [[TMP90:%.*]] = phi i129 [ [[TMP67]], [[UDIV_LOOP_EXIT]] ], [ [[TMP62]], [[UDIV_END1]] ] ; CHECK-NEXT: [[TMP91:%.*]] = mul i129 [[TMP50]], [[TMP90]] @@ -534,3 +534,10 @@ define @sdiv129_scalable( %a, %a, %b ret %res } + +!0 = !{!"function_entry_count", i64 1000} +;. +; CHECK: [[PROF0]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: [[PROF2]] = !{!"unknown", !"integer-division"} +;. diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt index 1646257a3fd67..7e897b84fd293 100644 --- a/llvm/utils/profcheck-xfail.txt +++ b/llvm/utils/profcheck-xfail.txt @@ -134,11 +134,6 @@ Transforms/CorrelatedValuePropagation/urem.ll Transforms/CrossDSOCFI/basic.ll Transforms/CrossDSOCFI/cfi_functions.ll Transforms/CrossDSOCFI/thumb.ll -Transforms/ExpandIRInsts/X86/sdiv129.ll -Transforms/ExpandIRInsts/X86/srem129.ll -Transforms/ExpandIRInsts/X86/udiv129.ll -Transforms/ExpandIRInsts/X86/urem129.ll -Transforms/ExpandIRInsts/X86/vector.ll Transforms/ExpandIRInsts/X86/expand-large-fp-convert-fptosi129.ll Transforms/ExpandIRInsts/X86/expand-large-fp-convert-fptoui129.ll Transforms/ExpandIRInsts/X86/expand-large-fp-convert-si129tofp.ll