1616#include " llvm/Transforms/Utils/IntegerDivision.h"
1717#include " llvm/IR/Function.h"
1818#include " llvm/IR/IRBuilder.h"
19+ #include " llvm/IR/Instruction.h"
1920#include " llvm/IR/Instructions.h"
2021#include " llvm/IR/Intrinsics.h"
22+ #include " llvm/IR/MDBuilder.h"
23+ #include " llvm/IR/ProfDataUtils.h"
24+ #include " llvm/Support/Casting.h"
25+ #include " llvm/Support/CommandLine.h"
2126
2227using namespace llvm ;
2328
2429#define DEBUG_TYPE " integer-division"
2530
31+ namespace llvm {
32+ extern cl::opt<bool > ProfcheckDisableMetadataFixes;
33+ }
34+
2635// / Generate code to compute the remainder of two signed integers. Returns the
2736// / remainder, which will have the sign of the dividend. Builder's insert point
2837// / should be pointing where the caller wants code generated, e.g. at the srem
@@ -235,11 +244,53 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
235244 Value *Tmp1 = Builder.CreateCall (CTLZ, {Dividend, True});
236245 Value *SR = Builder.CreateSub (Tmp0, Tmp1);
237246 Value *Ret0_4 = Builder.CreateICmpUGT (SR, MSB);
247+
248+ // Add 'unlikely' branch weights to the select instruction ('Ret0') generated
249+ // by 'CreateLogicalOr'. We assume the 'true' path, where either the divisor
250+ // or dividend is zero ('Ret0_3'), is less likely than the 'false' path,
251+ // which corresponds to the |divisor| > |dividend| ('Ret0_4').
238252 Value *Ret0 = Builder.CreateLogicalOr (Ret0_3, Ret0_4);
253+ if (!ProfcheckDisableMetadataFixes) {
254+ if (Instruction *Ret0SelectI = dyn_cast<Instruction>(Ret0)) {
255+ Ret0SelectI->setMetadata (
256+ LLVMContext::MD_prof,
257+ MDBuilder (Ret0SelectI->getContext ()).createUnlikelyBranchWeights ());
258+ }
259+ }
239260 Value *RetDividend = Builder.CreateICmpEQ (SR, MSB);
240- Value *RetVal = Builder.CreateSelect (Ret0, Zero, Dividend);
241- Value *EarlyRet = Builder.CreateLogicalOr (Ret0, RetDividend);
242- Builder.CreateCondBr (EarlyRet, End, BB1);
261+
262+ // This select instruction (RetVal) immediately follows the Ret0 check. Since
263+ // Ret0_4 determines if the divisor is greater then the dividend so the
264+ // unknown (50/50) branch weights are fairly accurate for RetVal.
265+ Value *RetVal;
266+ if (!ProfcheckDisableMetadataFixes) {
267+ RetVal = Builder.CreateSelectWithUnknownProfile (Ret0, Zero, Dividend,
268+ DEBUG_TYPE);
269+ } else {
270+ RetVal = Builder.CreateSelect (Ret0, Zero, Dividend);
271+ }
272+ // This select instruction (EarlyRet) is used to check another edge case, and
273+ // it share the same branch weights as RetVal so the unknown (50/50)branch
274+ // weights are also accurate for EarlyRet.
275+ Value *EarlyRet;
276+ if (!ProfcheckDisableMetadataFixes) {
277+ EarlyRet = Builder.CreateSelectWithUnknownProfile (
278+ Ret0, ConstantInt::getAllOnesValue (Ret0->getType ()), RetDividend,
279+ DEBUG_TYPE);
280+ } else {
281+ EarlyRet = Builder.CreateLogicalOr (Ret0, RetDividend);
282+ }
283+
284+ // The condition of this branch is based on `EarlyRet`. `EarlyRet` is true
285+ // only for special cases like dividend or divisor being zero, or the divisor
286+ // being greater than the dividend. Thus, the branch to `End` is unlikely,
287+ // and we expect to more frequently enter `BB1`.
288+ Instruction *ConBrSpecialCases = Builder.CreateCondBr (EarlyRet, End, BB1);
289+ if (!ProfcheckDisableMetadataFixes) {
290+ ConBrSpecialCases->setMetadata (LLVMContext::MD_prof,
291+ MDBuilder (ConBrSpecialCases->getContext ())
292+ .createUnlikelyBranchWeights ());
293+ }
243294
244295 // ; bb1: ; preds = %special-cases
245296 // ; %sr_1 = add i32 %sr, 1
@@ -252,7 +303,15 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
252303 Value *Tmp2 = Builder.CreateSub (MSB, SR);
253304 Value *Q = Builder.CreateShl (Dividend, Tmp2);
254305 Value *SkipLoop = Builder.CreateICmpEQ (SR_1, Zero);
255- Builder.CreateCondBr (SkipLoop, LoopExit, Preheader);
306+ // This branch is highly unlikely. SR_1 (SR + 1) is expected to be in [1, 129]
307+ // because the case where |divisor| > |dividend| (which would make SR
308+ // negative) has alredy been intercepted in the previous SpecialCases BB.
309+ Instruction *ConBrBB1 = Builder.CreateCondBr (SkipLoop, LoopExit, Preheader);
310+ if (!ProfcheckDisableMetadataFixes) {
311+ ConBrBB1->setMetadata (LLVMContext::MD_prof,
312+ MDBuilder (ConBrBB1->getContext ())
313+ .createUnlikelyBranchWeights ());
314+ }
256315
257316 // ; preheader: ; preds = %bb1
258317 // ; %tmp3 = lshr i32 %dividend, %sr_1
@@ -298,7 +357,13 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
298357 Value *R = Builder.CreateSub (Tmp7, Tmp11);
299358 Value *SR_2 = Builder.CreateAdd (SR_3, NegOne);
300359 Value *Tmp12 = Builder.CreateICmpEQ (SR_2, Zero);
301- Builder.CreateCondBr (Tmp12, LoopExit, DoWhile);
360+ // The loop implements the core bit-by-bit binary long division algorithm.
361+ // Since each iteration generates one bit of the quotient until covering all
362+ // significant bits so we would label it as unknown (50/50) branch weights.
363+ Instruction *ConBrDoWhile = Builder.CreateCondBr (Tmp12, LoopExit, DoWhile);
364+ if (!ProfcheckDisableMetadataFixes) {
365+ setExplicitlyUnknownBranchWeightsIfProfiled (*ConBrDoWhile, DEBUG_TYPE, F);
366+ }
302367
303368 // ; loop-exit: ; preds = %do-while, %bb1
304369 // ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
0 commit comments