|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
References
lib/CodeGen/SelectionDAG/DAGCombiner.cpp 220 ForCodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
687 SelectionDAG &getDAG() const { return DAG; }
693 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
705 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
705 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
790 DAG.DeleteNode(N);
922 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
923 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
928 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
949 if (SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
950 if (SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
952 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, C1, C2))
953 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
959 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
962 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
993 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
994 To[0].getNode()->dump(&DAG);
1002 DAG.ReplaceAllUsesWith(N, To);
1026 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1043 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1053 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
1054 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
1067 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1078 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
1079 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
1089 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1091 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
1092 Trunc.getNode()->dump(&DAG); dbgs() << '\n');
1094 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1095 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1109 return DAG.getExtLoad(ExtType, DL, PVT,
1119 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1123 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1128 return DAG.getNode(ExtOpc, DL, PVT, Op);
1134 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1150 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1151 DAG.getValueType(OldVT));
1165 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1191 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1203 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1203 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1256 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1273 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1273 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1307 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1308 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1342 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1345 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1347 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
1348 Result.getNode()->dump(&DAG); dbgs() << '\n');
1350 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1351 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1381 DAG.DeleteNode(N);
1402 for (SDNode &Node : DAG.allnodes())
1408 HandleSDNode Dummy(DAG.getRoot());
1424 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1434 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
1462 LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG));
1465 DAG.ReplaceAllUsesWith(N, RV.getNode());
1469 DAG.ReplaceAllUsesWith(N, &RV);
1484 DAG.setRoot(Dummy.getValue());
1485 DAG.RemoveDeadNodes();
1635 DagCombineInfo(DAG, Level, false, this);
1680 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1870 Result = DAG.getEntryNode();
1879 Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
1881 Result = DAG.getTokenFactor(SDLoc(N), Ops);
1903 DAG.ReplaceAllUsesWith(N, Ops.data());
1975 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
1976 : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
1982 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
1983 : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
1989 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
2099 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2101 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2102 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2104 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
2116 SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N1.getNode(),
2119 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2125 SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N1.getNode(),
2128 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2142 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2143 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2151 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2152 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2153 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2167 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2171 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2184 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2190 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2196 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2202 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2209 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2220 return DAG.getNode(ISD::SUB, DL, VT,
2221 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2222 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2233 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
2243 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2243 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2261 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
2270 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2271 DAG.getAllOnesConstant(DL, VT));
2272 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
2279 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2280 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
2301 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2304 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2309 DAG.haveNoCommonBitsSet(N0, N1))
2310 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2335 return DAG.getAllOnesConstant(DL, VT);
2337 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2339 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2340 return DAG.getNode(Opcode, DL, VT, N1, N0);
2342 return DAG.FoldConstantArithmetic(Opcode, DL, VT, N0.getNode(),
2352 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2353 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
2426 return DAG.getNode(ISD::SUB, DL, VT, N0,
2427 DAG.getNode(ISD::SHL, DL, VT,
2431 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2440 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2441 DAG.getAllOnesConstant(DL, VT));
2442 return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
2450 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2451 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2457 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2458 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
2467 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2468 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2475 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2476 DAG.getConstant(1, DL, VT));
2477 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2484 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2490 return DAG.getNode(ISD::ADDCARRY, DL,
2491 DAG.getVTList(VT, Carry.getValueType()), N0,
2492 DAG.getConstant(0, DL, VT), Carry);
2505 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2506 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2512 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2516 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2520 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2521 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2522 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2598 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2599 DAG.getUNDEF(CarryVT));
2602 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2603 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2604 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
2608 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2612 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2613 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2614 DAG.getConstant(0, DL, CarryVT));
2618 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2619 DAG.getConstant(0, DL, VT), N0.getOperand(0));
2621 flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2643 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2644 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2645 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2652 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2653 DAG.getConstant(0, SDLoc(N), VT), Carry);
2667 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2672 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2687 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2693 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2700 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2702 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2703 DAG.getConstant(1, DL, VT)),
2704 DAG.getConstant(0, DL, CarryVT));
2807 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
2809 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
2812 flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2823 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2833 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
2835 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
2872 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
2873 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2874 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2876 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2887 return DAG.getNode(ISD::ADD, DL, VT, N0,
2888 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2902 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2910 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2923 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2927 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
2945 SDValue NewC = DAG.FoldConstantArithmetic(
2948 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
2956 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2959 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2967 SDValue NewC = DAG.FoldConstantArithmetic(
2970 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
2977 SDValue NewC = DAG.FoldConstantArithmetic(
2980 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
2988 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2994 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
3000 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
3005 return DAG.getNode(ISD::ADD, DL, VT, N0,
3006 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
3013 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3016 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3020 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3023 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3033 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3036 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3039 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3044 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3045 DAG.getAllOnesConstant(DL, VT));
3046 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
3054 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3055 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3055 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3062 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3063 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
3068 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3069 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
3075 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3076 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
3081 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3082 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
3092 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3093 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
3105 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
3115 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
3121 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3129 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3130 DAG.getConstant(1, DL, VT));
3131 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
3142 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3143 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
3151 SDValue Zero = DAG.getConstant(0, DL, VT);
3152 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3153 return DAG.getNode(ISD::ADDCARRY, DL,
3154 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
3179 return DAG.getConstant(0, DL, VT);
3183 return DAG.getConstant(0, DL, VT);
3185 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3186 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
3188 return DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, N0.getNode(),
3207 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3208 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3212 return CombineTo(N, DAG.getConstant(0, DL, VT),
3213 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3217 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3221 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3222 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3238 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3239 DAG.getUNDEF(CarryVT));
3243 return CombineTo(N, DAG.getConstant(0, DL, VT),
3244 DAG.getConstant(0, DL, CarryVT));
3250 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
3251 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3256 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3260 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3261 DAG.getConstant(0, DL, CarryVT));
3273 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
3287 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
3303 return DAG.getConstant(0, SDLoc(N), VT);
3306 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3307 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3308 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
3312 return DAG.getConstant(0, SDLoc(N), VT);
3324 return DAG.getConstant(0, SDLoc(N), VT);
3359 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
3363 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3364 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3365 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
3379 return DAG.getNode(ISD::SUB, DL, VT,
3380 DAG.getConstant(0, DL, VT), N0);
3384 DAG.isKnownToBeAPowerOfTwo(N1) &&
3389 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3390 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
3398 return DAG.getNode(ISD::SUB, DL, VT,
3399 DAG.getConstant(0, DL, VT),
3400 DAG.getNode(ISD::SHL, DL, VT, N0,
3401 DAG.getConstant(Log2Val, DL,
3412 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
3430 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
3430 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
3431 SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
3433 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
3433 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
3442 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3444 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
3464 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3465 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
3470 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
3472 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
3474 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
3475 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
3477 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
3559 SDVTList VTs = DAG.getVTList(VT, VT);
3560 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
3640 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
3643 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
3643 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
3646 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3646 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3647 DAG.getConstant(1, DL, VT),
3648 DAG.getConstant(0, DL, VT));
3650 if (SDValue V = simplifyDivRem(N, DAG))
3658 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3658 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3659 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
3664 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
3666 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3667 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3678 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3715 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3716 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3717 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
3718 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3723 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3724 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
3728 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3730 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3732 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3737 SDValue One = DAG.getConstant(1, DL, VT);
3738 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3739 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3740 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3741 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3742 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
3746 SDValue Zero = DAG.getConstant(0, DL, VT);
3747 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3750 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3751 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3758 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3784 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
3789 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3789 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3790 DAG.getConstant(1, DL, VT),
3791 DAG.getConstant(0, DL, VT));
3793 if (SDValue V = simplifyDivRem(N, DAG))
3802 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
3804 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3805 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3816 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3830 DAG.isKnownToBeAPowerOfTwo(N1)) {
3835 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3837 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
3844 DAG.isKnownToBeAPowerOfTwo(N10)) {
3849 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3851 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3853 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3858 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3882 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3886 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3886 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3887 DAG.getConstant(0, DL, VT), N0);
3889 if (SDValue V = simplifyDivRem(N, DAG))
3898 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3898 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3899 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3901 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3902 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3904 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3906 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3909 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3911 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3913 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3917 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3926 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
3932 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
3935 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3936 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3961 return DAG.getConstant(0, DL, VT);
3969 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3970 DAG.getConstant(N0.getScalarValueSizeInBits() - 1, DL,
3975 return DAG.getConstant(0, DL, VT);
3982 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3984 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3985 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3986 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3987 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3988 DAG.getConstant(SimpleSize, DL,
3990 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4008 return DAG.getConstant(0, DL, VT);
4016 return DAG.getConstant(0, DL, N0.getValueType());
4019 return DAG.getConstant(0, DL, VT);
4023 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
4026 SDValue SRLAmt = DAG.getNode(
4027 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
4029 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4030 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4038 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4040 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4041 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4042 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4043 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4044 DAG.getConstant(SimpleSize, DL,
4046 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4062 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4070 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4080 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4090 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4114 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4116 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4117 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4118 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4120 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4121 DAG.getConstant(SimpleSize, DL,
4123 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4125 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4142 SDValue Zero = DAG.getConstant(0, DL, VT);
4148 SDValue Zero = DAG.getConstant(0, DL, VT);
4157 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4159 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4160 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4161 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4163 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4164 DAG.getConstant(SimpleSize, DL,
4166 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4168 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4186 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4187 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4188 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
4192 return CombineTo(N, DAG.getConstant(0, DL, VT),
4193 DAG.getConstant(0, DL, CarryVT));
4198 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
4218 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
4221 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4222 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4223 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
4228 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4230 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
4231 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
4241 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
4290 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4291 return DAG.getNode(HandOpcode, DL, VT, Logic);
4312 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4313 return DAG.getNode(HandOpcode, DL, VT, Logic);
4324 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4325 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
4333 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4334 return DAG.getNode(HandOpcode, DL, VT, Logic);
4350 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4351 return DAG.getNode(HandOpcode, DL, VT, Logic);
4385 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4389 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4391 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
4398 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4402 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4404 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
4457 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4459 return DAG.getSetCC(DL, VT, Or, LR, CC1);
4476 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4478 return DAG.getSetCC(DL, VT, And, LR, CC1);
4488 SDValue One = DAG.getConstant(1, DL, OpVT);
4489 SDValue Two = DAG.getConstant(2, DL, OpVT);
4490 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4492 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
4502 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4503 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4504 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4505 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4506 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
4526 SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4527 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4528 SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4529 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4530 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4531 return DAG.getSetCC(DL, VT, And, Zero, CC0);
4552 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
4568 return DAG.getConstant(0, DL, VT);
4588 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
4593 DAG.getNode(ISD::ADD, DL0, VT,
4594 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
4620 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
4640 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
4641 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4644 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4645 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4646 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4647 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4648 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
4664 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4718 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
4718 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
4815 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4974 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
4976 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5051 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
5055 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
5069 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
5071 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5072 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5073 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
5079 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
5081 return DAG.getConstant(0, SDLoc(N), VT);
5107 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
5108 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5206 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
5236 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
5246 if (BackwardsPropagateMask(N, DAG)) {
5274 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
5295 if (DAG.MaskedValueIsZero(N1, ExtBits) &&
5299 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
5318 if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5422 !DAG.MaskedValueIsZero(
5427 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5430 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
5431 DAG.getConstant(OpSizeInBits - 16, DL,
5587 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
5592 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
5594 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
5596 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
5597 return DAG.getNode(ISD::OR, DL, VT,
5598 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
5599 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
5610 return DAG.getAllOnesConstant(DL, VT);
5630 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
5631 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
5632 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5634 return DAG.getNode(ISD::AND, DL, VT, X,
5635 DAG.getConstant(LHSMask | RHSMask, DL, VT));
5647 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5649 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
5695 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
5698 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
5756 Mask, DAG);
5768 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
5770 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5771 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5772 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
5784 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
5807 if (SDValue COR = DAG.FoldConstantArithmetic(
5809 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
5811 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
5815 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
5817 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
5838 DAG.haveNoCommonBitsSet(N0, N1))
6124 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
6126 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
6153 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
6160 matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
6164 matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
6181 extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
6186 extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
6221 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
6226 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
6230 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
6231 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
6232 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
6235 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
6236 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
6237 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
6240 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
6490 *DAG.getContext(), Width * N->getMemoryVT().getSizeInBits());
6494 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6551 BaseIndexOffset Ptr = BaseIndexOffset::match(Store, DAG);
6555 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6583 bool NeedsBswap = DAG.getDataLayout().isBigEndian() != *IsBigEndian;
6594 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
6594 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
6602 CombinedValue = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
6607 CombinedValue = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, CombinedValue);
6610 DAG.getStore(Chain, SDLoc(N), CombinedValue, FirstStore->getBasePtr(),
6614 DAG.ReplaceAllUsesWith(N, NewStore.getNode());
6658 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6665 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
6706 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
6710 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6758 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
6758 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
6764 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
6769 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
6771 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
6842 SDValue NotX = DAG.getNOT(DL, X, VT);
6843 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
6844 SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
6845 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
6846 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
6849 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
6850 SDValue NotM = DAG.getNOT(DL, M, VT);
6851 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
6853 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
6876 return DAG.getConstant(0, DL, VT);
6886 return DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, N0C, N1C);
6888 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
6889 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
6890 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
6914 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
6916 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
6927 V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V,
6928 DAG.getConstant(1, DL0, V.getValueType()));
6930 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V);
6939 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
6940 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
6942 return DAG.getNode(NewOpcode, DL, VT, N00, N01);
6951 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
6952 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
6954 return DAG.getNode(NewOpcode, DL, VT, N00, N01);
6963 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1),
6964 DAG.getAllOnesConstant(DL, VT));
6970 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
6972 return DAG.getNode(ISD::AND, DL, VT, NotX, N1);
6990 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
6991 return DAG.getNode(N0Opcode, DL, VT, Not, N0.getOperand(1));
7008 return DAG.getNode(ISD::ABS, DL, VT, S0);
7015 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
7037 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
7037 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
7146 if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
7191 SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1),
7195 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
7197 return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS);
7212 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
7213 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
7216 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
7237 if (DAG.MaskedValueIsZero(N1, ModuloMask))
7246 return DAG.getNode(N->getOpcode(), dl, VT, N0,
7247 DAG.getConstant(RotAmt, dl, N1.getValueType()));
7255 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
7261 SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
7262 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
7268 DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
7269 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
7270 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
7273 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
7284 if (SDValue V = DAG.simplifyShift(N0, N1))
7308 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
7310 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
7322 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
7328 if (DAG.MaskedValueIsZero(SDValue(N, 0),
7330 return DAG.getConstant(0, SDLoc(N), VT);
7336 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
7353 return DAG.getConstant(0, SDLoc(N), VT);
7364 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7365 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
7394 return DAG.getConstant(0, SDLoc(N), VT);
7408 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
7409 SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
7410 Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1);
7411 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum);
7434 SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
7435 NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL);
7437 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
7451 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
7452 DAG.getConstant(C2 - C1, DL, ShiftVT));
7453 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
7454 DAG.getConstant(C1 - C2, DL, ShiftVT));
7475 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
7476 DAG.getConstant(c2 - c1, DL, ShiftVT));
7480 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
7481 DAG.getConstant(c1 - c2, DL, ShiftVT));
7484 return DAG.getNode(ISD::AND, DL, VT, Shift,
7485 DAG.getConstant(Mask, DL, VT));
7494 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
7495 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
7496 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
7508 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
7509 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7512 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
7519 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7521 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
7534 if (SDValue V = DAG.simplifyShift(N0, N1))
7543 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
7557 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
7566 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
7568 ExtVT = EVT::getVectorVT(*DAG.getContext(),
7572 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
7573 N0.getOperand(0), DAG.getValueType(ExtVT));
7591 ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
7597 ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
7600 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
7613 LLVMContext &Ctx = *DAG.getContext();
7632 SDValue Amt = DAG.getConstant(ShiftAmt, DL,
7634 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
7636 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
7638 return DAG.getNode(ISD::SIGN_EXTEND, DL,
7654 LLVMContext &Ctx = *DAG.getContext();
7666 SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
7667 SDValue ShiftC = DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).
7669 SDValue Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
7670 return DAG.getSExtOrTrunc(Add, DL, VT);
7679 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
7697 SDValue Amt = DAG.getConstant(N1C->getZExtValue() + TruncBits, DL,
7700 DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt);
7701 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
7712 if (DAG.SignBitIsZero(N0))
7713 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
7725 if (SDValue V = DAG.simplifyShift(N0, N1))
7742 return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
7748 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
7750 return DAG.getConstant(0, SDLoc(N), VT);
7762 return DAG.getConstant(0, SDLoc(N), VT);
7774 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7775 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
7793 return DAG.getConstant(0, DL, VT);
7794 return DAG.getNode(ISD::TRUNCATE, DL, VT,
7795 DAG.getNode(ISD::SRL, DL, InnerShiftVT,
7797 DAG.getConstant(c1 + c2, DL,
7809 DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
7809 DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
7811 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
7821 return DAG.getUNDEF(VT);
7826 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
7828 DAG.getConstant(ShiftAmt, DL0,
7833 return DAG.getNode(ISD::AND, DL, VT,
7834 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
7835 DAG.getConstant(Mask, DL, VT));
7843 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
7849 KnownBits Known = DAG.computeKnownBits(N0.getOperand(0));
7853 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
7858 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
7871 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
7872 DAG.getConstant(ShAmt, DL,
7878 return DAG.getNode(ISD::XOR, DL, VT,
7879 Op, DAG.getConstant(1, DL, VT));
7887 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
7947 if (DAG.MaskedValueIsZero(
7962 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1,
7963 DAG.getConstant(RotAmt, SDLoc(N), ShAmtTy));
7975 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1,
7976 DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt,
7979 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
7980 DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt,
7990 if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
7991 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2);
7992 if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
7993 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2);
8002 return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2);
8016 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8017 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
8022 if (DAG.SignBitIsZero(N0))
8032 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8033 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
8045 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8046 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
8058 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8059 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
8063 if (DAG.isKnownNeverZero(N0))
8064 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8075 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8076 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8085 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8086 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
8090 if (DAG.isKnownNeverZero(N0))
8091 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8102 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8103 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8112 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8113 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
8238 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8240 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
8245 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8247 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
8253 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8259 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
8273 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8274 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
8279 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
8280 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
8286 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8287 SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT);
8288 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC);
8291 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8314 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
8314 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
8317 return DAG.getZExtOrTrunc(NotCond, DL, VT);
8332 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8338 return DAG.getNode(ISD::OR, DL, VT, N0, N2);
8345 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8347 return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
8351 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8353 return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
8358 return DAG.getNode(ISD::AND, DL, VT, N0, N1);
8374 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
8381 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags);
8383 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
8393 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
8396 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
8411 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
8412 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1,
8417 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
8430 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
8431 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1,
8436 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
8443 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
8444 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1);
8458 if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, TLI))
8460 CC, TLI, DAG))
8486 SDVTList VTs = DAG.getVTList(VT, VT0);
8487 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
8488 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0));
8498 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1,
8655 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
8656 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
8663 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
8664 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
8665 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC);
8668 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8685 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8689 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
8690 return DAG.getSelect(DL, VT, F, N2, N1);
8713 return DAG.getNode(ISD::ABS, DL, VT, LHS);
8715 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS,
8716 DAG.getConstant(VT.getScalarSizeInBits() - 1,
8718 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
8721 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
8730 if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, TLI)) {
8732 combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC, TLI, DAG))
8758 SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
8759 SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
8761 SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
8762 return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
8783 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
8821 SDValue SelectOp = DAG.getNode(
8875 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
9046 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
9050 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
9104 SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
9105 SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
9125 SDValue SplitLoad = DAG.getExtLoad(
9130 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
9131 DAG.getConstant(Stride, DL, BasePtr.getValueType()));
9137 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
9138 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
9148 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
9202 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
9207 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
9213 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
9214 DAG.getConstant(Mask, DL0, VT));
9219 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
9221 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
9268 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
9269 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
9271 CastA = DAG.getNode(CastOpcode, DL, VT, A);
9272 CastB = DAG.getNode(CastOpcode, DL, VT, B);
9274 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
9415 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9421 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
9442 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
9453 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
9458 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
9465 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
9467 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
9468 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
9469 DAG.getValueType(N0.getValueType()));
9475 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
9480 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::SEXTLOAD,
9491 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
9509 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
9515 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
9516 ExtLoad, DAG.getConstant(Mask, DL, VT));
9524 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
9528 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
9530 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
9539 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
9566 return DAG.getSetCC(DL, VT, N00, N01, CC);
9573 SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
9574 return DAG.getSExtOrTrunc(VsetCC, DL, VT);
9592 ? DAG.getAllOnesConstant(DL, VT)
9593 : DAG.getBoolConstant(true, DL, VT, N00VT);
9594 SDValue Zero = DAG.getConstant(0, DL, VT);
9607 SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
9608 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
9615 DAG.SignBitIsZero(N0))
9616 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
9627 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
9628 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext);
9628 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext);
9636 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
9637 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
9637 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
9703 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9709 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
9717 if (isTruncateOf(DAG, N0, Op, Known)) {
9726 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
9752 Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
9754 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
9756 DAG.transferDbgValues(N0, ZExtOrTrunc);
9762 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
9764 SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
9767 DAG.transferDbgValues(N0, And);
9781 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
9785 return DAG.getNode(ISD::AND, DL, VT,
9786 X, DAG.getConstant(Mask, DL, VT));
9791 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
9796 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::ZEXTLOAD,
9833 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
9840 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
9841 ExtLoad, DAG.getConstant(Mask, DL, VT));
9849 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
9853 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
9855 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
9871 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
9874 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
9891 SDValue VecOnes = DAG.getConstant(1, DL, VT);
9894 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
9896 return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes);
9904 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
9906 return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
9906 return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
9913 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
9914 DAG.getConstant(0, DL, VT),
9939 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
9941 return DAG.getNode(N0.getOpcode(), DL, VT,
9942 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
9949 if (SDValue NewCtPop = widenCtPop(N, DAG))
9959 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9968 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
9986 return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
9997 X = DAG.getAnyExtOrTrunc(X, DL, VT);
10000 return DAG.getNode(ISD::AND, DL, VT,
10001 X, DAG.getConstant(Mask, DL, VT));
10018 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
10027 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10030 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
10047 SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
10051 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10074 return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
10083 DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
10086 return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT);
10092 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
10093 DAG.getConstant(0, DL, VT),
10098 if (SDValue NewCtPop = widenCtPop(N, DAG))
10130 SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
10131 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10133 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
10150 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10152 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
10199 ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShiftAmt);
10201 ExtVT = EVT::getIntegerVT(*DAG.getContext(),
10222 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
10263 EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(),
10304 if (DAG.getDataLayout().isBigEndian())
10314 SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
10316 DAG.getConstant(PtrOff, DL, PtrType),
10322 Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr,
10326 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
10333 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
10346 Result = DAG.getConstant(0, DL, VT);
10348 Result = DAG.getNode(ISD::SHL, DL, VT,
10349 Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
10355 if (DAG.getDataLayout().isBigEndian())
10362 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
10363 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC);
10364 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
10380 return DAG.getUNDEF(VT);
10383 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
10384 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
10387 if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
10393 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
10404 (N00Bits - DAG.ComputeNumSignBits(N00)) < EVTBits) &&
10406 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00);
10416 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
10426 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
10430 if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
10431 return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT.getScalarType());
10451 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
10453 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0),
10469 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10485 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10498 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
10510 return DAG.getUNDEF(VT);
10512 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10526 return DAG.getUNDEF(VT);
10528 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10541 bool isLE = DAG.getDataLayout().isLittleEndian();
10549 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
10552 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
10553 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
10564 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
10567 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
10596 EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
10602 EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
10606 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
10607 DAG.getBitcast(NVT, N0.getOperand(0)),
10608 DAG.getConstant(Index, DL, IndexTy));
10618 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10619 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
10620 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
10629 KnownBits Known = DAG.computeKnownBits(Amt);
10633 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
10635 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10637 Amt = DAG.getZExtOrTrunc(Amt, SL, AmtVT);
10640 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt);
10651 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op);
10654 return DAG.getBuildVector(VT, DL, TruncOps);
10683 return DAG.getBuildVector(VT, SDLoc(N), Opnds);
10695 if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
10696 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
10711 SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
10715 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
10739 VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
10745 return DAG.getUNDEF(VT);
10752 Opnds.push_back(DAG.getUNDEF(VTs[i]));
10755 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
10759 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
10775 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
10777 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc,
10778 DAG.getConstant(Idx, SL, IdxVT));
10796 auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10797 auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10798 auto VTs = DAG.getVTList(VT, N0->getValueType(1));
10799 return DAG.getNode(N0.getOpcode(), SL, VTs, X, Y, N0.getOperand(2));
10813 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
10840 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
10841 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
10842 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR);
10868 if (DAG.getDataLayout().isBigEndian())
10877 DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
10879 unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
10880 VT.getTypeForEVT(*DAG.getContext()));
10884 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
10955 return DAG.getUNDEF(VT);
10982 SDValue C = DAG.getBitcast(VT, N0);
10990 return DAG.getBitcast(VT, N0.getOperand(0));
10996 TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) ==
10997 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
11007 if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG,
11010 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
11013 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
11018 if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
11037 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
11043 SDValue SignBit = DAG.getConstant(
11052 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
11053 DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11053 DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11056 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
11060 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
11062 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
11066 return DAG.getNode(ISD::XOR, DL, VT,
11067 NewConv, DAG.getConstant(SignBit, DL, VT));
11069 return DAG.getNode(ISD::AND, DL, VT,
11070 NewConv, DAG.getConstant(~SignBit, DL, VT));
11088 EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
11090 SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
11096 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
11102 X = DAG.getNode(ISD::SRL, DL,
11104 DAG.getConstant(OrigXWidth-VTWidth, DL,
11107 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
11113 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11115 SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
11117 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
11119 SDValue XorResult64 = DAG.getNode(
11121 DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11121 DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11125 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
11126 DAG.getConstant(SignBit, SDLoc(XorResult64), MVT::i64));
11129 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
11131 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
11134 X = DAG.getNode(ISD::AND, SDLoc(X), VT,
11135 X, DAG.getConstant(SignBit, SDLoc(X), VT));
11138 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11139 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
11140 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
11143 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
11170 return DAG.getBitcast(VT, Op);
11190 TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG);
11223 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
11224 Ops.push_back(DAG.getBitcast(DstEltVT, Op));
11227 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
11229 return DAG.getBuildVector(VT, SDLoc(BV), Ops);
11238 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
11246 EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
11264 bool isLE = DAG.getDataLayout().isLittleEndian();
11279 Ops.push_back(DAG.getUNDEF(DstEltVT));
11281 Ops.push_back(DAG.getConstant(NewBits, DL, DstEltVT));
11284 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
11285 return DAG.getBuildVector(VT, DL, Ops);
11291 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
11297 Ops.append(NumOutputsPerInput, DAG.getUNDEF(DstEltVT));
11306 Ops.push_back(DAG.getConstant(ThisVal, DL, DstEltVT));
11311 if (DAG.getDataLayout().isBigEndian())
11315 return DAG.getBuildVector(VT, DL, Ops);
11330 const TargetOptions &Options = DAG.getTarget().Options;
11352 const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
11376 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11383 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11394 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11395 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11397 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11408 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11409 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11411 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11423 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11425 DAG.getNode(PreferredFusedOpcode, SL, VT,
11436 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11438 DAG.getNode(PreferredFusedOpcode, SL, VT,
11450 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
11451 DAG.getNode(PreferredFusedOpcode, SL, VT,
11452 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
11453 DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
11477 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11478 DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
11479 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
11480 DAG.getNode(PreferredFusedOpcode, SL, VT,
11481 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
11482 DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
11542 const TargetOptions &Options = DAG.getTarget().Options;
11564 const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
11582 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11584 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11590 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11591 DAG.getNode(ISD::FNEG, SL, VT,
11601 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11602 DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
11603 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11614 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11615 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11617 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11619 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11630 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11631 DAG.getNode(ISD::FNEG, SL, VT,
11632 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11634 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11652 return DAG.getNode(ISD::FNEG, SL, VT,
11653 DAG.getNode(PreferredFusedOpcode, SL, VT,
11654 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11656 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11675 return DAG.getNode(ISD::FNEG, SL, VT,
11676 DAG.getNode(PreferredFusedOpcode, SL, VT,
11677 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11679 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11693 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11695 DAG.getNode(PreferredFusedOpcode, SL, VT,
11698 DAG.getNode(ISD::FNEG, SL, VT,
11708 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11709 DAG.getNode(ISD::FNEG, SL, VT,
11712 DAG.getNode(PreferredFusedOpcode, SL, VT,
11713 DAG.getNode(ISD::FNEG, SL, VT, N20),
11726 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11728 DAG.getNode(PreferredFusedOpcode, SL, VT,
11729 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11731 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11733 DAG.getNode(ISD::FNEG, SL, VT,
11751 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11752 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11754 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11756 DAG.getNode(PreferredFusedOpcode, SL, VT,
11757 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11759 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11761 DAG.getNode(ISD::FNEG, SL, VT,
11776 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11777 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
11779 DAG.getNode(PreferredFusedOpcode, SL, VT,
11780 DAG.getNode(ISD::FNEG, SL, VT,
11781 DAG.getNode(ISD::FP_EXTEND, SL,
11783 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11805 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11806 DAG.getNode(ISD::FNEG, SL, VT,
11807 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11809 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
11810 DAG.getNode(PreferredFusedOpcode, SL, VT,
11811 DAG.getNode(ISD::FNEG, SL, VT,
11812 DAG.getNode(ISD::FP_EXTEND, SL,
11814 DAG.getNode(ISD::FP_EXTEND, SL, VT,
11836 const TargetOptions &Options = DAG.getTarget().Options;
11868 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
11871 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
11872 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
11891 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11892 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
11895 return DAG.getNode(PreferredFusedOpcode, SL, VT,
11896 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
11897 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
11901 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
11902 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
11904 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
11926 const TargetOptions &Options = DAG.getTarget().Options;
11936 return DAG.getNode(ISD::FADD, DL, VT, N0, N1, Flags);
11940 return DAG.getNode(ISD::FADD, DL, VT, N1, N0, Flags);
11953 TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) == 2)
11954 return DAG.getNode(
11956 TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
11960 TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) == 2)
11961 return DAG.getNode(
11963 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize), Flags);
11975 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
11976 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add, Flags);
11981 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
11982 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add, Flags);
11993 return DAG.getConstantFP(0.0, DL, VT);
11997 return DAG.getConstantFP(0.0, DL, VT);
12009 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1, Flags);
12010 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC, Flags);
12023 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12024 DAG.getConstantFP(1.0, DL, VT), Flags);
12025 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP, Flags);
12032 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12033 DAG.getConstantFP(2.0, DL, VT), Flags);
12034 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP, Flags);
12044 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12045 DAG.getConstantFP(1.0, DL, VT), Flags);
12046 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP, Flags);
12053 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12054 DAG.getConstantFP(2.0, DL, VT), Flags);
12055 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP, Flags);
12064 return DAG.getNode(ISD::FMUL, DL, VT,
12065 N1, DAG.getConstantFP(3.0, DL, VT), Flags);
12074 return DAG.getNode(ISD::FMUL, DL, VT,
12075 N0, DAG.getConstantFP(3.0, DL, VT), Flags);
12084 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0),
12085 DAG.getConstantFP(4.0, DL, VT), Flags);
12105 const TargetOptions &Options = DAG.getTarget().Options;
12115 return DAG.getNode(ISD::FSUB, DL, VT, N0, N1, Flags);
12131 return DAG.getConstantFP(0.0f, DL, VT);
12142 if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
12143 return TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12145 return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags);
12154 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1), Flags);
12157 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0), Flags);
12161 if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
12162 return DAG.getNode(
12164 TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
12179 TLI.isNegatibleForFree(X, DAG, LegalOperations, ForCodeSize))
12181 TLI.isNegatibleForFree(Y, DAG, LegalOperations, ForCodeSize))
12197 const TargetOptions &Options = DAG.getTarget().Options;
12209 return DAG.getNode(ISD::FMUL, DL, VT, N0, N1, Flags);
12214 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0, Flags);
12236 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags);
12237 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts, Flags);
12245 const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
12246 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags);
12247 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts, Flags);
12253 return DAG.getNode(ISD::FADD, DL, VT, N0, N0, Flags);
12258 return DAG.getNode(ISD::FNEG, DL, VT, N0);
12263 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
12265 TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12266 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1, Flags);
12305 return DAG.getNode(ISD::FNEG, DL, VT,
12306 DAG.getNode(ISD::FABS, DL, VT, X));
12308 return DAG.getNode(ISD::FABS, DL, VT, X);
12332 const TargetOptions &Options = DAG.getTarget().Options;
12342 return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2);
12348 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
12350 TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12351 return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2, Flags);
12362 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2);
12364 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2);
12369 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2);
12376 return DAG.getNode(ISD::FMUL, DL, VT, N0,
12377 DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1),
12385 return DAG.getNode(ISD::FMA, DL, VT,
12387 DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1),
12398 return DAG.getNode(ISD::FADD, DL, VT, N0, N2);
12402 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0);
12405 return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg);
12413 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0),
12414 DAG.getNode(ISD::FNEG, DL, VT, N1, Flags), N2);
12421 return DAG.getNode(ISD::FMUL, DL, VT, N0,
12422 DAG.getNode(ISD::FADD, DL, VT, N1,
12423 DAG.getConstantFP(1.0, DL, VT), Flags),
12429 return DAG.getNode(ISD::FMUL, DL, VT, N0,
12430 DAG.getNode(ISD::FADD, DL, VT, N1,
12431 DAG.getConstantFP(-1.0, DL, VT), Flags),
12450 bool UnsafeMath = DAG.getTarget().Options.UnsafeFPMath;
12470 if (VT.isVector() && DAG.isSplatValue(N1))
12494 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
12495 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
12501 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
12520 const TargetOptions &Options = DAG.getTarget().Options;
12530 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1, Flags);
12554 return DAG.getNode(ISD::FMUL, DL, VT, N0,
12555 DAG.getConstantFP(Recip, DL, VT), Flags);
12562 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12567 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV);
12569 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12575 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1));
12577 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12595 RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp, Flags);
12597 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12609 return DAG.getNode(
12611 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize),
12612 TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
12626 return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1, N->getFlags());
12636 if (!DAG.getTarget().Options.UnsafeFPMath &&
12641 if (TLI.isFsqrtCheap(N0, DAG))
12673 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1);
12681 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
12684 return DAG.getNode(ISD::FNEG, SDLoc(N), VT,
12685 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0));
12694 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1);
12698 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
12702 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1));
12707 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0));
12737 if (!DAG.getLibInfo().has(LibFunc_cbrt) ||
12738 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) &&
12739 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT)))
12742 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0), Flags);
12767 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT))
12772 if (DAG.getMachineFunction().getFunction().hasOptSize())
12777 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags);
12778 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
12782 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt, Flags);
12830 return DAG.getConstantFP(0.0, SDLoc(N), VT);
12833 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
12837 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
12844 if (DAG.SignBitIsZero(N0))
12845 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
12858 DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12858 DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12860 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
12872 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12872 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12874 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
12878 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
12891 return DAG.getConstantFP(0.0, SDLoc(N), VT);
12894 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
12898 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
12905 if (DAG.SignBitIsZero(N0))
12906 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
12918 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12918 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
12920 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
12924 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
12978 return DAG.getUNDEF(VT);
12982 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0);
12984 return FoldIntToFPToInt(N, DAG);
12993 return DAG.getUNDEF(VT);
12997 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0);
12999 return FoldIntToFPToInt(N, DAG);
13010 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1);
13036 if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc) {
13038 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0),
13039 DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc, DL));
13045 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT,
13048 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
13069 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0);
13074 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0));
13083 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT,
13085 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In);
13092 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
13098 DAG.getNode(ISD::FP_ROUND, SDLoc(N0),
13100 DAG.getIntPtrConstant(1, SDLoc(N0))),
13117 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0);
13128 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0);
13152 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0);
13164 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0);
13166 if (TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize))
13167 return TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
13188 Int = DAG.getNode(ISD::XOR, DL0, IntVT, Int,
13189 DAG.getConstant(SignMask, DL0, IntVT));
13191 return DAG.getBitcast(VT, Int);
13205 return DAG.getNode(
13207 DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0.getOperand(1)),
13238 return visitFMinMax(DAG, N, minnum);
13242 return visitFMinMax(DAG, N, maxnum);
13246 return visitFMinMax(DAG, N, minimum);
13250 return visitFMinMax(DAG, N, maximum);
13259 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
13268 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0));
13286 Int = DAG.getNode(ISD::AND, DL, IntVT, Int,
13287 DAG.getConstant(SignMask, DL, IntVT));
13289 return DAG.getBitcast(N->getValueType(0), Int);
13312 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
13319 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain, NewN1, N2);
13363 return DAG.getSetCC(DL, getSetCCResultType(Op0.getValueType()),
13364 Op0, DAG.getConstant(0, DL, Op0.getValueType()),
13413 return DAG.getSetCC(SDLoc(TheXor), SetCCVT, Op0, Op1,
13441 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
13539 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
13638 if (!canFoldInAddressingMode(Ptr.getNode(), Use, DAG, TLI))
13647 Result = DAG.getIndexedLoad(SDValue(N,0), SDLoc(N),
13650 Result = DAG.getIndexedStore(SDValue(N,0), SDLoc(N),
13654 LLVM_DEBUG(dbgs() << "\nReplacing.4 "; N->dump(&DAG); dbgs() << "\nWith: ";
13655 Result.getNode()->dump(&DAG); dbgs() << '\n');
13658 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
13659 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
13661 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
13710 SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
13713 SDValue NewUse = DAG.getNode(Opcode,
13716 DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse);
13721 DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0));
13771 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
13797 if (!canFoldInAddressingMode(Use, UseUse, DAG, TLI))
13821 ? DAG.getIndexedLoad(SDValue(N,0), SDLoc(N),
13823 : DAG.getIndexedStore(SDValue(N,0), SDLoc(N),
13827 LLVM_DEBUG(dbgs() << "\nReplacing.5 "; N->dump(&DAG);
13828 dbgs() << "\nWith: "; Result.getNode()->dump(&DAG);
13832 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
13833 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
13835 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
13842 DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
13868 Inc = DAG.getConstant(*ConstInc->getConstantIntValue(), SDLoc(Inc),
13874 return DAG.getNode(Opc, SDLoc(LD), BP.getSimpleValueType(), BP, Inc);
13891 Val = DAG.getNode(ISD::FTRUNC, SDLoc(ST), STMemType, Val);
13896 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(ST), STMemType, Val);
13900 Val = DAG.getBitcast(STMemType, Val);
13916 Val = DAG.getBitcast(LDType, Val);
13919 Val = DAG.getNode(ISD::ANY_EXTEND, SDLoc(LD), LDType, Val);
13922 Val = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(LD), LDType, Val);
13925 Val = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(LD), LDType, Val);
13946 BaseIndexOffset BasePtrLD = BaseIndexOffset::match(LD, DAG);
13947 BaseIndexOffset BasePtrST = BaseIndexOffset::match(ST, DAG);
13949 if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG, Offset))
13956 if (DAG.getDataLayout().isBigEndian())
13970 SDValue Idx = DAG.getNode(Opc, SDLoc(LD), LD->getOperand(1).getValueType(),
13991 DAG.getConstant(APInt::getLowBitsSet(STType.getSizeInBits(),
13994 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask);
14014 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(LD), LDMemType, Val);
14048 LLVM_DEBUG(dbgs() << "\nReplacing.6 "; N->dump(&DAG);
14049 dbgs() << "\nWith chain: "; Chain.getNode()->dump(&DAG);
14052 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
14072 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
14080 Index = DAG.getUNDEF(N->getValueType(1));
14081 LLVM_DEBUG(dbgs() << "\nReplacing.7 "; N->dump(&DAG);
14082 dbgs() << "\nWith: "; Undef.getNode()->dump(&DAG);
14085 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef);
14086 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Index);
14087 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain);
14101 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
14103 SDValue NewLoad = DAG.getExtLoad(
14124 ReplLoad = DAG.getLoad(N->getValueType(0), SDLoc(LD),
14127 ReplLoad = DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD),
14134 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N),
14652 LoadedSlice LS(User, LD, Shift, &DAG);
14693 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
14695 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
14898 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
14906 NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
14924 if (DAG.getDataLayout().isBigEndian())
14928 Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
14929 if (NewAlign < DAG.getDataLayout().getABITypeAlignment(NewVTTy))
14932 SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD),
14934 DAG.getConstant(PtrOff, SDLoc(LD),
14937 DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr,
14940 SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
14941 DAG.getConstant(NewImm, SDLoc(Value),
14944 DAG.getStore(Chain, SDLoc(N), NewVal, NewPtr,
14951 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1));
14978 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
14987 Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
14988 unsigned ABIAlign = DAG.getDataLayout().getABITypeAlignment(IntVTTy);
14993 DAG.getLoad(IntVT, SDLoc(Value), LD->getChain(), LD->getBasePtr(),
14997 DAG.getStore(ST->getChain(), SDLoc(N), NewLD, ST->getBasePtr(),
15003 DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1));
15075 DAG.isConstantIntBuildVectorOrConstantInt(OtherOp->getOperand(1)) &&
15102 return DAG.getTokenFactor(StoreDL, Chains);
15123 StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
15125 StoreTy = EVT::getIntegerVT(*DAG.getContext(), SizeInBits);
15140 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
15145 Val = DAG.getConstant(C->getAPIntValue()
15151 Val = DAG.getBitcast(MemVT, Val);
15155 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
15175 Val = DAG.getBitcast(MemVT, Val);
15181 Val = DAG.getNode(OpC, SDLoc(Val), MemVT, Vec, Idx);
15188 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
15201 bool IsLE = DAG.getDataLayout().isLittleEndian();
15227 StoredVal = DAG.getConstant(StoreInt, DL, StoreTy);
15236 NewStore = DAG.getStore(NewChain, DL, StoredVal, FirstInChain->getBasePtr(),
15241 TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType());
15245 DAG.getConstant(C->getAPIntValue().zextOrTrunc(LegalizedStoreSize), DL,
15247 NewStore = DAG.getTruncStore(
15267 BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);
15288 LBasePtr = BaseIndexOffset::match(Ld, DAG);
15319 BaseIndexOffset LPtr = BaseIndexOffset::match(OtherLd, DAG);
15333 if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
15354 Ptr = BaseIndexOffset::match(Other, DAG);
15355 return (BasePtr.equalBaseIndex(Ptr, DAG, Offset));
15496 bool NoVectors = DAG.getMachineFunction().getFunction().hasFnAttribute(
15579 LLVMContext &Context = *DAG.getContext();
15580 const DataLayout &DL = DAG.getDataLayout();
15617 TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
15629 TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
15649 TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
15713 EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
15721 TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
15783 BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld, DAG);
15788 if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
15852 TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
15866 TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
15880 TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
15974 DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(),
15977 NewStore = DAG.getStore(
15982 TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT);
15983 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy,
15987 NewStore = DAG.getTruncStore(NewStoreChain, StoreDL, NewLoad,
15997 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
16025 ReplStore = DAG.getTruncStore(BetterChain, SL, ST->getValue(),
16029 ReplStore = DAG.getStore(BetterChain, SL, ST->getValue(), ST->getBasePtr(),
16034 SDValue Token = DAG.getNode(ISD::TokenFactor, SL,
16074 Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
16077 return DAG.getStore(Chain, DL, Tmp, Ptr, ST->getMemOperand());
16086 Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
16088 return DAG.getStore(Chain, DL, Tmp,
16098 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, SDLoc(CFP), MVT::i32);
16099 SDValue Hi = DAG.getConstant(Val >> 32, SDLoc(CFP), MVT::i32);
16100 if (DAG.getDataLayout().isBigEndian())
16107 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16109 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
16110 DAG.getConstant(4, DL, Ptr.getValueType()));
16112 SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr,
16115 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
16143 DAG, *ST->getMemOperand())) {
16144 return DAG.getStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
16156 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
16159 DAG.getTruncStore(Chain, SDLoc(N), Value, Ptr, ST->getPointerInfo(),
16202 if (SDValue Shorter = DAG.GetDemandedBits(Value, TruncDemandedBits))
16203 return DAG.getTruncStore(Chain, SDLoc(N), Shorter, Ptr, ST->getMemoryVT(),
16246 const BaseIndexOffset STBase = BaseIndexOffset::match(ST, DAG);
16247 const BaseIndexOffset ChainBase = BaseIndexOffset::match(ST1, DAG);
16254 if (STBase.contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
16264 if (ChainBase.contains(DAG, ChainBitSize, STBase, STBitSize,
16272 if (!DAG.getDataLayout().isBigEndian()) {
16275 DAG.getConstant(Val, SDLoc(C), ChainValue.getValueType(),
16277 SDNode *NewST1 = DAG.UpdateNodeOperands(
16295 return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0),
16368 const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG);
16371 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
16476 EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize);
16477 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0));
16478 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0));
16483 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16486 DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
16487 DAG.getConstant(HalfValBitSize / 8, DL, Ptr.getValueType()));
16490 DAG.getStore(St0, DL, Hi, Ptr,
16533 Y, NewMask, DAG);
16570 EVT ShufVT = EVT::getVectorVT(*DAG.getContext(), SubVecEltVT, NumMaskVals);
16577 SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT));
16579 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps);
16582 SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec);
16583 SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask);
16587 return DAG.getBitcast(VT, Shuf);
16611 return DAG.getBuildVector(VT, DL, Ops);
16633 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16636 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()),
16655 Ops.append(NumElts, DAG.getUNDEF(InVal.getValueType()));
16666 Ops[Elt] = OpVT.isInteger() ? DAG.getAnyExtOrTrunc(InVal, DL, OpVT) : InVal;
16670 return DAG.getBuildVector(VT, DL, Ops);
16681 unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
16682 VecEltVT.getTypeForEVT(*DAG.getContext()));
16702 Offset = DAG.getConstant(PtrOff, DL, PtrType);
16705 Offset = DAG.getZExtOrTrunc(EltNo, DL, PtrType);
16706 Offset = DAG.getNode(
16708 DAG.getConstant(VecEltVT.getStoreSize(), DL, PtrType));
16713 NewPtr = DAG.getNode(ISD::ADD, DL, PtrType, NewPtr, Offset);
16730 Load = DAG.getExtLoad(ExtType, SDLoc(EVE), ResultVT,
16736 Load = DAG.getLoad(VecEltVT, SDLoc(EVE), OriginalLoad->getChain(), NewPtr,
16741 Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load);
16743 Load = DAG.getBitcast(ResultVT, Load);
16748 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
16800 return DAG.getUNDEF(ScalarVT);
16810 return VecVT.isInteger() ? DAG.getAnyExtOrTrunc(Elt, DL, ScalarVT) : Elt;
16821 return DAG.getSExtOrTrunc(InOp, DL, ScalarVT);
16830 return DAG.getUNDEF(ScalarVT);
16853 bool IsLE = DAG.getDataLayout().isLittleEndian();
16859 return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, BCSrc);
16879 return DAG.getAnyExtOrTrunc(X, DL, ScalarVT);
16884 if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
16900 return DAG.getUNDEF(ScalarVT);
16915 InOp = DAG.getSExtOrTrunc(InOp, DL, ScalarVT);
16928 EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
16929 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SVInVec,
16930 DAG.getConstant(OrigElt, DL, IndexTy));
17037 Index = DAG.getConstant(Elt, DL, Index.getValueType());
17048 return DAG.getUNDEF(LVT);
17120 bool isLE = DAG.getDataLayout().isLittleEndian();
17123 SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
17124 DAG.getConstant(0, DL, SourceType);
17137 In = DAG.getUNDEF(SourceType);
17148 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
17158 SDValue BV = DAG.getBuildVector(VecVT, DL, Ops);
17163 return DAG.getBitcast(VT, BV);
17170 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
17171 SDValue ZeroIdx = DAG.getConstant(0, DL, IdxTy);
17192 SmallVector<SDValue, 2> ConcatOps(NumConcats, DAG.getUNDEF(InVT1));
17194 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.getUNDEF(InVT1);
17195 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
17204 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1,
17205 DAG.getConstant(NumElems, DL, IdxTy));
17206 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx);
17225 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1,
17226 DAG.getUNDEF(InVT1), VecIn2, ZeroIdx);
17237 SmallVector<SDValue, 2> ConcatOps(2, DAG.getUNDEF(InVT2));
17239 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
17272 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(InVT1);
17275 SDValue Shuffle = DAG.getVectorShuffle(InVT1, DL, VecIn1, VecIn2, Mask);
17277 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx);
17370 if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
17446 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
17461 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(),
17464 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17465 DAG.getConstant(SplitSize, DL, IdxTy));
17466 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17467 DAG.getConstant(0, DL, IdxTy));
17518 Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT)
17519 : DAG.getConstantFP(0.0, DL, VT));
17545 Shuffles.push_back(DAG.getUNDEF(VT));
17549 Shuffles[CurSize] = DAG.getUNDEF(VT);
17567 DAG.getVectorShuffle(VT, DL, Shuffles[Left], Shuffles[Right], Mask);
17605 EVT InVT = EVT::getVectorVT(*DAG.getContext(), InSVT, NumElems);
17618 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In,
17620 return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL,
17629 return DAG.getUNDEF(VT);
17645 EVT NewVT = EVT::getVectorVT(*DAG.getContext(),
17649 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N),
17651 return DAG.getBitcast(VT, Concat);
17661 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V);
17689 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0),
17850 return DAG.getUNDEF(VT);
17864 Ops.resize(NumOps, DAG.getUNDEF(Ops[0].getValueType()));
17865 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
17902 EVT NVT = EVT::getVectorVT(*DAG.getContext(), SclTy, VNTNumElms);
17906 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar);
17907 return DAG.getBitcast(VT, Res);
17941 Opnds.append(NumElts, DAG.getUNDEF(MinVT));
17950 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i)));
17957 return DAG.getBuildVector(VT, SDLoc(N), Opnds);
17961 if (SDValue V = combineConcatVectorOfScalars(N, DAG))
17966 if (SDValue V = combineConcatVectorOfExtracts(N, DAG))
18228 return DAG.getUNDEF(NVT);
18231 if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG))
18242 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0),
18258 EVT NewExtVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
18263 SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL);
18264 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
18266 return DAG.getBitcast(NVT, NewExtract);
18300 : EVT::getVectorVT(*DAG.getContext(),
18313 Src = DAG.getNode(ISD::TRUNCATE, SDLoc(N), InVT, Src);
18314 return DAG.getBitcast(NVT, Src);
18318 SDValue BuildVec = DAG.getBuildVector(
18320 return DAG.getBitcast(NVT, BuildVec);
18344 return DAG.getBitcast(NVT, V.getOperand(1));
18345 return DAG.getNode(
18347 DAG.getBitcast(N->getOperand(0).getValueType(), V.getOperand(0)),
18352 if (SDValue NarrowBOp = narrowExtractedVectorBinOp(N, DAG))
18855 return DAG.getUNDEF(VT);
18867 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), NewMask);
18867 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), NewMask);
18872 return DAG.getCommutedVectorShuffle(*SVN);
18887 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask);
18890 if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG))
18894 if (SDValue V = combineShuffleOfSplatVal(SVN, DAG))
18908 SDValue Index = DAG.getIntPtrConstant(SplatIndex, DL);
18909 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index);
18910 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index);
18911 SDValue NewBO = DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR,
18913 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO);
18915 return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask);
18915 return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask);
18956 SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops);
18961 NewBV = DAG.getBitcast(VT, NewBV);
18976 if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations))
18980 if (SDValue V = combineTruncationShuffle(SVN, DAG))
18988 if (SDValue V = partitionShuffleOfConcats(N, DAG))
18995 if (SDValue Res = combineShuffleOfScalars(SVN, DAG, TLI))
19054 SV0 = DAG.getBitcast(ScaleVT, SV0);
19055 SV1 = DAG.getBitcast(ScaleVT, SV1);
19056 return DAG.getBitcast(
19057 VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask));
19082 return DAG.getCommutedVectorShuffle(*SVN);
19168 return DAG.getUNDEF(VT);
19171 SV0 = DAG.getUNDEF(VT);
19173 SV1 = DAG.getUNDEF(VT);
19182 return TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, Mask, DAG);
19185 if (SDValue V = foldShuffleOfConcatUndefs(SVN, DAG))
19211 DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal);
19212 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val);
19218 DAG.getUNDEF(InVecT), NewMask, DAG);
19218 DAG.getUNDEF(InVecT), NewMask, DAG);
19226 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
19227 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(N), IdxTy);
19229 EVT::getVectorVT(*DAG.getContext(), InVecT.getVectorElementType(),
19231 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), SubVT,
19269 return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0));
19284 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N),
19286 return DAG.getBitcast(VT, NewINSERT);
19296 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
19304 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0,
19326 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
19327 LLVMContext &Ctx = *DAG.getContext();
19333 NewIdx = DAG.getConstant(InsIdx * Scale, DL, IdxVT);
19338 NewIdx = DAG.getConstant(InsIdx / Scale, DL, IdxVT);
19342 SDValue Res = DAG.getBitcast(NewVT, N0Src);
19343 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx);
19344 return DAG.getBitcast(VT, Res);
19359 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT,
19362 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()),
19376 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
19403 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0),
19419 SDValue Res = DAG.getNode(
19421 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
19421 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
19423 Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res);
19434 DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits())
19435 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0);
19490 if (DAG.getDataLayout().isBigEndian()) {
19508 EVT ClearSVT = EVT::getIntegerVT(*DAG.getContext(), NumSubBits);
19509 EVT ClearVT = EVT::getVectorVT(*DAG.getContext(), ClearSVT, NumSubElts);
19513 SDValue Zero = DAG.getConstant(0, DL, ClearVT);
19514 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL,
19514 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL,
19515 DAG.getBitcast(ClearVT, LHS),
19591 if (SDValue Fold = DAG.FoldConstantVectorArithmetic(
19611 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0),
19614 return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask());
19635 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT));
19635 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT));
19635 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT));
19636 SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y);
19637 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z);
19666 ConcatOps.push_back(DAG.getNode(Opcode, DL, NarrowVT, LHS.getOperand(i),
19670 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
19674 if (SDValue V = scalarizeBinOpOfSplats(N, DAG))
19695 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0),
19700 SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
19836 Addr = DAG.getSelect(SDLoc(TheSelect),
19858 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect),
19878 Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect),
19883 Load = DAG.getExtLoad(
19940 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
19941 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt);
19945 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
19950 Shift = DAG.getNOT(DL, Shift, AType);
19952 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
19955 SDValue ShiftAmt = DAG.getConstant(XType.getSizeInBits() - 1, DL, ShiftAmtTy);
19956 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt);
19960 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
19965 Shift = DAG.getNOT(DL, Shift, AType);
19967 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
20002 const DataLayout &TD = DAG.getDataLayout();
20006 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
20006 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
20012 SDValue Zero = DAG.getIntPtrConstant(0, DL);
20014 SDValue One = DAG.getIntPtrConstant(EltSize, SDLoc(FV));
20016 DAG.getSetCC(DL, getSetCCResultType(N0.getValueType()), N0, N1, CC);
20018 SDValue CstOffset = DAG.getSelect(DL, Zero.getValueType(), Cond, One, Zero);
20020 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, CstOffset);
20022 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
20022 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
20024 DAG.getMachineFunction()), Alignment);
20043 if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) {
20078 DAG.getConstant(AndMask.countLeadingZeros(), SDLoc(AndLHS),
20080 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
20085 DAG.getConstant(AndMask.getBitWidth() - 1, SDLoc(Shl),
20087 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
20089 return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
20115 SCC = DAG.getSetCC(DL, CmpResVT, N0, N1, CC);
20117 Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT);
20119 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC);
20121 SCC = DAG.getSetCC(SDLoc(N0), MVT::i1, N0, N1, CC);
20122 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC);
20132 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
20133 DAG.getConstant(N2C->getAPIntValue().logBase2(),
20161 return DAG.getNode(ISD::CTTZ, DL, VT, N0);
20168 return DAG.getNode(ISD::CTLZ, DL, VT, N0);
20181 DagCombineInfo(DAG, Level, false, this);
20192 if (DAG.getMachineFunction().getFunction().hasMinSize())
20196 if (SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, Built)) {
20217 if (SDValue S = TLI.BuildSDIVPow2(N, C->getAPIntValue(), DAG, Built)) {
20233 if (DAG.getMachineFunction().getFunction().hasMinSize())
20237 if (SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, Built)) {
20251 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V);
20252 SDValue Base = DAG.getConstant(EltBits - 1, DL, VT);
20253 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz);
20276 MachineFunction &MF = DAG.getMachineFunction();
20284 if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
20289 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
20297 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags);
20301 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags);
20304 NewEst = DAG.getNode(ISD::FSUB, DL, VT,
20308 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
20311 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags);
20316 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags);
20337 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT);
20341 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags);
20342 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags);
20346 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
20347 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags);
20348 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags);
20349 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
20354 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags);
20369 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT);
20370 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT);
20379 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags);
20380 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags);
20381 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags);
20389 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags);
20392 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags);
20395 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags);
20415 MachineFunction &MF = DAG.getMachineFunction();
20426 TLI.getSqrtEstimate(Op, DAG, Enabled, Iterations, UseOneConstNR,
20441 const Function &F = DAG.getMachineFunction().getFunction();
20445 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
20447 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
20448 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20449 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
20450 SDValue IsDenorm = DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
20451 Est = DAG.getNode(SelOpcode, DL, VT, IsDenorm, FPZero, Est);
20454 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20455 SDValue IsZero = DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
20456 Est = DAG.getNode(SelOpcode, DL, VT, IsZero, FPZero, Est);
20539 DAG, IsAlias))
20577 : DAG.getSubtarget().useAA();
20580 CombinerAAOnlyFunc != DAG.getMachineFunction().getName())
20723 return DAG.getEntryNode();
20730 return DAG.getTokenFactor(SDLoc(N), Aliases);
20767 const BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);
20789 const BaseIndexOffset Ptr = BaseIndexOffset::match(Chain, DAG);
20792 if (!BasePtr.equalBaseIndex(Ptr, DAG, Offset))
20821 S = cast<StoreSDNode>(DAG.UpdateNodeOperands(
20831 NewST = DAG.getTruncStore(BetterChain, SDLoc(St), St->getValue(),
20835 NewST = DAG.getStore(BetterChain, SDLoc(St), St->getValue(),
20850 SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps);
20865 const BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);