efriedma updated this revision to Diff 237830.
efriedma edited the summary of this revision.
efriedma added a comment.
Fixed one more issue. While I'm at it, get rid of the old overload of
ConstantExpr::getShuffleVector, which only had a few remaining uses.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D72467/new/
https://reviews.llvm.org/D72467
Files:
clang/lib/CodeGen/CGExpr.cpp
llvm/include/llvm/ADT/ArrayRef.h
llvm/include/llvm/Analysis/ConstantFolding.h
llvm/include/llvm/Analysis/InstructionSimplify.h
llvm/include/llvm/Analysis/TargetFolder.h
llvm/include/llvm/IR/ConstantFolder.h
llvm/include/llvm/IR/Constants.h
llvm/include/llvm/IR/IRBuilder.h
llvm/include/llvm/IR/Instructions.h
llvm/include/llvm/IR/NoFolder.h
llvm/include/llvm/IR/PatternMatch.h
llvm/lib/Analysis/ConstantFolding.cpp
llvm/lib/Analysis/InstructionSimplify.cpp
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Analysis/ValueTracking.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
llvm/lib/IR/AsmWriter.cpp
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/ConstantFold.cpp
llvm/lib/IR/ConstantFold.h
llvm/lib/IR/Constants.cpp
llvm/lib/IR/ConstantsContext.h
llvm/lib/IR/Core.cpp
llvm/lib/IR/Instruction.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/ARM/MVETailPredication.cpp
llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
llvm/lib/Transforms/Scalar/GVN.cpp
llvm/lib/Transforms/Scalar/GVNSink.cpp
llvm/lib/Transforms/Scalar/NewGVN.cpp
llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/unittests/IR/PatternMatch.cpp
Index: llvm/unittests/IR/PatternMatch.cpp
===================================================================
--- llvm/unittests/IR/PatternMatch.cpp
+++ llvm/unittests/IR/PatternMatch.cpp
@@ -991,31 +991,31 @@
EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
// Test matching shufflevector
- EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
- EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
+ ArrayRef<int> Mask;
+ EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
+ EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
EXPECT_TRUE(A == VI3);
EXPECT_TRUE(B == VI4);
- EXPECT_TRUE(C == IdxVec);
A = B = C = nullptr; // reset
// Test matching the vector splat pattern
EXPECT_TRUE(match(
SI1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(A == Val);
}
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3399,7 +3399,7 @@
auto *O1 = B.CreateZExtOrTrunc(
SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
- NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
+ NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
} else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
// Don't do anything with the operands, just extend the result.
continue;
Index: llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -987,7 +987,7 @@
auto *SV = cast<ShuffleVectorInst>(I);
UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
std::string Name = suffixed_name_or(I, ".base", "base_sv");
- return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
+ return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
Name, SV);
}
};
Index: llvm/lib/Transforms/Scalar/NewGVN.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -2030,10 +2030,12 @@
case Instruction::Select:
case Instruction::ExtractElement:
case Instruction::InsertElement:
- case Instruction::ShuffleVector:
case Instruction::GetElementPtr:
E = createExpression(I);
break;
+ case Instruction::ShuffleVector:
+ // FIXME: Add support for shufflevector to createExpression.
+ return nullptr;
default:
return nullptr;
}
Index: llvm/lib/Transforms/Scalar/GVNSink.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -350,6 +350,7 @@
class InstructionUseExpr : public GVNExpression::BasicExpression {
unsigned MemoryUseOrder = -1;
bool Volatile = false;
+ std::vector<int> ShuffleMask;
public:
InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
@@ -366,15 +367,18 @@
void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; }
void setVolatile(bool V) { Volatile = V; }
+ void setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ }
hash_code getHashValue() const override {
return hash_combine(GVNExpression::BasicExpression::getHashValue(),
- MemoryUseOrder, Volatile);
+ MemoryUseOrder, Volatile, ArrayRef<int>(ShuffleMask));
}
template <typename Function> hash_code getHashValue(Function MapFn) {
- hash_code H =
- hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
+ hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder,
+ Volatile, ArrayRef<int>(ShuffleMask));
for (auto *V : operands())
H = hash_combine(H, MapFn(V));
return H;
@@ -402,6 +406,8 @@
CmpInst::Predicate Predicate = C->getPredicate();
E->setOpcode((C->getOpcode() << 8) | Predicate);
}
+ if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
+ E->setShuffleMask(SVI->getShuffleMask());
return E;
}
Index: llvm/lib/Transforms/Scalar/GVN.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/GVN.cpp
+++ llvm/lib/Transforms/Scalar/GVN.cpp
@@ -301,6 +301,9 @@
for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
II != IE; ++II)
e.varargs.push_back(*II);
+ } else if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) {
+ ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
+ e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
}
return e;
@@ -1710,7 +1713,8 @@
// instead of value numbers. Those index numbers should not be
// translated.
if ((i > 1 && Exp.opcode == Instruction::InsertValue) ||
- (i > 0 && Exp.opcode == Instruction::ExtractValue))
+ (i > 0 && Exp.opcode == Instruction::ExtractValue) ||
+ (i > 1 && Exp.opcode == Instruction::ShuffleVector))
continue;
Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn);
}
Index: llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
===================================================================
--- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1868,10 +1868,9 @@
}
void visitShuffleVectorInst(ShuffleVectorInst &I) {
- insertShadowCheck(I.getOperand(2), &I);
IRBuilder<> IRB(&I);
setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
- I.getOperand(2), "_msprop"));
+ I.getShuffleMask(), "_msprop"));
setOriginForNaryOp(I);
}
Index: llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1444,9 +1444,9 @@
// narrow binop on each pair of the source operands followed by concatenation
// of the results.
Value *L0, *L1, *R0, *R1;
- Constant *Mask;
- if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
+ ArrayRef<int> Mask;
+ if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
+ match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
LHS->hasOneUse() && RHS->hasOneUse() &&
cast<ShuffleVectorInst>(LHS)->isConcat() &&
cast<ShuffleVectorInst>(RHS)->isConcat()) {
@@ -1470,7 +1470,7 @@
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
- auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
+ auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
Value *XY = Builder.CreateBinOp(Opcode, X, Y);
if (auto *BO = dyn_cast<BinaryOperator>(XY))
BO->copyIRFlags(&Inst);
@@ -1480,8 +1480,8 @@
// If both arguments of the binary operation are shuffles that use the same
// mask and shuffle within a single vector, move the shuffle after the binop.
Value *V1, *V2;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
+ match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
// Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
@@ -1491,17 +1491,19 @@
// If both arguments of a commutative binop are select-shuffles that use the
// same mask with commuted operands, the shuffles are unnecessary.
if (Inst.isCommutative() &&
- match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
+ match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
- m_Specific(Mask)))) {
+ m_SpecificMask(Mask)))) {
auto *LShuf = cast<ShuffleVectorInst>(LHS);
auto *RShuf = cast<ShuffleVectorInst>(RHS);
// TODO: Allow shuffles that contain undefs in the mask?
// That is legal, but it reduces undef knowledge.
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
- if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
- RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
+ if (LShuf->isSelect() &&
+ !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+ RShuf->isSelect() &&
+ !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
@@ -1519,7 +1521,7 @@
// transforms.
Constant *C;
if (match(&Inst, m_c_BinOp(
- m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
+ m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))),
m_Constant(C))) &&
V1->getType()->getVectorNumElements() <= NumElts) {
assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
@@ -1531,8 +1533,7 @@
// reorder is not possible. A 1-to-1 mapping is not required. Example:
// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
bool ConstOp1 = isa<Constant>(RHS);
- SmallVector<int, 16> ShMask;
- ShuffleVectorInst::getShuffleMask(Mask, ShMask);
+ ArrayRef<int> ShMask = Mask;
unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
Index: llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -723,7 +723,7 @@
}
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
- int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
+ int MaskSize = Shuf.getShuffleMask().size();
int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
// A vector select does not change the size of the operands.
@@ -835,13 +835,10 @@
// inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
// --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *Zero = ConstantInt::getNullValue(I32Ty);
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (unsigned i = 0; i != NumMaskElts; ++i)
- NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
+ NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
}
@@ -871,26 +868,23 @@
// For example:
// inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
- Constant *OldMask = Shuf->getMask();
+ SmallVector<int, 16> NewMask(NumMaskElts);
+ ArrayRef<int> OldMask = Shuf->getShuffleMask();
for (unsigned i = 0; i != NumMaskElts; ++i) {
if (i != IdxC) {
// All mask elements besides the inserted element remain the same.
- NewMaskVec[i] = OldMask->getAggregateElement(i);
- } else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
+ NewMask[i] = OldMask[i];
+ } else if (OldMask[i] == (int)IdxC) {
// If the mask element was already set, there's nothing to do
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
+ assert(OldMask[i] == UndefMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
- NewMaskVec[i] = NewMaskEltC;
+ NewMask[i] = IdxC;
}
}
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
}
@@ -958,19 +952,18 @@
// mask vector with the insertelt index plus the length of the vector
// (because the constant vector operand of a shuffle is always the 2nd
// operand).
- Constant *Mask = Shuf->getMask();
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask = Shuf->getShuffleMask();
+ unsigned NumElts = Mask.size();
SmallVector<Constant *, 16> NewShufElts(NumElts);
- SmallVector<Constant *, 16> NewMaskElts(NumElts);
+ SmallVector<int, 16> NewMaskElts(NumElts);
for (unsigned I = 0; I != NumElts; ++I) {
if (I == InsEltIndex) {
NewShufElts[I] = InsEltScalar;
- Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
- NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
+ NewMaskElts[I] = InsEltIndex + NumElts;
} else {
// Copy over the existing values.
NewShufElts[I] = ShufConstVec->getAggregateElement(I);
- NewMaskElts[I] = Mask->getAggregateElement(I);
+ NewMaskElts[I] = Mask[I];
}
}
@@ -978,7 +971,7 @@
// original insertelt. The old shuffle will be dead now.
return new ShuffleVectorInst(Shuf->getOperand(0),
ConstantVector::get(NewShufElts),
- ConstantVector::get(NewMaskElts));
+ NewMaskElts);
} else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
// Transform sequences of insertelements ops with constant data/indexes into
// a single shuffle op.
@@ -1298,17 +1291,9 @@
if (isa<ConstantAggregateZero>(V))
return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size()));
- if (Constant *C = dyn_cast<Constant>(V)) {
- SmallVector<Constant *, 16> MaskValues;
- for (int i = 0, e = Mask.size(); i != e; ++i) {
- if (Mask[i] == -1)
- MaskValues.push_back(UndefValue::get(I32Ty));
- else
- MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i]));
- }
+ if (Constant *C = dyn_cast<Constant>(V))
return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()),
- ConstantVector::get(MaskValues));
- }
+ Mask);
Instruction *I = cast<Instruction>(V);
switch (I->getOpcode()) {
@@ -1397,7 +1382,7 @@
// Shuffles to: |EE|FF|GG|HH|
// +--+--+--+--+
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
- SmallVector<int, 16> &Mask) {
+ ArrayRef<int> Mask) {
unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
unsigned MaskElems = Mask.size();
unsigned BegIdx = Mask.front();
@@ -1480,12 +1465,12 @@
// Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
// Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
// The existing binop constant vector remains in the same operand position.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -1499,7 +1484,7 @@
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1511,14 +1496,15 @@
static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
InstCombiner::BuilderTy &Builder) {
Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Value *X;
uint64_t IndexC;
// Match a shuffle that is a splat to a non-zero element.
if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
m_ConstantInt(IndexC)))) ||
- !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
+ !match(Op1, m_Undef()) ||
+ all_of(Mask, [](int Elt) { return Elt == 0; }) || IndexC == 0)
return nullptr;
// Insert into element 0 of an undef vector.
@@ -1531,12 +1517,12 @@
// shuf (inselt undef, X, 2), undef, <2,2,undef>
// --> shuf (inselt undef, X, 0), undef, <0,0,undef>
unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
+ SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (isa<UndefValue>(Mask->getAggregateElement(i)))
- NewMask[i] = Mask->getAggregateElement(i);
+ if (Mask[i] == UndefMaskElem)
+ NewMask[i] = Mask[i];
- return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
}
/// Try to fold shuffles that are the equivalent of a vector select.
@@ -1605,14 +1591,14 @@
BinaryOperator::BinaryOps BOpc = Opc0;
// Select the constant elements needed for the single binop.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
// We are moving a binop after a shuffle. When a shuffle has an undefined
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
@@ -1661,7 +1647,7 @@
NewBO->andIRFlags(B1);
if (DropNSW)
NewBO->setHasNoSignedWrap(false);
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1687,8 +1673,7 @@
// and have the same number of elements as this shuffle.
unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
Value *NarrowCond;
- if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
- m_Constant()))) ||
+ if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
!cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
return nullptr;
@@ -1696,8 +1681,8 @@
// shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
// sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
Value *Undef = UndefValue::get(X->getType());
- Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
- Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
+ Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
+ Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
@@ -1708,9 +1693,13 @@
return nullptr;
Value *X, *Y;
- Constant *Mask;
- if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
+ ArrayRef<int> Mask;
+ ShuffleVectorInst *Op0Shuf = dyn_cast<ShuffleVectorInst>(Op0);
+ if (!Op0Shuf)
return nullptr;
+ X = Op0Shuf->getOperand(0);
+ Y = Op0Shuf->getOperand(1);
+ Mask = Op0Shuf->getShuffleMask();
// Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
// then combining may result in worse codegen.
@@ -1729,23 +1718,24 @@
// shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
// shuf X, Y, <C0, undef, C2, undef>
unsigned NumElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumElts);
- assert(NumElts < Mask->getType()->getVectorNumElements() &&
+ SmallVector<int, 16> NewMask(NumElts);
+ assert(NumElts < Mask.size() &&
"Identity with extract must have less elements than its inputs");
for (unsigned i = 0; i != NumElts; ++i) {
- Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
- Constant *MaskElt = Mask->getAggregateElement(i);
- NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
+ int ExtractMaskElt = Shuf.getMaskValue(i);
+ int MaskElt = Mask[i];
+ NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
}
- return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(X, Y, NewMask);
}
/// Try to replace a shuffle with an insertelement or try to replace a shuffle
/// operand with the operand of an insertelement.
static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf) {
Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ SmallVector<int, 16> Mask;
+ Mask.assign(Shuf.getShuffleMask().begin(), Shuf.getShuffleMask().end());
// The shuffle must not change vector sizes.
// TODO: This restriction could be removed if the insert has only one use
@@ -1868,7 +1858,7 @@
assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
@@ -1902,29 +1892,27 @@
Value *LHS = SVI.getOperand(0);
Value *RHS = SVI.getOperand(1);
if (auto *V = SimplifyShuffleVectorInst(
- LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
+ LHS, RHS, SVI.getShuffleMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
return replaceInstUsesWith(SVI, V);
// shuffle x, x, mask --> shuffle x, undef, mask'
unsigned VWidth = SVI.getType()->getVectorNumElements();
unsigned LHSWidth = LHS->getType()->getVectorNumElements();
- SmallVector<int, 16> Mask = SVI.getShuffleMask();
+ ArrayRef<int> Mask = SVI.getShuffleMask();
Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
if (LHS == RHS) {
assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
// Remap any references to RHS to use LHS.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) {
// Propagate undef elements or force mask to LHS.
if (Mask[i] < 0)
- Elts.push_back(UndefValue::get(Int32Ty));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
+ Elts.push_back(Mask[i] % LHSWidth);
}
- SVI.setOperand(0, SVI.getOperand(1));
- SVI.setOperand(1, UndefValue::get(RHS->getType()));
- SVI.setOperand(2, ConstantVector::get(Elts));
- return &SVI;
+ return new ShuffleVectorInst(SVI.getOperand(1),
+ UndefValue::get(RHS->getType()), Elts);
}
// shuffle undef, x, mask --> shuffle x, undef, mask'
@@ -2149,8 +2137,8 @@
if (newLHS == LHS && newRHS == RHS)
return MadeChange ? &SVI : nullptr;
- SmallVector<int, 16> LHSMask;
- SmallVector<int, 16> RHSMask;
+ ArrayRef<int> LHSMask;
+ ArrayRef<int> RHSMask;
if (newLHS != LHS)
LHSMask = LHSShuffle->getShuffleMask();
if (RHSShuffle && newRHS != RHS)
Index: llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1396,15 +1396,14 @@
}
if (NewUndefElts) {
// Add additional discovered undefs.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
- Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
- Shuffle->getMaskValue(i)));
+ Elts.push_back(Shuffle->getMaskValue(i));
}
- I->setOperand(2, ConstantVector::get(Elts));
+ Shuffle->setShuffleMask(Elts);
MadeChange = true;
}
break;
Index: llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2792,11 +2792,11 @@
return nullptr;
Value *Vec;
- Constant *Mask;
+ ArrayRef<int> Mask;
if (match(BCSrcOp,
- m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
+ m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
// Check whether every element of Mask is the same constant
- if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
+ if (is_splat(Mask)) {
auto *VecTy = cast<VectorType>(BCSrcOp->getType());
auto *EltTy = cast<IntegerType>(VecTy->getElementType());
if (C->isSplat(EltTy->getBitWidth())) {
@@ -2805,6 +2805,7 @@
// then:
// => %E = extractelement <N x iK> %vec, i32 Elem
// icmp <pred> iK %SplatVal, <pattern>
+ Value *Elem = Builder.getInt32(Mask[0]);
Value *Extract = Builder.CreateExtractElement(Vec, Elem);
Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
return new ICmpInst(Pred, Extract, NewC);
@@ -5389,9 +5390,9 @@
// shuffle within a single vector, move the shuffle after the cmp.
Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
Value *V1, *V2;
- Constant *M;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
+ ArrayRef<int> M;
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))) &&
+ match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse())) {
// cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
Index: llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -639,12 +639,12 @@
InstCombiner::BuilderTy &Builder) {
auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
- Shuf->getMask()->getSplatValue() &&
+ is_splat(Shuf->getShuffleMask()) &&
Shuf->getType() == Shuf->getOperand(0)->getType()) {
// trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
Constant *NarrowUndef = UndefValue::get(Trunc.getType());
Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
- return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
+ return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
}
return nullptr;
@@ -2517,7 +2517,7 @@
Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
// Return a new shuffle vector. Use the same element ID's, as we
// know the vector types match #elts.
- return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
+ return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
}
}
Index: llvm/lib/Target/ARM/MVETailPredication.cpp
===================================================================
--- llvm/lib/Target/ARM/MVETailPredication.cpp
+++ llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -226,7 +226,7 @@
// The invariant shuffle that broadcast the limit into a vector.
Instruction *Insert = nullptr;
if (!match(Shuffle, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
- m_Zero())))
+ m_ZeroMask())))
return false;
// Insert the limit into a vector.
@@ -262,7 +262,7 @@
// The shuffle which broadcasts the index iv into a vector.
if (!match(BroadcastSplat, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
- m_Zero())))
+ m_ZeroMask())))
return false;
// The insert element which initialises a vector with the index iv.
Index: llvm/lib/Target/ARM/ARMISelLowering.cpp
===================================================================
--- llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -14985,7 +14985,7 @@
return false;
if (!match(I->getOperand(Op),
m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
- m_Undef(), m_Zero()))) {
+ m_Undef(), m_ZeroMask()))) {
return false;
}
Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));
Index: llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -409,7 +409,7 @@
DL->getTypeSizeInBits(Val->getType())) {
assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
- { 0, 1, 2 });
+ ArrayRef<int>{0, 1, 2});
}
Val = B.CreateBitCast(Val, EffectiveEltTy);
Index: llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -210,7 +210,7 @@
Arg.replaceAllUsesWith(NewVal);
} else if (IsV3) {
Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
- {0, 1, 2},
+ ArrayRef<int>{0, 1, 2},
Arg.getName() + ".load");
Arg.replaceAllUsesWith(Shuf);
} else {
Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
===================================================================
--- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8905,10 +8905,10 @@
return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
};
- Constant *M1, *M2;
+ ArrayRef<int> M1, M2;
Value *S1Op1, *S2Op1;
- if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
- !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+ if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
+ !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
return false;
// Check that the operands are half as wide as the result and we extract
Index: llvm/lib/IR/Verifier.cpp
===================================================================
--- llvm/lib/IR/Verifier.cpp
+++ llvm/lib/IR/Verifier.cpp
@@ -3286,7 +3286,7 @@
void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
- SV.getOperand(2)),
+ SV.getShuffleMask()),
"Invalid shufflevector operands!", &SV);
visitInstruction(SV);
}
Index: llvm/lib/IR/Instructions.cpp
===================================================================
--- llvm/lib/IR/Instructions.cpp
+++ llvm/lib/IR/Instructions.cpp
@@ -1787,11 +1787,14 @@
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertBefore) {
- assert(isValidOperands(V1, V2, Mask) &&
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ assert(isValidOperands(V1, V2, MaskArr) &&
"Invalid shuffle vector instruction operands!");
+
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(MaskArr);
setName(Name);
}
@@ -1804,35 +1807,90 @@
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertAtEnd) {
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ assert(isValidOperands(V1, V2, MaskArr) &&
+ "Invalid shuffle vector instruction operands!");
+
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(MaskArr);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name,
+ Instruction *InsertBefore)
+: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector,
+ OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this),
+ InsertBefore) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(Mask);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name,
+ BasicBlock *InsertAtEnd)
+: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector,
+ OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this),
+ InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(Mask);
setName(Name);
}
void ShuffleVectorInst::commute() {
int NumOpElts = Op<0>()->getType()->getVectorNumElements();
- int NumMaskElts = getMask()->getType()->getVectorNumElements();
- SmallVector<Constant*, 16> NewMask(NumMaskElts);
- Type *Int32Ty = Type::getInt32Ty(getContext());
+ int NumMaskElts = ShuffleMask.size();
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
- if (MaskElt == -1) {
- NewMask[i] = UndefValue::get(Int32Ty);
+ if (MaskElt == UndefMaskElem) {
+ NewMask[i] = UndefMaskElem;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
- NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
+ NewMask[i] = MaskElt;
}
- Op<2>() = ConstantVector::get(NewMask);
+ setShuffleMask(NewMask);
Op<0>().swap(Op<1>());
}
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask) {
+ // V1 and V2 must be vectors of the same type.
+ if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
+ return false;
+
+ // Make sure the mask elements make sense.
+ int V1Size = cast<VectorType>(V1->getType())->getNumElements();
+ for (int Elem : Mask)
+ if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+ return false;
+
+ if (V1->getType()->getVectorIsScalable())
+ if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask))
+ return false;
+
+ return true;
+}
+
+bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
const Value *Mask) {
// V1 and V2 must be vectors of the same type.
if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
@@ -1868,14 +1926,6 @@
return true;
}
- // The bitcode reader can create a place holder for a forward reference
- // used as the shuffle mask. When this occurs, the shuffle mask will
- // fall into this case and fail. To avoid this error, do this bit of
- // ugliness to allow such a mask pass.
- if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
- if (CE->getOpcode() == Instruction::UserOp1)
- return true;
-
return false;
}
@@ -1905,6 +1955,19 @@
}
}
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ SmallVector<Constant*, 16> MaskConst;
+ Type *Int32Ty = Type::getInt32Ty(getContext());
+ for (int Elem : Mask) {
+ if (Elem == UndefMaskElem)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ ShuffleMaskForBitcode = ConstantVector::get(MaskConst);
+}
+
static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
assert(!Mask.empty() && "Shuffle mask must contain elements");
bool UsesLHS = false;
@@ -2054,7 +2117,7 @@
return false;
// The first part of the mask must choose elements from exactly 1 source op.
- SmallVector<int, 16> Mask = getShuffleMask();
+ ArrayRef<int> Mask = getShuffleMask();
if (!isIdentityMaskImpl(Mask, NumOpElts))
return false;
@@ -4254,7 +4317,7 @@
}
ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
- return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
+ return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
}
PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
Index: llvm/lib/IR/Instruction.cpp
===================================================================
--- llvm/lib/IR/Instruction.cpp
+++ llvm/lib/IR/Instruction.cpp
@@ -434,6 +434,8 @@
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
+ return SVI->getShuffleMask() == cast<ShuffleVectorInst>(I2)->getShuffleMask();
return true;
}
Index: llvm/lib/IR/Core.cpp
===================================================================
--- llvm/lib/IR/Core.cpp
+++ llvm/lib/IR/Core.cpp
@@ -1784,9 +1784,11 @@
LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
LLVMValueRef VectorBConstant,
LLVMValueRef MaskConstant) {
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask);
return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant),
unwrap<Constant>(VectorBConstant),
- unwrap<Constant>(MaskConstant)));
+ IntMask));
}
LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,
Index: llvm/lib/IR/ConstantsContext.h
===================================================================
--- llvm/lib/IR/ConstantsContext.h
+++ llvm/lib/IR/ConstantsContext.h
@@ -26,6 +26,7 @@
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@@ -146,20 +147,34 @@
/// shufflevector constant exprs.
class ShuffleVectorConstantExpr : public ConstantExpr {
public:
- ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
+ ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<unsigned> Mask)
: ConstantExpr(VectorType::get(
cast<VectorType>(C1->getType())->getElementType(),
- cast<VectorType>(C3->getType())->getElementCount()),
+ Mask.size(), C1->getType()->getVectorIsScalable()),
Instruction::ShuffleVector,
- &Op<0>(), 3) {
+ &Op<0>(), 2) {
Op<0>() = C1;
Op<1>() = C2;
- Op<2>() = C3;
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ assert(ShuffleVectorInst::isValidOperands(C1, C2, ShuffleMask) &&
+ "Invalid shuffle vector instruction operands!");
+ SmallVector<Constant*, 16> MaskConst;
+ Type *Int32Ty = Type::getInt32Ty(getContext());
+ for (int Elem : ShuffleMask) {
+ if (Elem == UndefMaskElem)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ ShuffleMaskForBitcode = ConstantVector::get(MaskConst);
}
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
// allocate space for exactly three operands
void *operator new(size_t s) {
- return User::operator new(s, 3);
+ return User::operator new(s, 2);
}
/// Transparently provide more efficient getOperand methods.
@@ -319,7 +334,7 @@
template <>
struct OperandTraits<ShuffleVectorConstantExpr>
- : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
+ : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
template <>
@@ -546,7 +561,7 @@
case Instruction::InsertElement:
return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Indexes);
case Instruction::InsertValue:
return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
case Instruction::ExtractValue:
Index: llvm/lib/IR/Constants.cpp
===================================================================
--- llvm/lib/IR/Constants.cpp
+++ llvm/lib/IR/Constants.cpp
@@ -1266,6 +1266,14 @@
return cast<CompareConstantExpr>(this)->predicate;
}
+ArrayRef<int> ConstantExpr::getShuffleMask() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
+}
+
+Constant *ConstantExpr::getShuffleMaskForBitcode() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
+}
+
Constant *
ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
assert(Op->getType() == getOperand(OpNo)->getType() &&
@@ -1317,7 +1325,7 @@
case Instruction::ExtractValue:
return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2],
+ return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
OnlyIfReducedTy);
case Instruction::GetElementPtr: {
auto *GEPO = cast<GEPOperator>(this);
@@ -2221,23 +2229,26 @@
}
Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask, Type *OnlyIfReducedTy) {
+ ArrayRef<int> Mask, Type *OnlyIfReducedTy) {
assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector constant expr operands!");
if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
return FC; // Fold a few common cases.
- ElementCount NElts = Mask->getType()->getVectorElementCount();
+ unsigned NElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
- Type *ShufTy = VectorType::get(EltTy, NElts);
+ bool TypeIsScalable = V1->getType()->getVectorIsScalable();
+ Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
if (OnlyIfReducedTy == ShufTy)
return nullptr;
// Look up the constant in the table first to ensure uniqueness
- Constant *ArgVec[] = { V1, V2, Mask };
- const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
+ Constant *ArgVec[] = { V1, V2 };
+ SmallVector<unsigned, 16> MaskUnsigned;
+ MaskUnsigned.assign(Mask.begin(), Mask.end());
+ const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, MaskUnsigned);
LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
@@ -3076,7 +3087,7 @@
case Instruction::ExtractValue:
return ExtractValueInst::Create(Ops[0], getIndices());
case Instruction::ShuffleVector:
- return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
case Instruction::GetElementPtr: {
const auto *GO = cast<GEPOperator>(this);
Index: llvm/lib/IR/ConstantFold.h
===================================================================
--- llvm/lib/IR/ConstantFold.h
+++ llvm/lib/IR/ConstantFold.h
@@ -38,7 +38,7 @@
Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
Constant *Idx);
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
ArrayRef<unsigned> Idxs);
Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
Index: llvm/lib/IR/ConstantFold.cpp
===================================================================
--- llvm/lib/IR/ConstantFold.cpp
+++ llvm/lib/IR/ConstantFold.cpp
@@ -862,17 +862,14 @@
Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
Constant *V2,
- Constant *Mask) {
- unsigned MaskNumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask) {
+ unsigned MaskNumElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
// Undefined shuffle mask -> undefined value.
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; }))
return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
- // Don't break the bitcode reader hack.
- if (isa<ConstantExpr>(Mask)) return nullptr;
-
// Do not iterate on scalable vector. The num of elements is unknown at
// compile-time.
VectorType *ValTy = cast<VectorType>(V1->getType());
@@ -884,7 +881,7 @@
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
for (unsigned i = 0; i != MaskNumElts; ++i) {
- int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
+ int Elt = Mask[i];
if (Elt == -1) {
Result.push_back(UndefValue::get(EltTy));
continue;
Index: llvm/lib/IR/AutoUpgrade.cpp
===================================================================
--- llvm/lib/IR/AutoUpgrade.cpp
+++ llvm/lib/IR/AutoUpgrade.cpp
@@ -2309,11 +2309,12 @@
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, 1);
if (NumSrcElts == 2)
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 0, 1 });
+ Rep = Builder.CreateShuffleVector(
+ Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
else
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 2, 3, 0, 1, 2, 3 });
+ Rep =
+ Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
+ ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
Index: llvm/lib/IR/AsmWriter.cpp
===================================================================
--- llvm/lib/IR/AsmWriter.cpp
+++ llvm/lib/IR/AsmWriter.cpp
@@ -462,6 +462,33 @@
isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
}
+static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
+ Out << ", <";
+ if (Ty->getVectorIsScalable())
+ Out << "vscale x ";
+ Out << Mask.size() << " x i32> ";
+ bool FirstElt = true;
+ if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
+ Out << "zeroinitializer";
+ } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ Out << "undef";
+ } else {
+ Out << "<";
+ for (int Elt : Mask) {
+ if (FirstElt)
+ FirstElt = false;
+ else
+ Out << ", ";
+ Out << "i32 ";
+ if (Elt == UndefMaskElem)
+ Out << "undef";
+ else
+ Out << Elt;
+ }
+ Out << ">";
+ }
+}
+
namespace {
class TypePrinting {
@@ -1545,6 +1572,9 @@
TypePrinter.print(CE->getType(), Out);
}
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
+
Out << ')';
return;
}
@@ -4079,6 +4109,8 @@
RMWI->getSyncScopeID());
} else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
+ } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
+ PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
}
// Print Metadata info.
Index: llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
===================================================================
--- llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -1868,7 +1868,6 @@
GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
- GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
GenericValue Dest;
// There is no need to check types of src1 and src2, because the compiled
@@ -1878,7 +1877,7 @@
Type *TyContained = Ty->getElementType();
unsigned src1Size = (unsigned)Src1.AggregateVal.size();
unsigned src2Size = (unsigned)Src2.AggregateVal.size();
- unsigned src3Size = (unsigned)Src3.AggregateVal.size();
+ unsigned src3Size = I.getShuffleMask().size();
Dest.AggregateVal.resize(src3Size);
@@ -1888,7 +1887,7 @@
break;
case Type::IntegerTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal;
else if(j < src1Size + src2Size)
@@ -1904,7 +1903,7 @@
break;
case Type::FloatTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal;
else if(j < src1Size + src2Size)
@@ -1915,7 +1914,7 @@
break;
case Type::DoubleTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal;
else if(j < src1Size + src2Size)
Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3583,14 +3583,19 @@
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
- Constant *MaskV = cast<Constant>(I.getOperand(2));
+ ArrayRef<int> Mask;
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();
unsigned SrcNumElts = SrcVT.getVectorNumElements();
- if (MaskV->isNullValue() && VT.isScalableVector()) {
+ if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
+ VT.isScalableVector()) {
// Canonical splat form of first element of first input vector.
SDValue FirstElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
SrcVT.getScalarType(), Src1,
@@ -3605,8 +3610,6 @@
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MaskV, Mask);
unsigned MaskNumElts = Mask.size();
if (SrcNumElts == MaskNumElts) {
Index: llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
===================================================================
--- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1933,8 +1933,11 @@
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
+ ArrayRef<int> Mask;
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&U))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(U).getShuffleMask();
ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
.addDef(getOrCreateVReg(U))
Index: llvm/lib/CodeGen/CodeGenPrepare.cpp
===================================================================
--- llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6216,7 +6216,7 @@
}
static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
- SmallVector<int, 16> Mask(SVI->getShuffleMask());
+ ArrayRef<int> Mask(SVI->getShuffleMask());
int SplatElem = -1;
for (unsigned i = 0; i < Mask.size(); ++i) {
if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
@@ -6266,7 +6266,7 @@
assert(InsertPt != UserBB->end());
InsertedShuffle =
new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getOperand(2), "", &*InsertPt);
+ SVI->getShuffleMask(), "", &*InsertPt);
InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
}
Index: llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -88,11 +88,16 @@
if (OM.lookup(V).first)
return;
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands() && !isa<GlobalValue>(C))
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands() && !isa<GlobalValue>(C)) {
for (const Value *Op : C->operands())
if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op))
orderValue(Op, OM);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ orderValue(CE->getShuffleMaskForBitcode(), OM);
+ }
+ }
// Note: we cannot cache this lookup above, since inserting into the map
// changes the map's size, and thus affects the other IDs.
@@ -155,11 +160,14 @@
for (const Argument &A : F.args())
orderValue(&A, OM);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
isa<InlineAsm>(*Op))
orderValue(Op, OM);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ orderValue(SVI->getShuffleMaskForBitcode(), OM);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
orderValue(&I, OM);
@@ -250,11 +258,16 @@
predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack);
// Recursive descent into constants.
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands()) // Visit GlobalValues.
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands()) { // Visit GlobalValues.
for (const Value *Op : C->operands())
if (isa<Constant>(Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, F, OM, Stack);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM, Stack);
+ }
+ }
}
static UseListOrderStack predictUseListOrder(const Module &M) {
@@ -279,10 +292,13 @@
for (const Argument &A : F.args())
predictValueUseListOrder(&A, &F, OM, Stack);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, &F, OM, Stack);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM, Stack);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
predictValueUseListOrder(&I, &F, OM, Stack);
@@ -413,6 +429,8 @@
EnumerateMetadata(&F, MD->getMetadata());
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
EnumerateType(I.getType());
if (const auto *Call = dyn_cast<CallBase>(&I))
EnumerateAttributes(Call->getAttributes());
@@ -836,6 +854,9 @@
I != E; ++I)
if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
EnumerateValue(*I);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateValue(CE->getShuffleMaskForBitcode());
// Finally, add the value. Doing this could make the ValueID reference be
// dangling, don't reuse it.
@@ -913,6 +934,9 @@
EnumerateOperandType(Op);
}
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateOperandType(CE->getShuffleMaskForBitcode());
}
void ValueEnumerator::EnumerateAttributes(AttributeList PAL) {
@@ -958,11 +982,14 @@
// Add all function-level constants to the value table.
for (const BasicBlock &BB : F) {
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Use &OI : I.operands()) {
if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI))
EnumerateValue(OI);
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateValue(SVI->getShuffleMaskForBitcode());
+ }
BasicBlocks.push_back(&BB);
ValueMap[&BB] = BasicBlocks.size();
}
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2504,7 +2504,7 @@
}
Record.push_back(VE.getValueID(C->getOperand(0)));
Record.push_back(VE.getValueID(C->getOperand(1)));
- Record.push_back(VE.getValueID(C->getOperand(2)));
+ Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
break;
case Instruction::ICmp:
case Instruction::FCmp:
@@ -2691,7 +2691,7 @@
Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
pushValueAndType(I.getOperand(0), InstID, Vals);
pushValue(I.getOperand(1), InstID, Vals);
- pushValue(I.getOperand(2), InstID, Vals);
+ pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID, Vals);
break;
case Instruction::ICmp:
case Instruction::FCmp: {
Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp
===================================================================
--- llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2333,6 +2333,14 @@
Type *CurFullTy = Type::getInt32Ty(Context);
unsigned NextCstNo = ValueList.size();
+ struct DelayedShufTy {
+ VectorType *OpTy;
+ Type *OpFullTy;
+ uint64_t Op0Idx;
+ uint64_t Op1Idx;
+ uint64_t Op2Idx;
+ };
+ std::vector<DelayedShufTy> DelayedShuffles;
while (true) {
Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
if (!MaybeEntry)
@@ -2349,6 +2357,30 @@
// Once all the constants have been read, go through and resolve forward
// references.
+ //
+ // We have to treat shuffles specially because they don't have three
+ // operands anymore. We need to convert the shuffle mask into an array,
+ // and we can't convert a forward reference.
+ //
+ // FIXME: Does this break the use list order?
+ for (auto &DelayedShuffle : DelayedShuffles) {
+ VectorType *OpTy = DelayedShuffle.OpTy;
+ uint64_t Op0Idx = DelayedShuffle.Op0Idx;
+ uint64_t Op1Idx = DelayedShuffle.Op1Idx;
+ uint64_t Op2Idx = DelayedShuffle.Op2Idx;
+ Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
+ Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
+ Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
+ OpTy->getNumElements());
+ Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy);
+ if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2))
+ return error("Invalid shufflevector operands");
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(Op2, Mask);
+ Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
+ ValueList.assignValue(V, NextCstNo, DelayedShuffle.OpFullTy);
+ ++NextCstNo;
+ }
ValueList.resolveConstantForwardRefs();
return Error::success();
case BitstreamEntry::Record:
@@ -2689,13 +2721,9 @@
VectorType *OpTy = dyn_cast<VectorType>(CurTy);
if (Record.size() < 3 || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- OpTy->getNumElements());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back({OpTy, CurFullTy, Record[0], Record[1],
+ Record[2]});
+ continue;
}
case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
VectorType *RTy = dyn_cast<VectorType>(CurTy);
@@ -2703,13 +2731,9 @@
dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
if (Record.size() < 4 || !RTy || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- RTy->getNumElements());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back({OpTy, CurFullTy, Record[1], Record[2],
+ Record[3]});
+ continue;
}
case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred]
if (Record.size() < 4)
Index: llvm/lib/AsmParser/LLParser.cpp
===================================================================
--- llvm/lib/AsmParser/LLParser.cpp
+++ llvm/lib/AsmParser/LLParser.cpp
@@ -3633,8 +3633,9 @@
return Error(ID.Loc, "expected three operands to shufflevector");
if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2]))
return Error(ID.Loc, "invalid operands to shufflevector");
- ID.ConstantVal =
- ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]);
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Elts[2]), Mask);
+ ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask);
} else if (Opc == Instruction::ExtractElement) {
if (Elts.size() != 2)
return Error(ID.Loc, "expected two operands to extractelement");
Index: llvm/lib/Analysis/VectorUtils.cpp
===================================================================
--- llvm/lib/Analysis/VectorUtils.cpp
+++ llvm/lib/Analysis/VectorUtils.cpp
@@ -320,7 +320,7 @@
Value *Splat;
if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
m_ZeroInt()),
- m_Value(), m_ZeroInt())))
+ m_Value(), m_ZeroMask())))
return Splat;
return nullptr;
@@ -342,9 +342,9 @@
}
// FIXME: Constant splat analysis does not allow undef elements.
- Constant *Mask;
- if (match(V, m_ShuffleVector(m_Value(), m_Value(), m_Constant(Mask))))
- return Mask->getSplatValue() != nullptr;
+ ArrayRef<int> Mask;
+ if (match(V, m_ShuffleVector(m_Value(), m_Value(), m_Mask(Mask))))
+ return is_splat(Mask);
// The remaining tests are all recursive, so bail out if we hit the limit.
if (Depth++ == MaxDepth)
Index: llvm/lib/Analysis/ValueTracking.cpp
===================================================================
--- llvm/lib/Analysis/ValueTracking.cpp
+++ llvm/lib/Analysis/ValueTracking.cpp
@@ -2710,7 +2710,7 @@
// element referenced by the shuffle.
auto *Shuf = cast<ShuffleVectorInst>(U);
int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
- int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+ int NumMaskElts = Shuf->getShuffleMask().size();
APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
for (int i = 0; i != NumMaskElts; ++i) {
int M = Shuf->getMaskValue(i);
Index: llvm/lib/Analysis/TargetTransformInfo.cpp
===================================================================
--- llvm/lib/Analysis/TargetTransformInfo.cpp
+++ llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -872,7 +872,7 @@
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
- SmallVector<int, 16> ActualMask = SI->getShuffleMask();
+ ArrayRef<int> ActualMask = SI->getShuffleMask();
return Mask == ActualMask;
}
@@ -1137,7 +1137,7 @@
// Fill the rest of the mask with -1 for undef.
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
- SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
+ ArrayRef<int> Mask = Shuffle->getShuffleMask();
if (ShuffleMask != Mask)
return RK_None;
Index: llvm/lib/Analysis/InstructionSimplify.cpp
===================================================================
--- llvm/lib/Analysis/InstructionSimplify.cpp
+++ llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4416,18 +4416,18 @@
return RootVec;
}
-static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q,
unsigned MaxRecurse) {
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
return UndefValue::get(RetTy);
Type *InVecTy = Op0->getType();
- unsigned MaskNumElts = Mask->getType()->getVectorNumElements();
+ unsigned MaskNumElts = Mask.size();
unsigned InVecNumElts = InVecTy->getVectorNumElements();
SmallVector<int, 32> Indices;
- ShuffleVectorInst::getShuffleMask(Mask, Indices);
+ Indices.assign(Mask.begin(), Mask.end());
assert(MaskNumElts == Indices.size() &&
"Size of Indices not same as number of mask elements?");
@@ -4489,7 +4489,7 @@
// value type is same as the input vectors' type.
if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
- OpShuf->getMask()->getSplatValue())
+ is_splat(OpShuf->getShuffleMask()))
return Op0;
// Don't fold a shuffle with undef mask elements. This may get folded in a
@@ -4517,7 +4517,7 @@
}
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q) {
return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
}
@@ -5464,7 +5464,7 @@
case Instruction::ShuffleVector: {
auto *SVI = cast<ShuffleVectorInst>(I);
Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getMask(), SVI->getType(), Q);
+ SVI->getShuffleMask(), SVI->getType(), Q);
break;
}
case Instruction::PHI:
Index: llvm/lib/Analysis/ConstantFolding.cpp
===================================================================
--- llvm/lib/Analysis/ConstantFolding.cpp
+++ llvm/lib/Analysis/ConstantFolding.cpp
@@ -1062,7 +1062,8 @@
case Instruction::InsertElement:
return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
+ return ConstantExpr::getShuffleVector(Ops[0], Ops[1],
+ cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
}
}
Index: llvm/include/llvm/IR/PatternMatch.h
===================================================================
--- llvm/include/llvm/IR/PatternMatch.h
+++ llvm/include/llvm/IR/PatternMatch.h
@@ -1276,12 +1276,58 @@
return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
}
+/// Matches shuffle.
+template <typename T0, typename T1, typename T2> struct Shuffle_match {
+ T0 Op1;
+ T1 Op2;
+ T2 Mask;
+
+ Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
+ : Op1(Op1), Op2(Op2), Mask(Mask) {}
+
+ template <typename OpTy> bool match(OpTy *V) {
+ if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
+ return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
+ Mask.match(I->getShuffleMask());
+ }
+ return false;
+ }
+};
+
+struct m_Mask {
+ ArrayRef<int> &MaskRef;
+ m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ MaskRef = Mask;
+ return true;
+ }
+};
+
+struct m_ZeroMask {
+ bool match(ArrayRef<int> Mask) {
+ return all_of(Mask, [](int Elem) { return Elem == 0; });
+ }
+};
+
+struct m_SpecificMask {
+ ArrayRef<int> &MaskRef;
+ m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ return MaskRef == Mask;
+ }
+};
+
/// Matches ShuffleVectorInst.
+template <typename V1_t, typename V2_t>
+inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
+ return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
+}
+
template <typename V1_t, typename V2_t, typename Mask_t>
-inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
-m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
- return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
- m);
+inline Shuffle_match<V1_t, V2_t, Mask_t>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
+ return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
}
/// Matches LoadInst.
Index: llvm/include/llvm/IR/NoFolder.h
===================================================================
--- llvm/include/llvm/IR/NoFolder.h
+++ llvm/include/llvm/IR/NoFolder.h
@@ -325,7 +325,7 @@
}
Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return new ShuffleVectorInst(V1, V2, Mask);
}
Index: llvm/include/llvm/IR/Instructions.h
===================================================================
--- llvm/include/llvm/IR/Instructions.h
+++ llvm/include/llvm/IR/Instructions.h
@@ -1977,10 +1977,23 @@
// ShuffleVectorInst Class
//===----------------------------------------------------------------------===//
+constexpr int UndefMaskElem = -1;
+
/// This instruction constructs a fixed permutation of two
/// input vectors.
///
+/// The shuffle mask operand specifies, for each element of the result vector,
+/// which element of the two input vectors the result element gets. The
+/// shuffle mask is represented as an array of integers. Positive integers
+/// represent elements of the two input vectors. UndefMaskElem (-1)
+/// represents an "undef" element in the result.
+///
+/// For scalable vectors, all the elements of the mask must be 0 or -1. This
+/// requirement may be relaxed in the future.
class ShuffleVectorInst : public Instruction {
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
@@ -1993,10 +2006,15 @@
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr = "",
+ Instruction *InsertBefor = nullptr);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr, BasicBlock *InsertAtEnd);
- // allocate space for exactly three operands
+ // allocate space for exactly two operands
void *operator new(size_t s) {
- return User::operator new(s, 3);
+ return User::operator new(s, 2);
}
/// Swap the first 2 operands and adjust the mask to preserve the semantics
@@ -2007,6 +2025,8 @@
/// formed with the specified operands.
static bool isValidOperands(const Value *V1, const Value *V2,
const Value *Mask);
+ static bool isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask);
/// Overload to return most specific vector type.
///
@@ -2017,10 +2037,6 @@
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
- Constant *getMask() const {
- return cast<Constant>(getOperand(2));
- }
-
/// Return the shuffle mask value for the specified element of the mask.
/// Return -1 if the element is undef.
static int getMaskValue(const Constant *Mask, unsigned Elt);
@@ -2028,7 +2044,7 @@
/// Return the shuffle mask value of this instruction for the given element
/// index. Return -1 if the element is undef.
int getMaskValue(unsigned Elt) const {
- return getMaskValue(getMask(), Elt);
+ return ShuffleMask[Elt];
}
/// Convert the input shuffle mask operand to a vector of integers. Undefined
@@ -2039,13 +2055,21 @@
/// Return the mask for this instruction as a vector of integers. Undefined
/// elements of the mask are returned as -1.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
- return getShuffleMask(getMask(), Result);
+ Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
- SmallVector<int, 16> getShuffleMask() const {
- SmallVector<int, 16> Mask;
- getShuffleMask(Mask);
- return Mask;
+ /// Return the mask for this instruction, for use in bitcode.
+ ///
+ /// TODO: This is temporary until we decide a new bitcode encoding for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const {
+ return ShuffleMaskForBitcode;
+ }
+
+ void setShuffleMask(ArrayRef<int> Mask);
+
+ ArrayRef<int> getShuffleMask() const {
+ return ShuffleMask;
}
/// Return true if this shuffle returns a vector with a different number of
@@ -2054,7 +2078,7 @@
/// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
bool changesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts != NumMaskElts;
}
@@ -2063,7 +2087,7 @@
/// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
bool increasesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts < NumMaskElts;
}
@@ -2084,7 +2108,7 @@
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(getMask());
+ return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2105,7 +2129,7 @@
/// from its input vectors.
/// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
bool isIdentity() const {
- return !changesLength() && isIdentityMask(getShuffleMask());
+ return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2146,7 +2170,7 @@
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(getMask());
+ return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
@@ -2166,7 +2190,7 @@
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(getMask());
+ return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
@@ -2188,7 +2212,7 @@
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(getMask());
+ return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2237,7 +2261,7 @@
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(getMask());
+ return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is an extract subvector mask.
@@ -2256,7 +2280,7 @@
/// Return true if this shuffle mask is an extract subvector mask.
bool isExtractSubvectorMask(int &Index) const {
int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
- return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
+ return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
}
/// Change values in a shuffle permute mask assuming the two vector operands
@@ -2283,7 +2307,7 @@
template <>
struct OperandTraits<ShuffleVectorInst> :
- public FixedNumOperandTraits<ShuffleVectorInst, 3> {
+ public FixedNumOperandTraits<ShuffleVectorInst, 2> {
};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)
Index: llvm/include/llvm/IR/IRBuilder.h
===================================================================
--- llvm/include/llvm/IR/IRBuilder.h
+++ llvm/include/llvm/IR/IRBuilder.h
@@ -2540,17 +2540,25 @@
Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
const Twine &Name = "") {
- if (auto *V1C = dyn_cast<Constant>(V1))
- if (auto *V2C = dyn_cast<Constant>(V2))
- if (auto *MC = dyn_cast<Constant>(Mask))
- return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
- return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
+ return CreateShuffleVector(V1, V2, IntMask, Name);
}
- Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
const Twine &Name = "") {
- Value *Mask = ConstantDataVector::get(Context, IntMask);
- return CreateShuffleVector(V1, V2, Mask, Name);
+ SmallVector<int, 16> IntMask;
+ IntMask.assign(Mask.begin(), Mask.end());
+ return CreateShuffleVector(V1, V2, IntMask, Name);
+ }
+
+ /// See class ShuffleVectorInst for a description of the mask representation.
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name = "") {
+ if (auto *V1C = dyn_cast<Constant>(V1))
+ if (auto *V2C = dyn_cast<Constant>(V2))
+ return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
+ return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}
Value *CreateExtractValue(Value *Agg,
Index: llvm/include/llvm/IR/Constants.h
===================================================================
--- llvm/include/llvm/IR/Constants.h
+++ llvm/include/llvm/IR/Constants.h
@@ -1198,7 +1198,7 @@
Type *OnlyIfReducedTy = nullptr);
static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
Type *OnlyIfReducedTy = nullptr);
- static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
+ static Constant *getShuffleVector(Constant *V1, Constant *V2, ArrayRef<int> Mask,
Type *OnlyIfReducedTy = nullptr);
static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
Type *OnlyIfReducedTy = nullptr);
@@ -1217,6 +1217,16 @@
/// expression and return the list of indices.
ArrayRef<unsigned> getIndices() const;
+ /// Assert that this is an shufflevector and return the mask. See class
+ /// ShuffleVectorInst for a description of the mask representation.
+ ArrayRef<int> getShuffleMask() const;
+
+ /// Assert that this is an shufflevector and return the mask.
+ ///
+ /// TODO: This is a temporary hack until we update the bitcode format for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const;
+
/// Return a string representation for an opcode.
const char *getOpcodeName() const;
Index: llvm/include/llvm/IR/ConstantFolder.h
===================================================================
--- llvm/include/llvm/IR/ConstantFolder.h
+++ llvm/include/llvm/IR/ConstantFolder.h
@@ -260,7 +260,7 @@
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return ConstantExpr::getShuffleVector(V1, V2, Mask);
}
Index: llvm/include/llvm/Analysis/TargetFolder.h
===================================================================
--- llvm/include/llvm/Analysis/TargetFolder.h
+++ llvm/include/llvm/Analysis/TargetFolder.h
@@ -252,7 +252,7 @@
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
}
Index: llvm/include/llvm/Analysis/InstructionSimplify.h
===================================================================
--- llvm/include/llvm/Analysis/InstructionSimplify.h
+++ llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -230,7 +230,8 @@
const SimplifyQuery &Q);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+/// See class ShuffleVectorInst for a description of the mask representation.
+Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q);
//=== Helper functions for higher up the class hierarchy.
Index: llvm/include/llvm/Analysis/ConstantFolding.h
===================================================================
--- llvm/include/llvm/Analysis/ConstantFolding.h
+++ llvm/include/llvm/Analysis/ConstantFolding.h
@@ -119,10 +119,11 @@
Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
/// Attempt to constant fold a shufflevector instruction with the
-/// specified operands and indices. The constant result is returned if
-/// successful; if not, null is returned.
+/// specified operands and mask. See class ShuffleVectorInst for a description
+/// of the mask representation. The constant result is returned if successful;
+/// if not, null is returned.
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
/// produce if it is constant and determinable. If this is not determinable,
Index: llvm/include/llvm/ADT/ArrayRef.h
===================================================================
--- llvm/include/llvm/ADT/ArrayRef.h
+++ llvm/include/llvm/ADT/ArrayRef.h
@@ -535,10 +535,20 @@
}
template<typename T>
+ inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return ArrayRef<T>(LHS).equals(RHS);
+ }
+
+ template<typename T>
inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
+ template<typename T>
+ inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return !(LHS == RHS);
+ }
+
/// @}
template <typename T> hash_code hash_value(ArrayRef<T> S) {
Index: clang/lib/CodeGen/CGExpr.cpp
===================================================================
--- clang/lib/CodeGen/CGExpr.cpp
+++ clang/lib/CodeGen/CGExpr.cpp
@@ -1659,7 +1659,7 @@
// Shuffle vector to get vec3.
V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
- {0, 1, 2}, "extractVec");
+ ArrayRef<int>{0, 1, 2}, "extractVec");
return EmitFromMemory(V, Ty);
}
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
