efriedma updated this revision to Diff 237453.
efriedma added a comment.
Herald added subscribers: cfe-commits, Petar.Avramovic, dexonsmith, nhaehnle,
jvesely, arsenm.
Herald added a project: clang.
Add more comments, fixed the GlobalISel representation of shuffles, misc other
cleanups. Still haven't addressed the bitcode reader/writer issues.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D72467/new/
https://reviews.llvm.org/D72467
Files:
clang/lib/CodeGen/CGExpr.cpp
llvm/include/llvm/ADT/ArrayRef.h
llvm/include/llvm/Analysis/ConstantFolding.h
llvm/include/llvm/Analysis/InstructionSimplify.h
llvm/include/llvm/Analysis/TargetFolder.h
llvm/include/llvm/CodeGen/MachineFunction.h
llvm/include/llvm/CodeGen/MachineInstrBuilder.h
llvm/include/llvm/CodeGen/MachineOperand.h
llvm/include/llvm/IR/ConstantFolder.h
llvm/include/llvm/IR/Constants.h
llvm/include/llvm/IR/IRBuilder.h
llvm/include/llvm/IR/Instructions.h
llvm/include/llvm/IR/NoFolder.h
llvm/include/llvm/IR/PatternMatch.h
llvm/lib/Analysis/ConstantFolding.cpp
llvm/lib/Analysis/InstructionSimplify.cpp
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Analysis/ValueTracking.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
llvm/lib/CodeGen/MIRParser/MIParser.cpp
llvm/lib/CodeGen/MachineFunction.cpp
llvm/lib/CodeGen/MachineOperand.cpp
llvm/lib/CodeGen/MachineVerifier.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/IR/AsmWriter.cpp
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/ConstantFold.cpp
llvm/lib/IR/ConstantFold.h
llvm/lib/IR/Constants.cpp
llvm/lib/IR/ConstantsContext.h
llvm/lib/IR/Instruction.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/lib/Target/ARM/MVETailPredication.cpp
llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
llvm/lib/Transforms/Scalar/GVN.cpp
llvm/lib/Transforms/Scalar/GVNSink.cpp
llvm/lib/Transforms/Scalar/NewGVN.cpp
llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir
llvm/unittests/IR/PatternMatch.cpp
Index: llvm/unittests/IR/PatternMatch.cpp
===================================================================
--- llvm/unittests/IR/PatternMatch.cpp
+++ llvm/unittests/IR/PatternMatch.cpp
@@ -991,31 +991,31 @@
EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
// Test matching shufflevector
- EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
- EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
+ ArrayRef<int> Mask;
+ EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
+ EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
EXPECT_TRUE(A == VI3);
EXPECT_TRUE(B == VI4);
- EXPECT_TRUE(C == IdxVec);
A = B = C = nullptr; // reset
// Test matching the vector splat pattern
EXPECT_TRUE(match(
SI1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(A == Val);
}
Index: llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir
===================================================================
--- llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir
+++ llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir
@@ -135,7 +135,7 @@
; CHECK: liveins: $d0, $d1
; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
- ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, -1)
+ ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, undef)
; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>)
%0:_(<2 x s32>) = COPY $d0
%1:_(<2 x s32>) = COPY $d1
@@ -277,7 +277,7 @@
; CHECK: liveins: $q0, $q1
; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
- ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, -1, -1)
+ ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, undef, undef)
; CHECK: RET_ReallyLR implicit [[SHUF]](<12 x s32>)
%0:_(<4 x s32>) = COPY $q0
%1:_(<4 x s32>) = COPY $q1
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3370,7 +3370,7 @@
auto *O1 = B.CreateZExtOrTrunc(
SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
- NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
+ NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
} else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
// Don't do anything with the operands, just extend the result.
continue;
Index: llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -987,7 +987,7 @@
auto *SV = cast<ShuffleVectorInst>(I);
UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
std::string Name = suffixed_name_or(I, ".base", "base_sv");
- return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
+ return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
Name, SV);
}
};
Index: llvm/lib/Transforms/Scalar/NewGVN.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -1179,6 +1179,8 @@
E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
return SimplifiedE;
+ } else if (isa<ShuffleVectorInst>(I)) {
+ llvm_unreachable("TODO: Represent shuffle mask in expression");
} else if (AllConstant) {
// We don't bother trying to simplify unless all of the operands
// were constant.
Index: llvm/lib/Transforms/Scalar/GVNSink.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -350,6 +350,7 @@
class InstructionUseExpr : public GVNExpression::BasicExpression {
unsigned MemoryUseOrder = -1;
bool Volatile = false;
+ std::vector<int> ShuffleMask;
public:
InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
@@ -366,15 +367,18 @@
void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; }
void setVolatile(bool V) { Volatile = V; }
+ void setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ }
hash_code getHashValue() const override {
return hash_combine(GVNExpression::BasicExpression::getHashValue(),
- MemoryUseOrder, Volatile);
+ MemoryUseOrder, Volatile, ArrayRef<int>(ShuffleMask));
}
template <typename Function> hash_code getHashValue(Function MapFn) {
- hash_code H =
- hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
+ hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder,
+ Volatile, ArrayRef<int>(ShuffleMask));
for (auto *V : operands())
H = hash_combine(H, MapFn(V));
return H;
@@ -402,6 +406,8 @@
CmpInst::Predicate Predicate = C->getPredicate();
E->setOpcode((C->getOpcode() << 8) | Predicate);
}
+ if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
+ E->setShuffleMask(SVI->getShuffleMask());
return E;
}
Index: llvm/lib/Transforms/Scalar/GVN.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/GVN.cpp
+++ llvm/lib/Transforms/Scalar/GVN.cpp
@@ -301,6 +301,9 @@
for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
II != IE; ++II)
e.varargs.push_back(*II);
+ } else if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) {
+ ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
+ e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
}
return e;
Index: llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
===================================================================
--- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1870,10 +1870,9 @@
}
void visitShuffleVectorInst(ShuffleVectorInst &I) {
- insertShadowCheck(I.getOperand(2), &I);
IRBuilder<> IRB(&I);
setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
- I.getOperand(2), "_msprop"));
+ I.getShuffleMask(), "_msprop"));
setOriginForNaryOp(I);
}
Index: llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1430,9 +1430,9 @@
// narrow binop on each pair of the source operands followed by concatenation
// of the results.
Value *L0, *L1, *R0, *R1;
- Constant *Mask;
- if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
+ ArrayRef<int> Mask;
+ if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
+ match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
LHS->hasOneUse() && RHS->hasOneUse() &&
cast<ShuffleVectorInst>(LHS)->isConcat() &&
cast<ShuffleVectorInst>(RHS)->isConcat()) {
@@ -1456,7 +1456,7 @@
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
- auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
+ auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
Value *XY = Builder.CreateBinOp(Opcode, X, Y);
if (auto *BO = dyn_cast<BinaryOperator>(XY))
BO->copyIRFlags(&Inst);
@@ -1466,8 +1466,8 @@
// If both arguments of the binary operation are shuffles that use the same
// mask and shuffle within a single vector, move the shuffle after the binop.
Value *V1, *V2;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
+ match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
// Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
@@ -1477,17 +1477,17 @@
// If both arguments of a commutative binop are select-shuffles that use the
// same mask with commuted operands, the shuffles are unnecessary.
if (Inst.isCommutative() &&
- match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
+ match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
- m_Specific(Mask)))) {
+ m_SpecificMask(Mask)))) {
auto *LShuf = cast<ShuffleVectorInst>(LHS);
auto *RShuf = cast<ShuffleVectorInst>(RHS);
// TODO: Allow shuffles that contain undefs in the mask?
// That is legal, but it reduces undef knowledge.
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
- if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
- RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
+ if (LShuf->isSelect() && !is_contained(LShuf->getShuffleMask(), -1) &&
+ RShuf->isSelect() && !is_contained(RShuf->getShuffleMask(), -1)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
@@ -1505,7 +1505,7 @@
// transforms.
Constant *C;
if (match(&Inst, m_c_BinOp(
- m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
+ m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))),
m_Constant(C))) &&
V1->getType()->getVectorNumElements() <= NumElts) {
assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
@@ -1517,8 +1517,7 @@
// reorder is not possible. A 1-to-1 mapping is not required. Example:
// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
bool ConstOp1 = isa<Constant>(RHS);
- SmallVector<int, 16> ShMask;
- ShuffleVectorInst::getShuffleMask(Mask, ShMask);
+ ArrayRef<int> ShMask = Mask;
unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
Index: llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -723,7 +723,7 @@
}
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
- int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
+ int MaskSize = Shuf.getShuffleMask().size();
int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
// A vector select does not change the size of the operands.
@@ -835,13 +835,10 @@
// inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
// --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *Zero = ConstantInt::getNullValue(I32Ty);
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (unsigned i = 0; i != NumMaskElts; ++i)
- NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
+ NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
}
@@ -871,26 +868,23 @@
// For example:
// inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
- Constant *OldMask = Shuf->getMask();
+ SmallVector<int, 16> NewMask(NumMaskElts);
+ ArrayRef<int> OldMask = Shuf->getShuffleMask();
for (unsigned i = 0; i != NumMaskElts; ++i) {
if (i != IdxC) {
// All mask elements besides the inserted element remain the same.
- NewMaskVec[i] = OldMask->getAggregateElement(i);
- } else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
+ NewMask[i] = OldMask[i];
+ } else if (OldMask[i] == (int)IdxC) {
// If the mask element was already set, there's nothing to do
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
+ assert(OldMask[i] == -1 &&
"Unexpected shuffle mask element for identity shuffle");
- NewMaskVec[i] = NewMaskEltC;
+ NewMask[i] = IdxC;
}
}
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
}
@@ -958,19 +952,18 @@
// mask vector with the insertelt index plus the length of the vector
// (because the constant vector operand of a shuffle is always the 2nd
// operand).
- Constant *Mask = Shuf->getMask();
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask = Shuf->getShuffleMask();
+ unsigned NumElts = Mask.size();
SmallVector<Constant *, 16> NewShufElts(NumElts);
- SmallVector<Constant *, 16> NewMaskElts(NumElts);
+ SmallVector<int, 16> NewMaskElts(NumElts);
for (unsigned I = 0; I != NumElts; ++I) {
if (I == InsEltIndex) {
NewShufElts[I] = InsEltScalar;
- Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
- NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
+ NewMaskElts[I] = InsEltIndex + NumElts;
} else {
// Copy over the existing values.
NewShufElts[I] = ShufConstVec->getAggregateElement(I);
- NewMaskElts[I] = Mask->getAggregateElement(I);
+ NewMaskElts[I] = Mask[I];
}
}
@@ -978,7 +971,7 @@
// original insertelt. The old shuffle will be dead now.
return new ShuffleVectorInst(Shuf->getOperand(0),
ConstantVector::get(NewShufElts),
- ConstantVector::get(NewMaskElts));
+ NewMaskElts);
} else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
// Transform sequences of insertelements ops with constant data/indexes into
// a single shuffle op.
@@ -1397,7 +1390,7 @@
// Shuffles to: |EE|FF|GG|HH|
// +--+--+--+--+
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
- SmallVector<int, 16> &Mask) {
+ ArrayRef<int> Mask) {
unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
unsigned MaskElems = Mask.size();
unsigned BegIdx = Mask.front();
@@ -1480,12 +1473,12 @@
// Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
// Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
// The existing binop constant vector remains in the same operand position.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, -1) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -1499,7 +1492,7 @@
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, -1) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1511,14 +1504,15 @@
static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
InstCombiner::BuilderTy &Builder) {
Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Value *X;
uint64_t IndexC;
// Match a shuffle that is a splat to a non-zero element.
if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
m_ConstantInt(IndexC)))) ||
- !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
+ !match(Op1, m_Undef()) ||
+ all_of(Mask, [](int Elt) { return Elt == 0; }) || IndexC == 0)
return nullptr;
// Insert into element 0 of an undef vector.
@@ -1531,12 +1525,12 @@
// shuf (inselt undef, X, 2), undef, <2,2,undef>
// --> shuf (inselt undef, X, 0), undef, <0,0,undef>
unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
+ SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (isa<UndefValue>(Mask->getAggregateElement(i)))
- NewMask[i] = Mask->getAggregateElement(i);
+ if (Mask[i] == -1)
+ NewMask[i] = Mask[i];
- return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
}
/// Try to fold shuffles that are the equivalent of a vector select.
@@ -1605,14 +1599,14 @@
BinaryOperator::BinaryOps BOpc = Opc0;
// Select the constant elements needed for the single binop.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
// We are moving a binop after a shuffle. When a shuffle has an undefined
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, -1) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
@@ -1661,7 +1655,7 @@
NewBO->andIRFlags(B1);
if (DropNSW)
NewBO->setHasNoSignedWrap(false);
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, -1) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -1687,8 +1681,7 @@
// and have the same number of elements as this shuffle.
unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
Value *NarrowCond;
- if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
- m_Constant()))) ||
+ if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
!cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
return nullptr;
@@ -1696,8 +1689,8 @@
// shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
// sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
Value *Undef = UndefValue::get(X->getType());
- Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
- Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
+ Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
+ Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
@@ -1708,9 +1701,13 @@
return nullptr;
Value *X, *Y;
- Constant *Mask;
- if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
+ ArrayRef<int> Mask;
+ ShuffleVectorInst *Op0Shuf = dyn_cast<ShuffleVectorInst>(Op0);
+ if (!Op0Shuf)
return nullptr;
+ X = Op0Shuf->getOperand(0);
+ Y = Op0Shuf->getOperand(1);
+ Mask = Op0Shuf->getShuffleMask();
// Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
// then combining may result in worse codegen.
@@ -1729,23 +1726,24 @@
// shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
// shuf X, Y, <C0, undef, C2, undef>
unsigned NumElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumElts);
- assert(NumElts < Mask->getType()->getVectorNumElements() &&
+ SmallVector<int, 16> NewMask(NumElts);
+ assert(NumElts < Mask.size() &&
"Identity with extract must have less elements than its inputs");
for (unsigned i = 0; i != NumElts; ++i) {
- Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
- Constant *MaskElt = Mask->getAggregateElement(i);
- NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
+ int ExtractMaskElt = Shuf.getMaskValue(i);
+ int MaskElt = Mask[i];
+ NewMask[i] = ExtractMaskElt == -1 ? ExtractMaskElt : MaskElt;
}
- return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(X, Y, NewMask);
}
/// Try to replace a shuffle with an insertelement or try to replace a shuffle
/// operand with the operand of an insertelement.
static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf) {
Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ SmallVector<int, 16> Mask;
+ Mask.assign(Shuf.getShuffleMask().begin(), Shuf.getShuffleMask().end());
// The shuffle must not change vector sizes.
// TODO: This restriction could be removed if the insert has only one use
@@ -1868,7 +1866,7 @@
assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
@@ -1902,29 +1900,27 @@
Value *LHS = SVI.getOperand(0);
Value *RHS = SVI.getOperand(1);
if (auto *V = SimplifyShuffleVectorInst(
- LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
+ LHS, RHS, SVI.getShuffleMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
return replaceInstUsesWith(SVI, V);
// shuffle x, x, mask --> shuffle x, undef, mask'
unsigned VWidth = SVI.getType()->getVectorNumElements();
unsigned LHSWidth = LHS->getType()->getVectorNumElements();
- SmallVector<int, 16> Mask = SVI.getShuffleMask();
+ ArrayRef<int> Mask = SVI.getShuffleMask();
Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
if (LHS == RHS) {
assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
// Remap any references to RHS to use LHS.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) {
// Propagate undef elements or force mask to LHS.
if (Mask[i] < 0)
- Elts.push_back(UndefValue::get(Int32Ty));
+ Elts.push_back(-1);
else
- Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
+ Elts.push_back(Mask[i] % LHSWidth);
}
- SVI.setOperand(0, SVI.getOperand(1));
- SVI.setOperand(1, UndefValue::get(RHS->getType()));
- SVI.setOperand(2, ConstantVector::get(Elts));
- return &SVI;
+ return new ShuffleVectorInst(SVI.getOperand(1),
+ UndefValue::get(RHS->getType()), Elts);
}
// shuffle undef, x, mask --> shuffle x, undef, mask'
@@ -2149,8 +2145,8 @@
if (newLHS == LHS && newRHS == RHS)
return MadeChange ? &SVI : nullptr;
- SmallVector<int, 16> LHSMask;
- SmallVector<int, 16> RHSMask;
+ ArrayRef<int> LHSMask;
+ ArrayRef<int> RHSMask;
if (newLHS != LHS)
LHSMask = LHSShuffle->getShuffleMask();
if (RHSShuffle && newRHS != RHS)
Index: llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1368,15 +1368,14 @@
}
if (NewUndefElts) {
// Add additional discovered undefs.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
- Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
+ Elts.push_back(-1);
else
- Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
- Shuffle->getMaskValue(i)));
+ Elts.push_back(Shuffle->getMaskValue(i));
}
- I->setOperand(2, ConstantVector::get(Elts));
+ Shuffle->setShuffleMask(Elts);
MadeChange = true;
}
break;
Index: llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2792,11 +2792,11 @@
return nullptr;
Value *Vec;
- Constant *Mask;
+ ArrayRef<int> Mask;
if (match(BCSrcOp,
- m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
+ m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
// Check whether every element of Mask is the same constant
- if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
+ if (is_splat(Mask)) {
auto *VecTy = cast<VectorType>(BCSrcOp->getType());
auto *EltTy = cast<IntegerType>(VecTy->getElementType());
if (C->isSplat(EltTy->getBitWidth())) {
@@ -2805,6 +2805,7 @@
// then:
// => %E = extractelement <N x iK> %vec, i32 Elem
// icmp <pred> iK %SplatVal, <pattern>
+ Value *Elem = Builder.getInt32(Mask[0]);
Value *Extract = Builder.CreateExtractElement(Vec, Elem);
Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
return new ICmpInst(Pred, Extract, NewC);
@@ -5372,9 +5373,9 @@
// shuffle within a single vector, move the shuffle after the cmp.
Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
Value *V1, *V2;
- Constant *M;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
+ ArrayRef<int> M;
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))) &&
+ match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse())) {
// cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
Index: llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -639,12 +639,12 @@
InstCombiner::BuilderTy &Builder) {
auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
- Shuf->getMask()->getSplatValue() &&
+ is_splat(Shuf->getShuffleMask()) &&
Shuf->getType() == Shuf->getOperand(0)->getType()) {
// trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
Constant *NarrowUndef = UndefValue::get(Trunc.getType());
Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
- return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
+ return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
}
return nullptr;
@@ -2486,7 +2486,7 @@
Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
// Return a new shuffle vector. Use the same element ID's, as we
// know the vector types match #elts.
- return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
+ return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
}
}
Index: llvm/lib/Target/ARM/MVETailPredication.cpp
===================================================================
--- llvm/lib/Target/ARM/MVETailPredication.cpp
+++ llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -217,7 +217,7 @@
// The invariant shuffle that broadcast the limit into a vector.
Instruction *Insert = nullptr;
if (!match(Shuffle, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
- m_Zero())))
+ m_ZeroMask())))
return false;
// Insert the limit into a vector.
@@ -253,7 +253,7 @@
// The shuffle which broadcasts the index iv into a vector.
if (!match(BroadcastSplat, m_ShuffleVector(m_Instruction(Insert), m_Undef(),
- m_Zero())))
+ m_ZeroMask())))
return false;
// The insert element which initialises a vector with the index iv.
Index: llvm/lib/Target/ARM/ARMISelLowering.cpp
===================================================================
--- llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -14904,7 +14904,7 @@
return false;
if (!match(I->getOperand(Op),
m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
- m_Undef(), m_Zero()))) {
+ m_Undef(), m_ZeroMask()))) {
return false;
}
Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));
Index: llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -409,7 +409,7 @@
DL->getTypeSizeInBits(Val->getType())) {
assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
- { 0, 1, 2 });
+ ArrayRef<int>{0, 1, 2});
}
Val = B.CreateBitCast(Val, EffectiveEltTy);
Index: llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -210,7 +210,7 @@
Arg.replaceAllUsesWith(NewVal);
} else if (IsV3) {
Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
- {0, 1, 2},
+ ArrayRef<int>{0, 1, 2},
Arg.getName() + ".load");
Arg.replaceAllUsesWith(Shuf);
} else {
Index: llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
===================================================================
--- llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
+++ llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
@@ -3672,8 +3672,8 @@
return false;
// The shuffle's second operand doesn't matter if the mask is all zero.
- const Constant *Mask = I.getOperand(3).getShuffleMask();
- if (!isa<ConstantAggregateZero>(Mask))
+ ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();
+ if (!all_of(Mask, [](int Elem) { return Elem == 0; }))
return false;
// We're done, now find out what kind of splat we need.
@@ -3722,15 +3722,12 @@
const LLT Src1Ty = MRI.getType(Src1Reg);
Register Src2Reg = I.getOperand(2).getReg();
const LLT Src2Ty = MRI.getType(Src2Reg);
- const Constant *ShuffleMask = I.getOperand(3).getShuffleMask();
+ ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
LLVMContext &Ctx = MF.getFunction().getContext();
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(ShuffleMask, Mask);
-
// G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
// it's originated from a <1 x T> type. Those should have been lowered into
// G_BUILD_VECTOR earlier.
Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
===================================================================
--- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8696,10 +8696,10 @@
return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
};
- Constant *M1, *M2;
+ ArrayRef<int> M1, M2;
Value *S1Op1, *S2Op1;
- if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
- !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+ if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
+ !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
return false;
// Check that the operands are half as wide as the result and we extract
Index: llvm/lib/IR/Verifier.cpp
===================================================================
--- llvm/lib/IR/Verifier.cpp
+++ llvm/lib/IR/Verifier.cpp
@@ -3274,7 +3274,7 @@
void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
- SV.getOperand(2)),
+ SV.getShuffleMask()),
"Invalid shufflevector operands!", &SV);
visitInstruction(SV);
}
Index: llvm/lib/IR/Instructions.cpp
===================================================================
--- llvm/lib/IR/Instructions.cpp
+++ llvm/lib/IR/Instructions.cpp
@@ -1795,11 +1795,14 @@
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertBefore) {
- assert(isValidOperands(V1, V2, Mask) &&
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ assert(isValidOperands(V1, V2, MaskArr) &&
"Invalid shuffle vector instruction operands!");
+
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(MaskArr);
setName(Name);
}
@@ -1812,35 +1815,90 @@
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertAtEnd) {
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ assert(isValidOperands(V1, V2, MaskArr) &&
+ "Invalid shuffle vector instruction operands!");
+
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(MaskArr);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name,
+ Instruction *InsertBefore)
+: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector,
+ OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this),
+ InsertBefore) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(Mask);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name,
+ BasicBlock *InsertAtEnd)
+: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector,
+ OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this),
+ InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(Mask);
setName(Name);
}
void ShuffleVectorInst::commute() {
int NumOpElts = Op<0>()->getType()->getVectorNumElements();
- int NumMaskElts = getMask()->getType()->getVectorNumElements();
- SmallVector<Constant*, 16> NewMask(NumMaskElts);
- Type *Int32Ty = Type::getInt32Ty(getContext());
+ int NumMaskElts = ShuffleMask.size();
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
if (MaskElt == -1) {
- NewMask[i] = UndefValue::get(Int32Ty);
+ NewMask[i] = -1;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
- NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
+ NewMask[i] = MaskElt;
}
- Op<2>() = ConstantVector::get(NewMask);
+ setShuffleMask(NewMask);
Op<0>().swap(Op<1>());
}
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask) {
+ // V1 and V2 must be vectors of the same type.
+ if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
+ return false;
+
+ // Make sure the mask elements make sense.
+ int V1Size = cast<VectorType>(V1->getType())->getNumElements();
+ for (int Elem : Mask)
+ if (Elem != -1 && Elem >= V1Size * 2)
+ return false;
+
+ if (V1->getType()->getVectorIsScalable())
+ if ((Mask[0] != 0 && Mask[0] != -1) || !is_splat(Mask))
+ return false;
+
+ return true;
+}
+
+bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
const Value *Mask) {
// V1 and V2 must be vectors of the same type.
if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
@@ -1876,14 +1934,6 @@
return true;
}
- // The bitcode reader can create a place holder for a forward reference
- // used as the shuffle mask. When this occurs, the shuffle mask will
- // fall into this case and fail. To avoid this error, do this bit of
- // ugliness to allow such a mask pass.
- if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
- if (CE->getOpcode() == Instruction::UserOp1)
- return true;
-
return false;
}
@@ -1913,6 +1963,19 @@
}
}
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ SmallVector<Constant*, 16> MaskConst;
+ Type *Int32Ty = Type::getInt32Ty(getContext());
+ for (int Elem : Mask) {
+ if (Elem == -1)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ ShuffleMaskForBitcode = ConstantVector::get(MaskConst);
+}
+
static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
assert(!Mask.empty() && "Shuffle mask must contain elements");
bool UsesLHS = false;
@@ -2062,7 +2125,7 @@
return false;
// The first part of the mask must choose elements from exactly 1 source op.
- SmallVector<int, 16> Mask = getShuffleMask();
+ ArrayRef<int> Mask = getShuffleMask();
if (!isIdentityMaskImpl(Mask, NumOpElts))
return false;
@@ -4262,7 +4325,7 @@
}
ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
- return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
+ return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
}
PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
Index: llvm/lib/IR/Instruction.cpp
===================================================================
--- llvm/lib/IR/Instruction.cpp
+++ llvm/lib/IR/Instruction.cpp
@@ -434,6 +434,8 @@
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
+ return SVI->getShuffleMask() == cast<ShuffleVectorInst>(I2)->getShuffleMask();
return true;
}
Index: llvm/lib/IR/ConstantsContext.h
===================================================================
--- llvm/lib/IR/ConstantsContext.h
+++ llvm/lib/IR/ConstantsContext.h
@@ -146,20 +146,34 @@
/// shufflevector constant exprs.
class ShuffleVectorConstantExpr : public ConstantExpr {
public:
- ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
+ ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<unsigned> Mask)
: ConstantExpr(VectorType::get(
cast<VectorType>(C1->getType())->getElementType(),
- cast<VectorType>(C3->getType())->getElementCount()),
+ Mask.size(), C1->getType()->getVectorIsScalable()),
Instruction::ShuffleVector,
- &Op<0>(), 3) {
+ &Op<0>(), 2) {
Op<0>() = C1;
Op<1>() = C2;
- Op<2>() = C3;
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ //assert(ShuffleVectorInst::isValidOperands(C1, C2, ShuffleMask) &&
+ // "Invalid shuffle vector instruction operands!");
+ SmallVector<Constant*, 16> MaskConst;
+ Type *Int32Ty = Type::getInt32Ty(getContext());
+ for (int Elem : ShuffleMask) {
+ if (Elem == -1)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ ShuffleMaskForBitcode = ConstantVector::get(MaskConst);
}
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
// allocate space for exactly three operands
void *operator new(size_t s) {
- return User::operator new(s, 3);
+ return User::operator new(s, 2);
}
/// Transparently provide more efficient getOperand methods.
@@ -319,7 +333,7 @@
template <>
struct OperandTraits<ShuffleVectorConstantExpr>
- : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
+ : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
template <>
@@ -546,7 +560,7 @@
case Instruction::InsertElement:
return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Indexes);
case Instruction::InsertValue:
return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
case Instruction::ExtractValue:
Index: llvm/lib/IR/Constants.cpp
===================================================================
--- llvm/lib/IR/Constants.cpp
+++ llvm/lib/IR/Constants.cpp
@@ -1266,6 +1266,14 @@
return cast<CompareConstantExpr>(this)->predicate;
}
+ArrayRef<int> ConstantExpr::getShuffleMask() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
+}
+
+Constant *ConstantExpr::getShuffleMaskForBitcode() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
+}
+
Constant *
ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
assert(Op->getType() == getOperand(OpNo)->getType() &&
@@ -2222,22 +2230,32 @@
Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
Constant *Mask, Type *OnlyIfReducedTy) {
+ SmallVector<int, 16> ShuffleMask;
+ ShuffleVectorInst::getShuffleMask(Mask, ShuffleMask);
+ return getShuffleVector(V1, V2, ShuffleMask);
+}
+
+Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
+ ArrayRef<int> Mask, Type *OnlyIfReducedTy) {
assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector constant expr operands!");
if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
return FC; // Fold a few common cases.
- ElementCount NElts = Mask->getType()->getVectorElementCount();
+ unsigned NElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
- Type *ShufTy = VectorType::get(EltTy, NElts);
+ bool TypeIsScalable = V1->getType()->getVectorIsScalable();
+ Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
if (OnlyIfReducedTy == ShufTy)
return nullptr;
// Look up the constant in the table first to ensure uniqueness
- Constant *ArgVec[] = { V1, V2, Mask };
- const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
+ Constant *ArgVec[] = { V1, V2 };
+ SmallVector<unsigned, 16> MaskUnsigned;
+ MaskUnsigned.assign(Mask.begin(), Mask.end());
+ const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, MaskUnsigned);
LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
@@ -3076,7 +3094,7 @@
case Instruction::ExtractValue:
return ExtractValueInst::Create(Ops[0], getIndices());
case Instruction::ShuffleVector:
- return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
case Instruction::GetElementPtr: {
const auto *GO = cast<GEPOperator>(this);
Index: llvm/lib/IR/ConstantFold.h
===================================================================
--- llvm/lib/IR/ConstantFold.h
+++ llvm/lib/IR/ConstantFold.h
@@ -38,7 +38,7 @@
Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
Constant *Idx);
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
ArrayRef<unsigned> Idxs);
Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
Index: llvm/lib/IR/ConstantFold.cpp
===================================================================
--- llvm/lib/IR/ConstantFold.cpp
+++ llvm/lib/IR/ConstantFold.cpp
@@ -862,17 +862,14 @@
Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
Constant *V2,
- Constant *Mask) {
- unsigned MaskNumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask) {
+ unsigned MaskNumElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
// Undefined shuffle mask -> undefined value.
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elt) { return Elt == -1; }))
return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
- // Don't break the bitcode reader hack.
- if (isa<ConstantExpr>(Mask)) return nullptr;
-
// Do not iterate on scalable vector. The num of elements is unknown at
// compile-time.
VectorType *ValTy = cast<VectorType>(V1->getType());
@@ -884,7 +881,7 @@
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
for (unsigned i = 0; i != MaskNumElts; ++i) {
- int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
+ int Elt = Mask[i];
if (Elt == -1) {
Result.push_back(UndefValue::get(EltTy));
continue;
Index: llvm/lib/IR/AutoUpgrade.cpp
===================================================================
--- llvm/lib/IR/AutoUpgrade.cpp
+++ llvm/lib/IR/AutoUpgrade.cpp
@@ -2309,11 +2309,12 @@
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, 1);
if (NumSrcElts == 2)
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 0, 1 });
+ Rep = Builder.CreateShuffleVector(
+ Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
else
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 2, 3, 0, 1, 2, 3 });
+ Rep =
+ Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
+ ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
Index: llvm/lib/IR/AsmWriter.cpp
===================================================================
--- llvm/lib/IR/AsmWriter.cpp
+++ llvm/lib/IR/AsmWriter.cpp
@@ -462,6 +462,33 @@
isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
}
+static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
+ Out << ", <";
+ if (Ty->getVectorIsScalable())
+ Out << "vscale x ";
+ Out << Mask.size() << " x i32> ";
+ bool FirstElt = true;
+ if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
+ Out << "zeroinitializer";
+ } else if (all_of(Mask, [](int Elt) { return Elt == -1; })) {
+ Out << "undef";
+ } else {
+ Out << "<";
+ for (int Elt : Mask) {
+ if (FirstElt)
+ FirstElt = false;
+ else
+ Out << ", ";
+ Out << "i32 ";
+ if (Elt == -1)
+ Out << "undef";
+ else
+ Out << Elt;
+ }
+ Out << ">";
+ }
+}
+
namespace {
class TypePrinting {
@@ -1545,6 +1572,9 @@
TypePrinter.print(CE->getType(), Out);
}
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
+
Out << ')';
return;
}
@@ -4071,6 +4101,8 @@
RMWI->getSyncScopeID());
} else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
+ } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
+ PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
}
// Print Metadata info.
Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
===================================================================
--- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3543,14 +3543,19 @@
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
- Constant *MaskV = cast<Constant>(I.getOperand(2));
+ ArrayRef<int> Mask;
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();
unsigned SrcNumElts = SrcVT.getVectorNumElements();
- if (MaskV->isNullValue() && VT.isScalableVector()) {
+ if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
+ VT.isScalableVector()) {
// Canonical splat form of first element of first input vector.
SDValue FirstElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
SrcVT.getScalarType(), Src1,
@@ -3565,8 +3570,6 @@
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MaskV, Mask);
unsigned MaskNumElts = Mask.size();
if (SrcNumElts == MaskNumElts) {
Index: llvm/lib/CodeGen/MachineVerifier.cpp
===================================================================
--- llvm/lib/CodeGen/MachineVerifier.cpp
+++ llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1407,18 +1407,6 @@
break;
}
- const Constant *Mask = MaskOp.getShuffleMask();
- auto *MaskVT = dyn_cast<VectorType>(Mask->getType());
- if (!MaskVT || !MaskVT->getElementType()->isIntegerTy(32)) {
- report("Invalid shufflemask constant type", MI);
- break;
- }
-
- if (!Mask->getAggregateElement(0u)) {
- report("Invalid shufflemask constant type", MI);
- break;
- }
-
LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg());
LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg());
@@ -1434,8 +1422,7 @@
int SrcNumElts = Src0Ty.isVector() ? Src0Ty.getNumElements() : 1;
int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
- SmallVector<int, 32> MaskIdxes;
- ShuffleVectorInst::getShuffleMask(Mask, MaskIdxes);
+ ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
report("Wrong result type for shufflemask", MI);
Index: llvm/lib/CodeGen/MachineOperand.cpp
===================================================================
--- llvm/lib/CodeGen/MachineOperand.cpp
+++ llvm/lib/CodeGen/MachineOperand.cpp
@@ -940,13 +940,13 @@
}
case MachineOperand::MO_ShuffleMask:
OS << "shufflemask(";
- const Constant* C = getShuffleMask();
- const int NumElts = C->getType()->getVectorNumElements();
-
+ ArrayRef<int> Mask = getShuffleMask();
StringRef Separator;
- for (int I = 0; I != NumElts; ++I) {
- OS << Separator;
- C->getAggregateElement(I)->printAsOperand(OS, false, MST);
+ for (int Elt : Mask) {
+ if (Elt == -1)
+ OS << Separator << "undef";
+ else
+ OS << Separator << Elt;
Separator = ", ";
}
Index: llvm/lib/CodeGen/MachineFunction.cpp
===================================================================
--- llvm/lib/CodeGen/MachineFunction.cpp
+++ llvm/lib/CodeGen/MachineFunction.cpp
@@ -484,6 +484,10 @@
return Mask;
}
+MutableArrayRef<int> MachineFunction::allocateShuffleMask(unsigned NumElems) {
+ return {Allocator.Allocate<int>(NumElems), NumElems};
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void MachineFunction::dump() const {
print(dbgs());
Index: llvm/lib/CodeGen/MIRParser/MIParser.cpp
===================================================================
--- llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -2303,23 +2303,13 @@
if (expectAndConsume(MIToken::lparen))
return error("expected syntax shufflemask(<integer or undef>, ...)");
- SmallVector<Constant *, 32> ShufMask;
- LLVMContext &Ctx = MF.getFunction().getContext();
- Type *I32Ty = Type::getInt32Ty(Ctx);
-
- bool AllZero = true;
- bool AllUndef = true;
-
+ SmallVector<int, 32> ShufMask;
do {
if (Token.is(MIToken::kw_undef)) {
- ShufMask.push_back(UndefValue::get(I32Ty));
- AllZero = false;
+ ShufMask.push_back(-1);
} else if (Token.is(MIToken::IntegerLiteral)) {
- AllUndef = false;
const APSInt &Int = Token.integerValue();
- if (!Int.isNullValue())
- AllZero = false;
- ShufMask.push_back(ConstantInt::get(I32Ty, Int.getExtValue()));
+ ShufMask.push_back(Int.getExtValue());
} else
return error("expected integer constant");
@@ -2329,13 +2319,9 @@
if (expectAndConsume(MIToken::rparen))
return error("shufflemask should be terminated by ')'.");
- if (AllZero || AllUndef) {
- VectorType *VT = VectorType::get(I32Ty, ShufMask.size());
- Constant *C = AllZero ? Constant::getNullValue(VT) : UndefValue::get(VT);
- Dest = MachineOperand::CreateShuffleMask(C);
- } else
- Dest = MachineOperand::CreateShuffleMask(ConstantVector::get(ShufMask));
-
+ MutableArrayRef<int> MaskAlloc = MF.allocateShuffleMask(ShufMask.size());
+ llvm::copy(ShufMask, MaskAlloc.begin());
+ Dest = MachineOperand::CreateShuffleMask(MaskAlloc);
return false;
}
Index: llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
===================================================================
--- llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4134,10 +4134,7 @@
LLT DstTy = MRI.getType(DstReg);
LLT IdxTy = LLT::scalar(32);
- const Constant *ShufMask = MI.getOperand(3).getShuffleMask();
-
- SmallVector<int, 32> Mask;
- ShuffleVectorInst::getShuffleMask(ShufMask, Mask);
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
if (DstTy.isScalar()) {
if (Src0Ty.isVector())
Index: llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
===================================================================
--- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1922,11 +1922,16 @@
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
+ ArrayRef<int> Mask;
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&U))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(U).getShuffleMask();
MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
.addDef(getOrCreateVReg(U))
.addUse(getOrCreateVReg(*U.getOperand(0)))
.addUse(getOrCreateVReg(*U.getOperand(1)))
- .addShuffleMask(cast<Constant>(U.getOperand(2)));
+ .addShuffleMask(Mask);
return true;
}
Index: llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
===================================================================
--- llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -239,8 +239,7 @@
// vectors.
unsigned NumConcat = DstNumElts / SrcNumElts;
SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask);
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
for (unsigned i = 0; i != DstNumElts; ++i) {
int Idx = Mask[i];
// Undef value.
Index: llvm/lib/CodeGen/CodeGenPrepare.cpp
===================================================================
--- llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6216,7 +6216,7 @@
}
static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
- SmallVector<int, 16> Mask(SVI->getShuffleMask());
+ ArrayRef<int> Mask(SVI->getShuffleMask());
int SplatElem = -1;
for (unsigned i = 0; i < Mask.size(); ++i) {
if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
@@ -6266,7 +6266,7 @@
assert(InsertPt != UserBB->end());
InsertedShuffle =
new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getOperand(2), "", &*InsertPt);
+ SVI->getShuffleMask(), "", &*InsertPt);
InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
}
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2504,7 +2504,7 @@
}
Record.push_back(VE.getValueID(C->getOperand(0)));
Record.push_back(VE.getValueID(C->getOperand(1)));
- Record.push_back(VE.getValueID(C->getOperand(2)));
+ Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
break;
case Instruction::ICmp:
case Instruction::FCmp:
@@ -2691,7 +2691,7 @@
Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
pushValueAndType(I.getOperand(0), InstID, Vals);
pushValue(I.getOperand(1), InstID, Vals);
- pushValue(I.getOperand(2), InstID, Vals);
+ pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID, Vals);
break;
case Instruction::ICmp:
case Instruction::FCmp: {
Index: llvm/lib/Analysis/VectorUtils.cpp
===================================================================
--- llvm/lib/Analysis/VectorUtils.cpp
+++ llvm/lib/Analysis/VectorUtils.cpp
@@ -320,7 +320,7 @@
Value *Splat;
if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
m_ZeroInt()),
- m_Value(), m_ZeroInt())))
+ m_Value(), m_ZeroMask())))
return Splat;
return nullptr;
@@ -342,9 +342,9 @@
}
// FIXME: Constant splat analysis does not allow undef elements.
- Constant *Mask;
- if (match(V, m_ShuffleVector(m_Value(), m_Value(), m_Constant(Mask))))
- return Mask->getSplatValue() != nullptr;
+ ArrayRef<int> Mask;
+ if (match(V, m_ShuffleVector(m_Value(), m_Value(), m_Mask(Mask))))
+ return is_splat(Mask);
// The remaining tests are all recursive, so bail out if we hit the limit.
if (Depth++ == MaxDepth)
Index: llvm/lib/Analysis/ValueTracking.cpp
===================================================================
--- llvm/lib/Analysis/ValueTracking.cpp
+++ llvm/lib/Analysis/ValueTracking.cpp
@@ -2641,7 +2641,7 @@
// element referenced by the shuffle.
auto *Shuf = cast<ShuffleVectorInst>(U);
int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
- int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+ int NumMaskElts = Shuf->getShuffleMask().size();
APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
for (int i = 0; i != NumMaskElts; ++i) {
int M = Shuf->getMaskValue(i);
Index: llvm/lib/Analysis/TargetTransformInfo.cpp
===================================================================
--- llvm/lib/Analysis/TargetTransformInfo.cpp
+++ llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -870,7 +870,7 @@
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
- SmallVector<int, 16> ActualMask = SI->getShuffleMask();
+ ArrayRef<int> ActualMask = SI->getShuffleMask();
return Mask == ActualMask;
}
@@ -1135,7 +1135,7 @@
// Fill the rest of the mask with -1 for undef.
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
- SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
+ ArrayRef<int> Mask = Shuffle->getShuffleMask();
if (ShuffleMask != Mask)
return RK_None;
Index: llvm/lib/Analysis/InstructionSimplify.cpp
===================================================================
--- llvm/lib/Analysis/InstructionSimplify.cpp
+++ llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4407,18 +4407,18 @@
return RootVec;
}
-static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q,
unsigned MaxRecurse) {
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elem) { return Elem == -1; }))
return UndefValue::get(RetTy);
Type *InVecTy = Op0->getType();
- unsigned MaskNumElts = Mask->getType()->getVectorNumElements();
+ unsigned MaskNumElts = Mask.size();
unsigned InVecNumElts = InVecTy->getVectorNumElements();
SmallVector<int, 32> Indices;
- ShuffleVectorInst::getShuffleMask(Mask, Indices);
+ Indices.assign(Mask.begin(), Mask.end());
assert(MaskNumElts == Indices.size() &&
"Size of Indices not same as number of mask elements?");
@@ -4456,7 +4456,7 @@
// value type is same as the input vectors' type.
if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
- OpShuf->getMask()->getSplatValue())
+ is_splat(OpShuf->getShuffleMask()))
return Op0;
// Don't fold a shuffle with undef mask elements. This may get folded in a
@@ -4484,7 +4484,7 @@
}
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q) {
return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
}
@@ -5431,7 +5431,7 @@
case Instruction::ShuffleVector: {
auto *SVI = cast<ShuffleVectorInst>(I);
Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getMask(), SVI->getType(), Q);
+ SVI->getShuffleMask(), SVI->getType(), Q);
break;
}
case Instruction::PHI:
Index: llvm/lib/Analysis/ConstantFolding.cpp
===================================================================
--- llvm/lib/Analysis/ConstantFolding.cpp
+++ llvm/lib/Analysis/ConstantFolding.cpp
@@ -1062,7 +1062,8 @@
case Instruction::InsertElement:
return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
+ return ConstantExpr::getShuffleVector(Ops[0], Ops[1],
+ cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
}
}
Index: llvm/include/llvm/IR/PatternMatch.h
===================================================================
--- llvm/include/llvm/IR/PatternMatch.h
+++ llvm/include/llvm/IR/PatternMatch.h
@@ -1276,12 +1276,58 @@
return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
}
+/// Matches shuffle.
+template <typename T0, typename T1, typename T2> struct Shuffle_match {
+ T0 Op1;
+ T1 Op2;
+ T2 Mask;
+
+ Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
+ : Op1(Op1), Op2(Op2), Mask(Mask) {}
+
+ template <typename OpTy> bool match(OpTy *V) {
+ if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
+ return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
+ Mask.match(I->getShuffleMask());
+ }
+ return false;
+ }
+};
+
+struct m_Mask {
+ ArrayRef<int> &MaskRef;
+ m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ MaskRef = Mask;
+ return true;
+ }
+};
+
+struct m_ZeroMask {
+ bool match(ArrayRef<int> Mask) {
+ return all_of(Mask, [](int Elem) { return Elem == 0; });
+ }
+};
+
+struct m_SpecificMask {
+ ArrayRef<int> &MaskRef;
+ m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ return MaskRef == Mask;
+ }
+};
+
/// Matches ShuffleVectorInst.
+template <typename V1_t, typename V2_t>
+inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
+ return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
+}
+
template <typename V1_t, typename V2_t, typename Mask_t>
-inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
-m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
- return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
- m);
+inline Shuffle_match<V1_t, V2_t, Mask_t>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
+ return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
}
/// Matches LoadInst.
Index: llvm/include/llvm/IR/NoFolder.h
===================================================================
--- llvm/include/llvm/IR/NoFolder.h
+++ llvm/include/llvm/IR/NoFolder.h
@@ -325,7 +325,7 @@
}
Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return new ShuffleVectorInst(V1, V2, Mask);
}
Index: llvm/include/llvm/IR/Instructions.h
===================================================================
--- llvm/include/llvm/IR/Instructions.h
+++ llvm/include/llvm/IR/Instructions.h
@@ -1969,7 +1969,18 @@
/// This instruction constructs a fixed permutation of two
/// input vectors.
///
+/// The shuffle mask operand specifies, for each element of the result vector,
+/// which element of the two input vectors the result element gets. The
+/// shuffle mask is represented as an array of integers. Positive integers
+/// represent elements of the two input vectors. -1 represents an "undef"
+/// element in the result.
+///
+/// For scalable vectors, all the elements of the mask must be 0 or -1. This
+/// requirement may be relaxed in the future.
class ShuffleVectorInst : public Instruction {
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
@@ -1982,10 +1993,15 @@
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr = "",
+ Instruction *InsertBefor = nullptr);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr, BasicBlock *InsertAtEnd);
- // allocate space for exactly three operands
+ // allocate space for exactly two operands
void *operator new(size_t s) {
- return User::operator new(s, 3);
+ return User::operator new(s, 2);
}
/// Swap the first 2 operands and adjust the mask to preserve the semantics
@@ -1996,6 +2012,8 @@
/// formed with the specified operands.
static bool isValidOperands(const Value *V1, const Value *V2,
const Value *Mask);
+ static bool isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask);
/// Overload to return most specific vector type.
///
@@ -2006,10 +2024,6 @@
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
- Constant *getMask() const {
- return cast<Constant>(getOperand(2));
- }
-
/// Return the shuffle mask value for the specified element of the mask.
/// Return -1 if the element is undef.
static int getMaskValue(const Constant *Mask, unsigned Elt);
@@ -2017,7 +2031,7 @@
/// Return the shuffle mask value of this instruction for the given element
/// index. Return -1 if the element is undef.
int getMaskValue(unsigned Elt) const {
- return getMaskValue(getMask(), Elt);
+ return ShuffleMask[Elt];
}
/// Convert the input shuffle mask operand to a vector of integers. Undefined
@@ -2028,13 +2042,21 @@
/// Return the mask for this instruction as a vector of integers. Undefined
/// elements of the mask are returned as -1.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
- return getShuffleMask(getMask(), Result);
+ Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
- SmallVector<int, 16> getShuffleMask() const {
- SmallVector<int, 16> Mask;
- getShuffleMask(Mask);
- return Mask;
+ /// Return the mask for this instruction, for use in bitcode.
+ ///
+ /// TODO: This is temporary until we decide a new bitcode encoding for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const {
+ return ShuffleMaskForBitcode;
+ }
+
+ void setShuffleMask(ArrayRef<int> Mask);
+
+ ArrayRef<int> getShuffleMask() const {
+ return ShuffleMask;
}
/// Return true if this shuffle returns a vector with a different number of
@@ -2043,7 +2065,7 @@
/// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
bool changesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts != NumMaskElts;
}
@@ -2052,7 +2074,7 @@
/// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
bool increasesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts < NumMaskElts;
}
@@ -2073,7 +2095,7 @@
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(getMask());
+ return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
@@ -2094,7 +2116,7 @@
/// from its input vectors.
/// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
bool isIdentity() const {
- return !changesLength() && isIdentityMask(getShuffleMask());
+ return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2135,7 +2157,7 @@
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(getMask());
+ return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
@@ -2155,7 +2177,7 @@
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(getMask());
+ return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
@@ -2177,7 +2199,7 @@
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(getMask());
+ return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
@@ -2226,7 +2248,7 @@
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(getMask());
+ return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is an extract subvector mask.
@@ -2245,7 +2267,7 @@
/// Return true if this shuffle mask is an extract subvector mask.
bool isExtractSubvectorMask(int &Index) const {
int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
- return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
+ return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
}
/// Change values in a shuffle permute mask assuming the two vector operands
@@ -2272,7 +2294,7 @@
template <>
struct OperandTraits<ShuffleVectorInst> :
- public FixedNumOperandTraits<ShuffleVectorInst, 3> {
+ public FixedNumOperandTraits<ShuffleVectorInst, 2> {
};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)
Index: llvm/include/llvm/IR/IRBuilder.h
===================================================================
--- llvm/include/llvm/IR/IRBuilder.h
+++ llvm/include/llvm/IR/IRBuilder.h
@@ -2460,17 +2460,25 @@
Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
const Twine &Name = "") {
- if (auto *V1C = dyn_cast<Constant>(V1))
- if (auto *V2C = dyn_cast<Constant>(V2))
- if (auto *MC = dyn_cast<Constant>(Mask))
- return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
- return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
+ return CreateShuffleVector(V1, V2, IntMask, Name);
}
- Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
const Twine &Name = "") {
- Value *Mask = ConstantDataVector::get(Context, IntMask);
- return CreateShuffleVector(V1, V2, Mask, Name);
+ SmallVector<int, 16> IntMask;
+ IntMask.assign(Mask.begin(), Mask.end());
+ return CreateShuffleVector(V1, V2, IntMask, Name);
+ }
+
+ /// See class ShuffleVectorInst for a description of the mask representation.
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name = "") {
+ if (auto *V1C = dyn_cast<Constant>(V1))
+ if (auto *V2C = dyn_cast<Constant>(V2))
+ return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
+ return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}
Value *CreateExtractValue(Value *Agg,
Index: llvm/include/llvm/IR/Constants.h
===================================================================
--- llvm/include/llvm/IR/Constants.h
+++ llvm/include/llvm/IR/Constants.h
@@ -1200,6 +1200,8 @@
Type *OnlyIfReducedTy = nullptr);
static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
Type *OnlyIfReducedTy = nullptr);
+ static Constant *getShuffleVector(Constant *V1, Constant *V2, ArrayRef<int> Mask,
+ Type *OnlyIfReducedTy = nullptr);
static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
Type *OnlyIfReducedTy = nullptr);
static Constant *getInsertValue(Constant *Agg, Constant *Val,
@@ -1217,6 +1219,16 @@
/// expression and return the list of indices.
ArrayRef<unsigned> getIndices() const;
+ /// Assert that this is an shufflevector and return the mask. See class
+ /// ShuffleVectorInst for a description of the mask representation.
+ ArrayRef<int> getShuffleMask() const;
+
+ /// Assert that this is an shufflevector and return the mask.
+ ///
+ /// TODO: This is a temporary hack until we update the bitcode format for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const;
+
/// Return a string representation for an opcode.
const char *getOpcodeName() const;
Index: llvm/include/llvm/IR/ConstantFolder.h
===================================================================
--- llvm/include/llvm/IR/ConstantFolder.h
+++ llvm/include/llvm/IR/ConstantFolder.h
@@ -260,7 +260,7 @@
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return ConstantExpr::getShuffleVector(V1, V2, Mask);
}
Index: llvm/include/llvm/CodeGen/MachineOperand.h
===================================================================
--- llvm/include/llvm/CodeGen/MachineOperand.h
+++ llvm/include/llvm/CodeGen/MachineOperand.h
@@ -163,7 +163,8 @@
MachineInstr *ParentMI;
/// Contents union - This contains the payload for the various operand types.
- union {
+ union ContentsUnion {
+ ContentsUnion() {}
MachineBasicBlock *MBB; // For MO_MachineBasicBlock.
const ConstantFP *CFP; // For MO_FPImmediate.
const ConstantInt *CI; // For MO_CImmediate. Integers > 64bit.
@@ -174,7 +175,7 @@
unsigned CFIIndex; // For MO_CFI.
Intrinsic::ID IntrinsicID; // For MO_IntrinsicID.
unsigned Pred; // For MO_Predicate
- const Constant *ShuffleMask; // For MO_ShuffleMask
+ ArrayRef<int> ShuffleMask; // For MO_ShuffleMask
struct { // For MO_Register.
// Register number is in SmallContents.RegNo.
@@ -583,7 +584,7 @@
return Contents.Pred;
}
- const Constant *getShuffleMask() const {
+ ArrayRef<int> getShuffleMask() const {
assert(isShuffleMask() && "Wrong MachineOperand accessor");
return Contents.ShuffleMask;
}
@@ -911,9 +912,9 @@
return Op;
}
- static MachineOperand CreateShuffleMask(const Constant *C) {
+ static MachineOperand CreateShuffleMask(ArrayRef<int> Mask) {
MachineOperand Op(MachineOperand::MO_ShuffleMask);
- Op.Contents.ShuffleMask = C;
+ Op.Contents.ShuffleMask = Mask;
return Op;
}
Index: llvm/include/llvm/CodeGen/MachineInstrBuilder.h
===================================================================
--- llvm/include/llvm/CodeGen/MachineInstrBuilder.h
+++ llvm/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -250,7 +250,7 @@
return *this;
}
- const MachineInstrBuilder &addShuffleMask(const Constant *Val) const {
+ const MachineInstrBuilder &addShuffleMask(ArrayRef<int> Val) const {
MI->addOperand(*MF, MachineOperand::CreateShuffleMask(Val));
return *this;
}
Index: llvm/include/llvm/CodeGen/MachineFunction.h
===================================================================
--- llvm/include/llvm/CodeGen/MachineFunction.h
+++ llvm/include/llvm/CodeGen/MachineFunction.h
@@ -796,6 +796,8 @@
/// Allocate and initialize a register mask with @p NumRegister bits.
uint32_t *allocateRegMask();
+ MutableArrayRef<int> allocateShuffleMask(unsigned NumElems);
+
/// Allocate and construct an extra info structure for a `MachineInstr`.
///
/// This is allocated on the function's allocator and so lives the life of
Index: llvm/include/llvm/Analysis/TargetFolder.h
===================================================================
--- llvm/include/llvm/Analysis/TargetFolder.h
+++ llvm/include/llvm/Analysis/TargetFolder.h
@@ -252,7 +252,7 @@
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const {
+ ArrayRef<int> Mask) const {
return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
}
Index: llvm/include/llvm/Analysis/InstructionSimplify.h
===================================================================
--- llvm/include/llvm/Analysis/InstructionSimplify.h
+++ llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -230,7 +230,8 @@
const SimplifyQuery &Q);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+/// See class ShuffleVectorInst for a description of the mask representation.
+Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q);
//=== Helper functions for higher up the class hierarchy.
Index: llvm/include/llvm/Analysis/ConstantFolding.h
===================================================================
--- llvm/include/llvm/Analysis/ConstantFolding.h
+++ llvm/include/llvm/Analysis/ConstantFolding.h
@@ -119,10 +119,11 @@
Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
/// Attempt to constant fold a shufflevector instruction with the
-/// specified operands and indices. The constant result is returned if
-/// successful; if not, null is returned.
+/// specified operands and mask. See class ShuffleVectorInst for a description
+/// of the mask representation. The constant result is returned if successful;
+/// if not, null is returned.
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
/// produce if it is constant and determinable. If this is not determinable,
Index: llvm/include/llvm/ADT/ArrayRef.h
===================================================================
--- llvm/include/llvm/ADT/ArrayRef.h
+++ llvm/include/llvm/ADT/ArrayRef.h
@@ -535,10 +535,20 @@
}
template<typename T>
+ inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return ArrayRef<T>(LHS).equals(RHS);
+ }
+
+ template<typename T>
inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
+ template<typename T>
+ inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return !(LHS == RHS);
+ }
+
/// @}
template <typename T> hash_code hash_value(ArrayRef<T> S) {
Index: clang/lib/CodeGen/CGExpr.cpp
===================================================================
--- clang/lib/CodeGen/CGExpr.cpp
+++ clang/lib/CodeGen/CGExpr.cpp
@@ -1656,7 +1656,7 @@
// Shuffle vector to get vec3.
V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
- {0, 1, 2}, "extractVec");
+ ArrayRef<int>{0, 1, 2}, "extractVec");
return EmitFromMemory(V, Ty);
}
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
