X86ShuffleDecodeConstantPool.cpp

Go to the documentation of this file.

1//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//

2//

3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4// See https://llvm.org/LICENSE.txt for license information.

5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6//

7//===----------------------------------------------------------------------===//

8//

9// Define several functions to decode x86 specific shuffle semantics using

10// constants from the constant pool.

11//

12//===----------------------------------------------------------------------===//

13

14#include "X86ShuffleDecodeConstantPool.h"

15#include "MCTargetDesc/X86ShuffleDecode.h"

16#include "llvm/ADT/APInt.h"

17#include "llvm/ADT/SmallVector.h"

18#include "llvm/IR/Constants.h"

19

20//===----------------------------------------------------------------------===//

21// Vector Mask Decoding

22//===----------------------------------------------------------------------===//

23

24namespace llvm {

25

26 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,

27 APInt &UndefElts,

28 SmallVectorImpl<uint64_t> &RawMask) {

29 // It is not an error for shuffle masks to not be a vector of

30 // MaskEltSizeInBits because the constant pool uniques constants by their

31 // bit representation.

32 // e.g. the following take up the same space in the constant pool:

33 // i128 -170141183420855150465331762880109871104

34 //

35 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>

36 //

37 // <4 x i32> <i32 -2147483648, i32 -2147483648,

38 // i32 -2147483648, i32 -2147483648>

39 auto *CstTy = dyn_cast<FixedVectorType>(C->getType());

40 if (!CstTy)

41 return false;

42

43 Type *CstEltTy = CstTy->getElementType();

44 if (!CstEltTy->isIntegerTy())

45 return false;

46

47 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();

48 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();

49 unsigned NumCstElts = CstTy->getNumElements();

50

51 assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&

52 "Unaligned shuffle mask size");

53

54 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;

55 UndefElts = APInt(NumMaskElts, 0);

56 RawMask.resize(NumMaskElts, 0);

57

58 // Fast path - if the constants match the mask size then copy direct.

59 if (MaskEltSizeInBits == CstEltSizeInBits) {

60 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");

61 for (unsigned i = 0; i != NumMaskElts; ++i) {

62 Constant *COp = C->getAggregateElement(i);

63 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))

64 return false;

65

66 if (isa<UndefValue>(COp)) {

67 UndefElts.setBit(i);

68 RawMask[i] = 0;

69 continue;

70 }

71

72 auto *Elt = cast<ConstantInt>(COp);

73 RawMask[i] = Elt->getValue().getZExtValue();

74 }

75 return true;

76 }

77

78 // Extract all the undef/constant element data and pack into single bitsets.

79 APInt UndefBits(CstSizeInBits, 0);

80 APInt MaskBits(CstSizeInBits, 0);

81 for (unsigned i = 0; i != NumCstElts; ++i) {

82 Constant *COp = C->getAggregateElement(i);

83 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))

84 return false;

85

86 unsigned BitOffset = i * CstEltSizeInBits;

87

88 if (isa<UndefValue>(COp)) {

89 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);

90 continue;

91 }

92

93 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);

94 }

95

96 // Now extract the undef/constant bit data into the raw shuffle masks.

97 for (unsigned i = 0; i != NumMaskElts; ++i) {

98 unsigned BitOffset = i * MaskEltSizeInBits;

99 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);

100

101 // Only treat the element as UNDEF if all bits are UNDEF, otherwise

102 // treat it as zero.

103 if (EltUndef.isAllOnes()) {

104 UndefElts.setBit(i);

105 RawMask[i] = 0;

106 continue;

107 }

108

109 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);

110 RawMask[i] = EltBits.getZExtValue();

111 }

112

113 return true;

114}

115

116 void DecodePSHUFBMask(const Constant *C, unsigned Width,

117 SmallVectorImpl<int> &ShuffleMask) {

118 assert((Width == 128 || Width == 256 || Width == 512) &&

119 C->getType()->getPrimitiveSizeInBits() >= Width &&

120 "Unexpected vector size.");

121

122 // The shuffle mask requires a byte vector.

123 APInt UndefElts;

124 SmallVector<uint64_t, 64> RawMask;

125 if (!extractConstantMask(C, 8, UndefElts, RawMask))

126 return;

127

128 unsigned NumElts = Width / 8;

129 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&

130 "Unexpected number of vector elements.");

131

132 for (unsigned i = 0; i != NumElts; ++i) {

133 if (UndefElts[i]) {

134 ShuffleMask.push_back(SM_SentinelUndef);

135 continue;

136 }

137

138 uint64_t Element = RawMask[i];

139 // If the high bit (7) of the byte is set, the element is zeroed.

140 if (Element & (1 << 7))

141 ShuffleMask.push_back(SM_SentinelZero);

142 else {

143 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte

144 // lane of the vector we're inside.

145 unsigned Base = i & ~0xf;

146

147 // Only the least significant 4 bits of the byte are used.

148 int Index = Base + (Element & 0xf);

149 ShuffleMask.push_back(Index);

150 }

151 }

152}

153

154 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,

155 SmallVectorImpl<int> &ShuffleMask) {

156 assert((Width == 128 || Width == 256 || Width == 512) &&

157 C->getType()->getPrimitiveSizeInBits() >= Width &&

158 "Unexpected vector size.");

159 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");

160

161 // The shuffle mask requires elements the same size as the target.

162 APInt UndefElts;

163 SmallVector<uint64_t, 16> RawMask;

164 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))

165 return;

166

167 unsigned NumElts = Width / ElSize;

168 unsigned NumEltsPerLane = 128 / ElSize;

169 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&

170 "Unexpected number of vector elements.");

171

172 for (unsigned i = 0; i != NumElts; ++i) {

173 if (UndefElts[i]) {

174 ShuffleMask.push_back(SM_SentinelUndef);

175 continue;

176 }

177

178 int Index = i & ~(NumEltsPerLane - 1);

179 uint64_t Element = RawMask[i];

180 if (ElSize == 64)

181 Index += (Element >> 1) & 0x1;

182 else

183 Index += Element & 0x3;

184

185 ShuffleMask.push_back(Index);

186 }

187}

188

189 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,

190 unsigned Width, SmallVectorImpl<int> &ShuffleMask) {

191 Type *MaskTy = C->getType();

192 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();

193 (void)MaskTySize;

194 assert((MaskTySize == 128 || MaskTySize == 256) && Width >= MaskTySize &&

195 "Unexpected vector size.");

196

197 // The shuffle mask requires elements the same size as the target.

198 APInt UndefElts;

199 SmallVector<uint64_t, 8> RawMask;

200 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))

201 return;

202

203 unsigned NumElts = Width / ElSize;

204 unsigned NumEltsPerLane = 128 / ElSize;

205 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&

206 "Unexpected number of vector elements.");

207

208 for (unsigned i = 0; i != NumElts; ++i) {

209 if (UndefElts[i]) {

210 ShuffleMask.push_back(SM_SentinelUndef);

211 continue;

212 }

213

214 // VPERMIL2 Operation.

215 // Bits[3] - Match Bit.

216 // Bits[2:1] - (Per Lane) PD Shuffle Mask.

217 // Bits[2:0] - (Per Lane) PS Shuffle Mask.

218 uint64_t Selector = RawMask[i];

219 unsigned MatchBit = (Selector >> 3) & 0x1;

220

221 // M2Z[0:1] MatchBit

222 // 0Xb X Source selected by Selector index.

223 // 10b 0 Source selected by Selector index.

224 // 10b 1 Zero.

225 // 11b 0 Zero.

226 // 11b 1 Source selected by Selector index.

227 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {

228 ShuffleMask.push_back(SM_SentinelZero);

229 continue;

230 }

231

232 int Index = i & ~(NumEltsPerLane - 1);

233 if (ElSize == 64)

234 Index += (Selector >> 1) & 0x1;

235 else

236 Index += Selector & 0x3;

237

238 int Src = (Selector >> 2) & 0x1;

239 Index += Src * NumElts;

240 ShuffleMask.push_back(Index);

241 }

242}

243

244 void DecodeVPPERMMask(const Constant *C, unsigned Width,

245 SmallVectorImpl<int> &ShuffleMask) {

246 Type *MaskTy = C->getType();

247 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();

248 (void)MaskTySize;

249 assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");

250

251 // The shuffle mask requires a byte vector.

252 APInt UndefElts;

253 SmallVector<uint64_t, 16> RawMask;

254 if (!extractConstantMask(C, 8, UndefElts, RawMask))

255 return;

256

257 unsigned NumElts = Width / 8;

258 assert(NumElts == 16 && "Unexpected number of vector elements.");

259

260 for (unsigned i = 0; i != NumElts; ++i) {

261 if (UndefElts[i]) {

262 ShuffleMask.push_back(SM_SentinelUndef);

263 continue;

264 }

265

266 // VPPERM Operation

267 // Bits[4:0] - Byte Index (0 - 31)

268 // Bits[7:5] - Permute Operation

269 //

270 // Permute Operation:

271 // 0 - Source byte (no logical operation).

272 // 1 - Invert source byte.

273 // 2 - Bit reverse of source byte.

274 // 3 - Bit reverse of inverted source byte.

275 // 4 - 00h (zero - fill).

276 // 5 - FFh (ones - fill).

277 // 6 - Most significant bit of source byte replicated in all bit positions.

278 // 7 - Invert most significant bit of source byte and replicate in all bit

279 // positions.

280 uint64_t Element = RawMask[i];

281 uint64_t Index = Element & 0x1F;

282 uint64_t PermuteOp = (Element >> 5) & 0x7;

283

284 if (PermuteOp == 4) {

285 ShuffleMask.push_back(SM_SentinelZero);

286 continue;

287 }

288 if (PermuteOp != 0) {

289 ShuffleMask.clear();

290 return;

291 }

292 ShuffleMask.push_back((int)Index);

293 }

294}

295

296} // namespace llvm

assert

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

APInt.h

This file implements a class to represent arbitrary precision integral constant values and operations...

Constants.h

This file contains the declarations for the subclasses of Constant, which represent the different fla...

SmallVector.h

This file defines the SmallVector class.

X86ShuffleDecodeConstantPool.h

X86ShuffleDecode.h

llvm::APInt

Class for arbitrary precision integers.

Definition APInt.h:78

llvm::APInt::getZExtValue

uint64_t getZExtValue() const

Get zero extended value.

Definition APInt.h:1541

llvm::APInt::setBit

void setBit(unsigned BitPosition)

Set the given bit to 1 whose position is given as "bitPosition".

Definition APInt.h:1331

llvm::APInt::isAllOnes

bool isAllOnes() const

Determine if all bits are set. This is true for zero-width values.

Definition APInt.h:372

llvm::APInt::insertBits

LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)

Insert the bits from a smaller APInt starting at bitPosition.

Definition APInt.cpp:397

llvm::APInt::setBits

void setBits(unsigned loBit, unsigned hiBit)

Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.

Definition APInt.h:1368

llvm::APInt::extractBits

LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const

Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).

Definition APInt.cpp:482

llvm::Constant

This is an important base class in LLVM.

Definition Constant.h:43

llvm::SmallVectorImpl

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

Definition SmallVector.h:574

llvm::SmallVectorImpl::resize

void resize(size_type N)

Definition SmallVector.h:639

llvm::SmallVectorTemplateBase::push_back

void push_back(const T &Elt)

Definition SmallVector.h:417

llvm::SmallVector

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

Definition SmallVector.h:1203

llvm::Type

The instances of the Type class are immutable: once they are created, they are never changed.

Definition Type.h:45

llvm::Type::getPrimitiveSizeInBits

LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY

Return the basic size of this type if it is a primitive type.

Definition Type.cpp:198

llvm::Type::isIntegerTy

bool isIntegerTy() const

True if this is an instance of IntegerType.

Definition Type.h:240

uint64_t

llvm::CallingConv::C

@ C

The default llvm calling convention, compatible with C.

Definition CallingConv.h:34

llvm::sampleprof::Base

@ Base

Definition Discriminator.h:58

llvm

This is an optimization pass for GlobalISel generic memory operations.

Definition AddressRanges.h:18

llvm::DecodeVPERMILPMask

void DecodeVPERMILPMask(unsigned NumElts, unsigned ScalarBits, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:477

llvm::dyn_cast

decltype(auto) dyn_cast(const From &Val)

dyn_cast<X> - Return the argument parameter cast to the specified type.

Definition Casting.h:643

llvm::DecodeVPERMIL2PMask

void DecodeVPERMIL2PMask(unsigned NumElts, unsigned ScalarBits, unsigned M2Z, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERMIL2PD/VPERMIL2PS variable mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:499

llvm::extractConstantMask

static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, APInt &UndefElts, SmallVectorImpl< uint64_t > &RawMask)

Definition X86ShuffleDecodeConstantPool.cpp:26

llvm::isa

bool isa(const From &Val)

isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...

Definition Casting.h:547

llvm::DecodeVPPERMMask

void DecodeVPPERMMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPPERM mask from a raw array of constants such as from BUILD_VECTOR.