Blame - llvm/lib/IR/AutoUpgrade.cpp - toolchain/llvm-project

2007-08-04 01:51:18 +0000

[diff] [blame]

1

//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//

2

//

3

// The LLVM Compiler Infrastructure

4

//

Chris Lattner

f3ebc3f

2007-12-29 20:36:04 +0000

[diff] [blame]

5

// This file is distributed under the University of Illinois Open Source

6

// License. See LICENSE.TXT for details.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

7

//

8

//===----------------------------------------------------------------------===//

9

//

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

10

// This file implements the auto-upgrade helper functions.

11

// This is where deprecated IR intrinsics and other IR features are updated to

12

// current specifications.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

13

//

14

//===----------------------------------------------------------------------===//

15

Chandler Carruth

9106521

2014-03-05 10:34:14 +0000

[diff] [blame]

16

#include "llvm/IR/AutoUpgrade.h"

Chandler Carruth

1305dc3

2014-03-04 11:45:46 +0000

[diff] [blame]

17

#include "llvm/IR/CFG.h"

Chandler Carruth

219b89b

2014-03-04 11:01:28 +0000

[diff] [blame]

18

#include "llvm/IR/CallSite.h"

Chandler Carruth

9fb823b

2013-01-02 11:36:10 +0000

[diff] [blame]

19

#include "llvm/IR/Constants.h"

Chandler Carruth

d990388

2015-01-14 11:23:27 +0000

[diff] [blame]

20

#include "llvm/IR/DIBuilder.h"

Chandler Carruth

9a4c9e5

2014-03-06 00:46:21 +0000

[diff] [blame]

21

#include "llvm/IR/DebugInfo.h"

Manman Ren

2ebfb42

2014-01-16 01:51:12 +0000

[diff] [blame]

22

#include "llvm/IR/DiagnosticInfo.h"

Chandler Carruth

9fb823b

2013-01-02 11:36:10 +0000

[diff] [blame]

23

#include "llvm/IR/Function.h"

24

#include "llvm/IR/IRBuilder.h"

25

#include "llvm/IR/Instruction.h"

26

#include "llvm/IR/IntrinsicInst.h"

27

#include "llvm/IR/LLVMContext.h"

28

#include "llvm/IR/Module.h"

Torok Edwin

56d0659

2009-07-11 20:10:48 +0000

[diff] [blame]

29

#include "llvm/Support/ErrorHandling.h"

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

30

#include "llvm/Support/Regex.h"

Anton Korobeynikov

579f071

2008-02-20 11:08:44 +0000

[diff] [blame]

31

#include <cstring>

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

32

using namespace llvm;

33

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

34

static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }

35

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

36

// Upgrade the declarations of the SSE4.1 functions whose arguments have

37

// changed their type from v4f32 to v2i64.

38

static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,

39

Function *&NewFn) {

40

// Check whether this is an old version of the function, which received

41

// v4f32 arguments.

42

Type *Arg0Type = F->getFunctionType()->getParamType(0);

43

if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))

44

return false;

45

46

// Yes, it's old, replace it with new version.

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

47

rename(F);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

48

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

49

return true;

50

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

51

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

52

// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask

53

// arguments have changed their type from i32 to i8.

54

static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,

55

Function *&NewFn) {

56

// Check that the last argument is an i32.

57

Type *LastArgType = F->getFunctionType()->getParamType(

58

F->getFunctionType()->getNumParams() - 1);

59

if (!LastArgType->isIntegerTy(32))

60

return false;

61

62

// Move this function aside and map down.

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

63

rename(F);

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

64

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

return true;

}

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

68

static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

69

assert(F && "Illegal to upgrade a non-existent Function.");

70

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

71

// Quickly eliminate it, if it's not a candidate.

Eric Liu

c9c6817

2016-07-08 16:09:51 +0000

[diff] [blame]

72

StringRef Name = F->getName();

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

73

if (Name.size() <= 8 || !Name.startswith("llvm."))

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

74

return false;

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

75

Name = Name.substr(5); // Strip off "llvm."

Chris Lattner

0bcbde4

2011-11-27 08:42:07 +0000

[diff] [blame]

76

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

77

switch (Name[0]) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

78

default: break;

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

79

case 'a': {

80

if (Name.startswith("arm.neon.vclz")) {

81

Type* args[2] = {

Matt Arsenault

c4c9226

2013-07-20 17:46:00 +0000

[diff] [blame]

82

F->arg_begin()->getType(),

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

83

Type::getInt1Ty(F->getContext())

84

};

85

// Can't use Intrinsic::getDeclaration here as it adds a ".i1" to

86

// the end of the name. Change name from llvm.arm.neon.vclz.* to

87

// llvm.ctlz.*

88

FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);

Matt Arsenault

c4c9226

2013-07-20 17:46:00 +0000

[diff] [blame]

89

NewFn = Function::Create(fType, F->getLinkage(),

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

90

"llvm.ctlz." + Name.substr(14), F->getParent());

91

return true;

92

}

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

93

if (Name.startswith("arm.neon.vcnt")) {

94

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,

95

F->arg_begin()->getType());

96

return true;

97

}

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

98

Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");

99

if (vldRegex.match(Name)) {

100

auto fArgs = F->getFunctionType()->params();

101

SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());

102

// Can't use Intrinsic::getDeclaration here as the return types might

103

// then only be structurally equal.

104

FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);

105

NewFn = Function::Create(fType, F->getLinkage(),

106

"llvm." + Name + ".p0i8", F->getParent());

107

return true;

108

}

109

Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");

110

if (vstRegex.match(Name)) {

Craig Topper

2626094

2015-10-18 05:15:34 +0000

[diff] [blame]

111

static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,

112

Intrinsic::arm_neon_vst2,

113

Intrinsic::arm_neon_vst3,

114

Intrinsic::arm_neon_vst4};

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

115

Craig Topper

2626094

2015-10-18 05:15:34 +0000

[diff] [blame]

116

static const Intrinsic::ID StoreLaneInts[] = {

117

Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,

118

Intrinsic::arm_neon_vst4lane

119

};

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

120

121

auto fArgs = F->getFunctionType()->params();

122

Type *Tys[] = {fArgs[0], fArgs[1]};

123

if (Name.find("lane") == StringRef::npos)

124

NewFn = Intrinsic::getDeclaration(F->getParent(),

125

StoreInts[fArgs.size() - 3], Tys);

126

else

127

NewFn = Intrinsic::getDeclaration(F->getParent(),

128

StoreLaneInts[fArgs.size() - 5], Tys);

129

return true;

130

}

Marcin Koscielnicki

3fdc257

2016-04-19 20:51:05 +0000

[diff] [blame]

131

if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {

132

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);

133

return true;

134

}

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

135

break;

136

}

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

137

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

138

case 'c': {

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

139

if (Name.startswith("ctlz.") && F->arg_size() == 1) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

140

rename(F);

Chandler Carruth

d4a0240

2011-12-12 10:57:20 +0000

[diff] [blame]

141

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,

142

F->arg_begin()->getType());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

143

return true;

144

}

145

if (Name.startswith("cttz.") && F->arg_size() == 1) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

146

rename(F);

Chandler Carruth

d4a0240

2011-12-12 10:57:20 +0000

[diff] [blame]

147

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,

148

F->arg_begin()->getType());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

return true;

}

break;

}

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

153

case 'i': {

154

if (Name.startswith("invariant.start")) {

155

auto Args = F->getFunctionType()->params();

156

Type* ObjectPtr[1] = {Args[1]};

157

if (F->getName() !=

158

Intrinsic::getName(Intrinsic::invariant_start, ObjectPtr)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

159

rename(F);

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

160

NewFn = Intrinsic::getDeclaration(

161

F->getParent(), Intrinsic::invariant_start, ObjectPtr);

return true;

}

}

if (Name.startswith("invariant.end")) {

166

auto Args = F->getFunctionType()->params();

167

Type* ObjectPtr[1] = {Args[2]};

168

if (F->getName() !=

169

Intrinsic::getName(Intrinsic::invariant_end, ObjectPtr)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

170

rename(F);

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

171

NewFn = Intrinsic::getDeclaration(F->getParent(),

172

Intrinsic::invariant_end, ObjectPtr);

return true;

}

}

break;

}

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

178

case 'm': {

179

if (Name.startswith("masked.load.")) {

180

Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };

181

if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

182

rename(F);

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

183

NewFn = Intrinsic::getDeclaration(F->getParent(),

184

Intrinsic::masked_load,

Tys);

return true;

}

}

if (Name.startswith("masked.store.")) {

190

auto Args = F->getFunctionType()->params();

191

Type *Tys[] = { Args[0], Args[1] };

192

if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

193

rename(F);

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

194

NewFn = Intrinsic::getDeclaration(F->getParent(),

195

Intrinsic::masked_store,

Tys);

return true;

}

}

break;

}

Matt Arsenault

2013-10-07 18:06:48 +0000

[diff] [blame]

203

case 'o':

204

// We only need to change the name to match the mangling including the

205

// address space.

206

if (F->arg_size() == 2 && Name.startswith("objectsize.")) {

207

Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };

208

if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

209

rename(F);

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

210

NewFn = Intrinsic::getDeclaration(F->getParent(),

211

Intrinsic::objectsize, Tys);

return true;

}

}

break;

Tim Shen

2016-04-08 21:26:31 +0000

[diff] [blame]

217

case 's':

218

if (Name == "stackprotectorcheck") {

219

NewFn = nullptr;

220

return true;

221

}

George Burgess IV

63b06e1

2016-11-08 04:01:50 +0000

[diff] [blame]

222

break;

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

223

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

224

case 'x': {

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

225

bool IsX86 = Name.startswith("x86.");

226

if (IsX86)

227

Name = Name.substr(4);

228

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

229

// All of the intrinsics matches below should be marked with which llvm

230

// version started autoupgrading them. At some point in the future we would

231

// like to use this information to remove upgrade code for some older

232

// intrinsics. It is currently undecided how we will determine that future

233

// point.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

234

if (IsX86 &&

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

235

(Name.startswith("sse2.pcmpeq.") || // Added in 3.1

236

Name.startswith("sse2.pcmpgt.") || // Added in 3.1

237

Name.startswith("avx2.pcmpeq.") || // Added in 3.1

238

Name.startswith("avx2.pcmpgt.") || // Added in 3.1

239

Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9

240

Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9

Craig Topper

6910fa0

2016-11-16 05:24:10 +0000

[diff] [blame]

241

Name == "sse.add.ss" || // Added in 4.0

242

Name == "sse2.add.sd" || // Added in 4.0

243

Name == "sse.sub.ss" || // Added in 4.0

244

Name == "sse2.sub.sd" || // Added in 4.0

245

Name == "sse.mul.ss" || // Added in 4.0

246

Name == "sse2.mul.sd" || // Added in 4.0

247

Name == "sse.div.ss" || // Added in 4.0

248

Name == "sse2.div.sd" || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

249

Name == "sse41.pmaxsb" || // Added in 3.9

250

Name == "sse2.pmaxs.w" || // Added in 3.9

251

Name == "sse41.pmaxsd" || // Added in 3.9

252

Name == "sse2.pmaxu.b" || // Added in 3.9

253

Name == "sse41.pmaxuw" || // Added in 3.9

254

Name == "sse41.pmaxud" || // Added in 3.9

255

Name == "sse41.pminsb" || // Added in 3.9

256

Name == "sse2.pmins.w" || // Added in 3.9

257

Name == "sse41.pminsd" || // Added in 3.9

258

Name == "sse2.pminu.b" || // Added in 3.9

259

Name == "sse41.pminuw" || // Added in 3.9

260

Name == "sse41.pminud" || // Added in 3.9

Craig Topper

edab02b

2016-12-10 23:09:43 +0000

[diff] [blame^]

261

Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

262

Name.startswith("avx2.pmax") || // Added in 3.9

263

Name.startswith("avx2.pmin") || // Added in 3.9

264

Name.startswith("avx512.mask.pmax") || // Added in 4.0

265

Name.startswith("avx512.mask.pmin") || // Added in 4.0

266

Name.startswith("avx2.vbroadcast") || // Added in 3.8

267

Name.startswith("avx2.pbroadcast") || // Added in 3.8

268

Name.startswith("avx.vpermil.") || // Added in 3.1

269

Name.startswith("sse2.pshuf") || // Added in 3.9

270

Name.startswith("avx512.pbroadcast") || // Added in 3.9

271

Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9

272

Name.startswith("avx512.mask.movddup") || // Added in 3.9

273

Name.startswith("avx512.mask.movshdup") || // Added in 3.9

274

Name.startswith("avx512.mask.movsldup") || // Added in 3.9

275

Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9

276

Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9

277

Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9

278

Name.startswith("avx512.mask.shuf.p") || // Added in 4.0

279

Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9

280

Name.startswith("avx512.mask.perm.df.") || // Added in 3.9

281

Name.startswith("avx512.mask.perm.di.") || // Added in 3.9

282

Name.startswith("avx512.mask.punpckl") || // Added in 3.9

283

Name.startswith("avx512.mask.punpckh") || // Added in 3.9

284

Name.startswith("avx512.mask.unpckl.") || // Added in 3.9

285

Name.startswith("avx512.mask.unpckh.") || // Added in 3.9

286

Name.startswith("avx512.mask.pand.") || // Added in 3.9

287

Name.startswith("avx512.mask.pandn.") || // Added in 3.9

288

Name.startswith("avx512.mask.por.") || // Added in 3.9

289

Name.startswith("avx512.mask.pxor.") || // Added in 3.9

290

Name.startswith("avx512.mask.and.") || // Added in 3.9

291

Name.startswith("avx512.mask.andn.") || // Added in 3.9

292

Name.startswith("avx512.mask.or.") || // Added in 3.9

293

Name.startswith("avx512.mask.xor.") || // Added in 3.9

294

Name.startswith("avx512.mask.padd.") || // Added in 4.0

295

Name.startswith("avx512.mask.psub.") || // Added in 4.0

296

Name.startswith("avx512.mask.pmull.") || // Added in 4.0

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

297

Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0

298

Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

299

Name == "avx512.mask.add.pd.128" || // Added in 4.0

300

Name == "avx512.mask.add.pd.256" || // Added in 4.0

301

Name == "avx512.mask.add.ps.128" || // Added in 4.0

302

Name == "avx512.mask.add.ps.256" || // Added in 4.0

303

Name == "avx512.mask.div.pd.128" || // Added in 4.0

304

Name == "avx512.mask.div.pd.256" || // Added in 4.0

305

Name == "avx512.mask.div.ps.128" || // Added in 4.0

306

Name == "avx512.mask.div.ps.256" || // Added in 4.0

307

Name == "avx512.mask.mul.pd.128" || // Added in 4.0

308

Name == "avx512.mask.mul.pd.256" || // Added in 4.0

309

Name == "avx512.mask.mul.ps.128" || // Added in 4.0

310

Name == "avx512.mask.mul.ps.256" || // Added in 4.0

311

Name == "avx512.mask.sub.pd.128" || // Added in 4.0

312

Name == "avx512.mask.sub.pd.256" || // Added in 4.0

313

Name == "avx512.mask.sub.ps.128" || // Added in 4.0

314

Name == "avx512.mask.sub.ps.256" || // Added in 4.0

Craig Topper

abe7c5b

2016-12-10 21:15:52 +0000

[diff] [blame]

315

Name == "avx512.mask.vpermilvar.ps.128" || // Added in 4.0

316

Name == "avx512.mask.vpermilvar.ps.256" || // Added in 4.0

317

Name == "avx512.mask.vpermilvar.pd.128" || // Added in 4.0

318

Name == "avx512.mask.vpermilvar.pd.256" || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

319

Name.startswith("avx512.mask.psll.d") || // Added in 4.0

320

Name.startswith("avx512.mask.psll.q") || // Added in 4.0

321

Name.startswith("avx512.mask.psll.w") || // Added in 4.0

322

Name.startswith("avx512.mask.psra.d") || // Added in 4.0

323

Name.startswith("avx512.mask.psra.q") || // Added in 4.0

324

Name.startswith("avx512.mask.psra.w") || // Added in 4.0

325

Name.startswith("avx512.mask.psrl.d") || // Added in 4.0

326

Name.startswith("avx512.mask.psrl.q") || // Added in 4.0

327

Name.startswith("avx512.mask.psrl.w") || // Added in 4.0

328

Name.startswith("avx512.mask.pslli") || // Added in 4.0

329

Name.startswith("avx512.mask.psrai") || // Added in 4.0

330

Name.startswith("avx512.mask.psrli") || // Added in 4.0

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

331

Name.startswith("avx512.mask.psllv") || // Added in 4.0

332

Name.startswith("avx512.mask.psrav") || // Added in 4.0

333

Name.startswith("avx512.mask.psrlv") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

334

Name.startswith("sse41.pmovsx") || // Added in 3.8

335

Name.startswith("sse41.pmovzx") || // Added in 3.9

336

Name.startswith("avx2.pmovsx") || // Added in 3.9

337

Name.startswith("avx2.pmovzx") || // Added in 3.9

338

Name.startswith("avx512.mask.pmovsx") || // Added in 4.0

339

Name.startswith("avx512.mask.pmovzx") || // Added in 4.0

340

Name == "sse2.cvtdq2pd" || // Added in 3.9

341

Name == "sse2.cvtps2pd" || // Added in 3.9

342

Name == "avx.cvtdq2.pd.256" || // Added in 3.9

343

Name == "avx.cvt.ps2.pd.256" || // Added in 3.9

344

Name.startswith("avx.vinsertf128.") || // Added in 3.7

345

Name == "avx2.vinserti128" || // Added in 3.7

346

Name.startswith("avx.vextractf128.") || // Added in 3.7

347

Name == "avx2.vextracti128" || // Added in 3.7

348

Name.startswith("sse4a.movnt.") || // Added in 3.9

349

Name.startswith("avx.movnt.") || // Added in 3.2

350

Name.startswith("avx512.storent.") || // Added in 3.9

351

Name == "sse2.storel.dq" || // Added in 3.9

352

Name.startswith("sse.storeu.") || // Added in 3.9

353

Name.startswith("sse2.storeu.") || // Added in 3.9

354

Name.startswith("avx.storeu.") || // Added in 3.9

355

Name.startswith("avx512.mask.storeu.") || // Added in 3.9

356

Name.startswith("avx512.mask.store.p") || // Added in 3.9

357

Name.startswith("avx512.mask.store.b.") || // Added in 3.9

358

Name.startswith("avx512.mask.store.w.") || // Added in 3.9

359

Name.startswith("avx512.mask.store.d.") || // Added in 3.9

360

Name.startswith("avx512.mask.store.q.") || // Added in 3.9

361

Name.startswith("avx512.mask.loadu.") || // Added in 3.9

362

Name.startswith("avx512.mask.load.") || // Added in 3.9

363

Name == "sse42.crc32.64.8" || // Added in 3.4

364

Name.startswith("avx.vbroadcast.s") || // Added in 3.5

365

Name.startswith("avx512.mask.palignr.") || // Added in 3.9

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

366

Name.startswith("avx512.mask.valign.") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

367

Name.startswith("sse2.psll.dq") || // Added in 3.7

368

Name.startswith("sse2.psrl.dq") || // Added in 3.7

369

Name.startswith("avx2.psll.dq") || // Added in 3.7

370

Name.startswith("avx2.psrl.dq") || // Added in 3.7

371

Name.startswith("avx512.psll.dq") || // Added in 3.9

372

Name.startswith("avx512.psrl.dq") || // Added in 3.9

373

Name == "sse41.pblendw" || // Added in 3.7

374

Name.startswith("sse41.blendp") || // Added in 3.7

375

Name.startswith("avx.blend.p") || // Added in 3.7

376

Name == "avx2.pblendw" || // Added in 3.7

377

Name.startswith("avx2.pblendd.") || // Added in 3.7

378

Name.startswith("avx.vbroadcastf128") || // Added in 4.0

379

Name == "avx2.vbroadcasti128" || // Added in 3.7

380

Name == "xop.vpcmov" || // Added in 3.8

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

381

Name.startswith("avx512.mask.move.s") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

382

(Name.startswith("xop.vpcom") && // Added in 3.2

383

F->arg_size() == 2))) {

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

384

NewFn = nullptr;

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

385

return true;

386

}

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

387

// SSE4.1 ptest functions may have an old signature.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

388

if (IsX86 && Name.startswith("sse41.ptest")) { // Added in 3.2

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

389

if (Name.substr(11) == "c")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

390

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

391

if (Name.substr(11) == "z")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

392

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

393

if (Name.substr(11) == "nzc")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

394

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);

395

}

Sanjay Patel

1c3eaec

2015-02-28 22:25:06 +0000

[diff] [blame]

396

// Several blend and other instructions with masks used the wrong number of

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

397

// bits.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

398

if (IsX86 && Name == "sse41.insertps") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

399

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,

400

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

401

if (IsX86 && Name == "sse41.dppd") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

402

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,

403

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

404

if (IsX86 && Name == "sse41.dpps") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

405

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,

406

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

407

if (IsX86 && Name == "sse41.mpsadbw") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

408

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,

409

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

410

if (IsX86 && Name == "avx.dp.ps.256") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

411

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,

412

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

413

if (IsX86 && Name == "avx2.mpsadbw") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

414

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,

415

NewFn);

Craig Topper

29f2e95

2015-01-25 23:26:02 +0000

[diff] [blame]

416

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

417

// frcz.ss/sd may need to have an argument dropped. Added in 3.2

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

418

if (IsX86 && Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

419

rename(F);

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

420

NewFn = Intrinsic::getDeclaration(F->getParent(),

421

Intrinsic::x86_xop_vfrcz_ss);

422

return true;

423

}

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

424

if (IsX86 && Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

425

rename(F);

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

426

NewFn = Intrinsic::getDeclaration(F->getParent(),

427

Intrinsic::x86_xop_vfrcz_sd);

428

return true;

429

}

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

430

// Upgrade any XOP PERMIL2 index operand still using a float/double vector.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

431

if (IsX86 && Name.startswith("xop.vpermil2")) { // Added in 3.9

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

432

auto Params = F->getFunctionType()->params();

433

auto Idx = Params[2];

434

if (Idx->getScalarType()->isFloatingPointTy()) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

435

rename(F);

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

436

unsigned IdxSize = Idx->getPrimitiveSizeInBits();

437

unsigned EltSize = Idx->getScalarSizeInBits();

438

Intrinsic::ID Permil2ID;

439

if (EltSize == 64 && IdxSize == 128)

440

Permil2ID = Intrinsic::x86_xop_vpermil2pd;

441

else if (EltSize == 32 && IdxSize == 128)

442

Permil2ID = Intrinsic::x86_xop_vpermil2ps;

443

else if (EltSize == 64 && IdxSize == 256)

444

Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;

445

else

446

Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;

447

NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);

448

return true;

449

}

450

}

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

451

break;

452

}

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

453

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

454

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

455

// This may not belong here. This function is effectively being overloaded

456

// to both detect an intrinsic which needs upgrading, and to provide the

457

// upgraded form of the intrinsic. We should perhaps have two separate

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

458

// functions for this.

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

459

return false;

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

460

}

461

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

462

bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

463

NewFn = nullptr;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

464

bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

465

assert(F != NewFn && "Intrinsic function upgraded to the same function");

Duncan Sands

38ef3a8

2007-12-03 20:06:50 +0000

[diff] [blame]

466

467

// Upgrade intrinsic attributes. This does not change the function.

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

468

if (NewFn)

469

F = NewFn;

Pete Cooper

9e1d335

2015-05-20 17:16:39 +0000

[diff] [blame]

470

if (Intrinsic::ID id = F->getIntrinsicID())

471

F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));

Duncan Sands

38ef3a8

2007-12-03 20:06:50 +0000

[diff] [blame]

return Upgraded;

}

Bill Wendling

2010-09-10 18:51:56 +0000

[diff] [blame]

475

bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {

Chris Lattner

80ed9dc

2011-06-18 06:05:24 +0000

[diff] [blame]

476

// Nothing to do yet.

Bill Wendling

e26fffc

2010-09-10 18:51:56 +0000

[diff] [blame]

return false;

}

Simon Pilgrim

2016-06-09 21:09:03 +0000

[diff] [blame]

480

// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

481

// to byte shuffles.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

482

static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

483

Value *Op, unsigned Shift) {

484

Type *ResultTy = Op->getType();

485

unsigned NumElts = ResultTy->getVectorNumElements() * 8;

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

486

487

// Bitcast from a 64-bit element type to a byte element type.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

488

Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

489

Op = Builder.CreateBitCast(Op, VecTy, "cast");

490

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

491

// We'll be shuffling in zeroes.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

492

Value *Res = Constant::getNullValue(VecTy);

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

493

494

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

495

// we'll just return the zero vector.

496

if (Shift < 16) {

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

497

uint32_t Idxs[64];

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

498

// 256/512-bit version is split into 2/4 16-byte lanes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

499

for (unsigned l = 0; l != NumElts; l += 16)

500

for (unsigned i = 0; i != 16; ++i) {

501

unsigned Idx = NumElts + i - Shift;

502

if (Idx < NumElts)

503

Idx -= NumElts - 16; // end of lane, switch operand.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

504

Idxs[l + i] = Idx + l;

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

505

}

506

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

507

Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

508

}

509

510

// Bitcast back to a 64-bit element type.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

511

return Builder.CreateBitCast(Res, ResultTy, "cast");

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

512

}

513

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

514

// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them

515

// to byte shuffles.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

516

static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

517

unsigned Shift) {

518

Type *ResultTy = Op->getType();

519

unsigned NumElts = ResultTy->getVectorNumElements() * 8;

520

521

// Bitcast from a 64-bit element type to a byte element type.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

522

Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

523

Op = Builder.CreateBitCast(Op, VecTy, "cast");

524

525

// We'll be shuffling in zeroes.

526

Value *Res = Constant::getNullValue(VecTy);

527

528

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

529

// we'll just return the zero vector.

530

if (Shift < 16) {

531

uint32_t Idxs[64];

532

// 256/512-bit version is split into 2/4 16-byte lanes.

533

for (unsigned l = 0; l != NumElts; l += 16)

534

for (unsigned i = 0; i != 16; ++i) {

535

unsigned Idx = i + Shift;

536

if (Idx >= 16)

537

Idx += NumElts - 16; // end of lane, switch operand.

538

Idxs[l + i] = Idx + l;

539

}

540

541

Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));

542

}

543

544

// Bitcast back to a 64-bit element type.

545

return Builder.CreateBitCast(Res, ResultTy, "cast");

546

}

547

548

static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,

549

unsigned NumElts) {

550

llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(),

551

cast<IntegerType>(Mask->getType())->getBitWidth());

552

Mask = Builder.CreateBitCast(Mask, MaskTy);

553

554

// If we have less than 8 elements, then the starting mask was an i8 and

555

// we need to extract down to the right number of elements.

556

if (NumElts < 8) {

557

uint32_t Indices[4];

558

for (unsigned i = 0; i != NumElts; ++i)

559

Indices[i] = i;

560

Mask = Builder.CreateShuffleVector(Mask, Mask,

561

makeArrayRef(Indices, NumElts),

"extract");

}

return Mask;

}

static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,

569

Value *Op0, Value *Op1) {

570

// If the mask is all ones just emit the align operation.

571

if (const auto *C = dyn_cast<Constant>(Mask))

572

if (C->isAllOnesValue())

573

return Op0;

574

575

Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements());

576

return Builder.CreateSelect(Mask, Op0, Op1);

577

}

578

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

579

// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.

580

// PALIGNR handles large immediates by shifting while VALIGN masks the immediate

581

// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.

582

static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,

583

Value *Op1, Value *Shift,

584

Value *Passthru, Value *Mask,

585

bool IsVALIGN) {

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

586

unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();

587

588

unsigned NumElts = Op0->getType()->getVectorNumElements();

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

589

assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");

590

assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");

591

assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");

592

593

// Mask the immediate for VALIGN.

594

if (IsVALIGN)

595

ShiftVal &= (NumElts - 1);

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

596

597

// If palignr is shifting the pair of vectors more than the size of two

598

// lanes, emit zero.

599

if (ShiftVal >= 32)

600

return llvm::Constant::getNullValue(Op0->getType());

601

602

// If palignr is shifting the pair of input vectors more than one lane,

603

// but less than two lanes, convert to shifting in zeroes.

if (ShiftVal > 16) {

ShiftVal -= 16;

Op1 = Op0;

Op0 = llvm::Constant::getNullValue(Op0->getType());

608

}

609

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

610

uint32_t Indices[64];

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

611

// 256-bit palignr operates on 128-bit lanes so we need to handle that

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

612

for (unsigned l = 0; l < NumElts; l += 16) {

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

613

for (unsigned i = 0; i != 16; ++i) {

614

unsigned Idx = ShiftVal + i;

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

615

if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

616

Idx += NumElts - 16; // End of lane, switch operand.

617

Indices[l + i] = Idx + l;

}

}

Value *Align = Builder.CreateShuffleVector(Op1, Op0,

622

makeArrayRef(Indices, NumElts),

623

"palignr");

624

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

625

return EmitX86Select(Builder, Mask, Align, Passthru);

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

626

}

627

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

628

static Value *UpgradeMaskedStore(IRBuilder<> &Builder,

Craig Topper

50f85c2

2016-05-31 01:50:02 +0000

[diff] [blame]

629

Value *Ptr, Value *Data, Value *Mask,

630

bool Aligned) {

631

// Cast the pointer to the right type.

632

Ptr = Builder.CreateBitCast(Ptr,

633

llvm::PointerType::getUnqual(Data->getType()));

634

unsigned Align =

635

Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1;

636

637

// If the mask is all ones just emit a regular store.

638

if (const auto *C = dyn_cast<Constant>(Mask))

639

if (C->isAllOnesValue())

640

return Builder.CreateAlignedStore(Data, Ptr, Align);

641

642

// Convert the mask from an integer type to a vector of i1.

643

unsigned NumElts = Data->getType()->getVectorNumElements();

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

644

Mask = getX86MaskVec(Builder, Mask, NumElts);

Craig Topper

50f85c2

2016-05-31 01:50:02 +0000

[diff] [blame]

645

return Builder.CreateMaskedStore(Data, Ptr, Align, Mask);

646

}

647

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

648

static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

649

Value *Ptr, Value *Passthru, Value *Mask,

650

bool Aligned) {

651

// Cast the pointer to the right type.

652

Ptr = Builder.CreateBitCast(Ptr,

653

llvm::PointerType::getUnqual(Passthru->getType()));

654

unsigned Align =

655

Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;

656

657

// If the mask is all ones just emit a regular store.

658

if (const auto *C = dyn_cast<Constant>(Mask))

659

if (C->isAllOnesValue())

660

return Builder.CreateAlignedLoad(Ptr, Align);

661

662

// Convert the mask from an integer type to a vector of i1.

663

unsigned NumElts = Passthru->getType()->getVectorNumElements();

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

664

Mask = getX86MaskVec(Builder, Mask, NumElts);

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

665

return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru);

666

}

667

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

668

static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,

669

ICmpInst::Predicate Pred) {

670

Value *Op0 = CI.getArgOperand(0);

671

Value *Op1 = CI.getArgOperand(1);

672

Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1);

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

673

Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);

674

675

if (CI.getNumArgOperands() == 4)

676

Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));

677

678

return Res;

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

679

}

680

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

681

static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,

682

ICmpInst::Predicate Pred) {

683

Value *Op0 = CI.getArgOperand(0);

684

unsigned NumElts = Op0->getType()->getVectorNumElements();

685

Value *Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));

686

687

Value *Mask = CI.getArgOperand(2);

688

const auto *C = dyn_cast<Constant>(Mask);

689

if (!C || !C->isAllOnesValue())

690

Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts));

if (NumElts < 8) {

uint32_t Indices[8];

for (unsigned i = 0; i != NumElts; ++i)

695

Indices[i] = i;

696

for (unsigned i = NumElts; i != 8; ++i)

Craig Topper

d5d2a35

2016-07-07 06:11:07 +0000

[diff] [blame]

697

Indices[i] = NumElts + i % NumElts;

698

Cmp = Builder.CreateShuffleVector(Cmp,

699

Constant::getNullValue(Cmp->getType()),

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

700

Indices);

701

}

702

return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(),

703

std::max(NumElts, 8U)));

704

}

705

Craig Topper

c9467ed

2016-11-06 16:29:08 +0000

[diff] [blame]

706

// Replace a masked intrinsic with an older unmasked intrinsic.

707

static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,

708

Intrinsic::ID IID) {

709

Function *F = CI.getCalledFunction();

710

Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);

711

Value *Rep = Builder.CreateCall(Intrin,

712

{ CI.getArgOperand(0), CI.getArgOperand(1) });

713

return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));

714

}

715

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

716

static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {

717

Value* A = CI.getArgOperand(0);

718

Value* B = CI.getArgOperand(1);

719

Value* Src = CI.getArgOperand(2);

720

Value* Mask = CI.getArgOperand(3);

721

722

Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));

723

Value* Cmp = Builder.CreateIsNotNull(AndNode);

724

Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);

725

Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);

726

Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);

727

return Builder.CreateInsertElement(A, Select, (uint64_t)0);

728

}

Craig Topper

c9467ed

2016-11-06 16:29:08 +0000

[diff] [blame]

729

Sanjay Patel

595098f

2016-06-15 22:01:28 +0000

[diff] [blame]

730

/// Upgrade a call to an old intrinsic. All argument and return casting must be

731

/// provided to seamlessly integrate with existing context.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

732

void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

733

Function *F = CI->getCalledFunction();

Nick Lewycky

2eb3ade

2011-12-12 22:59:34 +0000

[diff] [blame]

734

LLVMContext &C = CI->getContext();

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

735

IRBuilder<> Builder(C);

Duncan P. N. Exon Smith

52888a6

2015-10-08 23:49:46 +0000

[diff] [blame]

736

Builder.SetInsertPoint(CI->getParent(), CI->getIterator());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

737

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

738

assert(F && "Intrinsic call is not direct?");

739

740

if (!NewFn) {

741

// Get the Function's name.

742

StringRef Name = F->getName();

743

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

744

assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");

745

Name = Name.substr(5);

746

747

bool IsX86 = Name.startswith("x86.");

748

if (IsX86)

749

Name = Name.substr(4);

750

Craig Topper

a4744d1

2016-12-10 21:15:48 +0000

[diff] [blame]

751

if (IsX86 && Name.startswith("sse4a.movnt.")) {

752

Module *M = F->getParent();

753

SmallVector<Metadata *, 1> Elts;

754

Elts.push_back(

755

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

756

MDNode *Node = MDNode::get(C, Elts);

757

758

Value *Arg0 = CI->getArgOperand(0);

759

Value *Arg1 = CI->getArgOperand(1);

760

761

// Nontemporal (unaligned) store of the 0'th element of the float/double

762

// vector.

763

Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();

764

PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);

765

Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");

766

Value *Extract =

767

Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");

768

769

StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1);

770

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

771

772

// Remove intrinsic.

773

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx.movnt.") ||

778

Name.startswith("avx512.storent."))) {

779

Module *M = F->getParent();

780

SmallVector<Metadata *, 1> Elts;

781

Elts.push_back(

782

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

783

MDNode *Node = MDNode::get(C, Elts);

784

785

Value *Arg0 = CI->getArgOperand(0);

786

Value *Arg1 = CI->getArgOperand(1);

787

788

// Convert the type of the pointer to a pointer to the stored type.

789

Value *BC = Builder.CreateBitCast(Arg0,

790

PointerType::getUnqual(Arg1->getType()),

791

"cast");

792

VectorType *VTy = cast<VectorType>(Arg1->getType());

793

StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC,

794

VTy->getBitWidth() / 8);

795

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

796

797

// Remove intrinsic.

798

CI->eraseFromParent();

return;

}

if (IsX86 && Name == "sse2.storel.dq") {

803

Value *Arg0 = CI->getArgOperand(0);

804

Value *Arg1 = CI->getArgOperand(1);

805

806

Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);

807

Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

808

Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);

809

Value *BC = Builder.CreateBitCast(Arg0,

810

PointerType::getUnqual(Elt->getType()),

811

"cast");

812

Builder.CreateAlignedStore(Elt, BC, 1);

813

814

// Remove intrinsic.

815

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("sse.storeu.") ||

820

Name.startswith("sse2.storeu.") ||

821

Name.startswith("avx.storeu."))) {

822

Value *Arg0 = CI->getArgOperand(0);

823

Value *Arg1 = CI->getArgOperand(1);

824

825

Arg0 = Builder.CreateBitCast(Arg0,

826

PointerType::getUnqual(Arg1->getType()),

827

"cast");

828

Builder.CreateAlignedStore(Arg1, Arg0, 1);

829

830

// Remove intrinsic.

831

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx512.mask.storeu."))) {

836

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

837

CI->getArgOperand(2), /*Aligned*/false);

838

839

// Remove intrinsic.

840

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx512.mask.store."))) {

845

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

846

CI->getArgOperand(2), /*Aligned*/true);

847

848

// Remove intrinsic.

849

CI->eraseFromParent();

return;

}

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

853

Value *Rep;

Sanjay Patel

595098f

2016-06-15 22:01:28 +0000

[diff] [blame]

854

// Upgrade packed integer vector compare intrinsics to compare instructions.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

855

if (IsX86 && (Name.startswith("sse2.pcmpeq.") ||

856

Name.startswith("avx2.pcmpeq."))) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

857

Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),

858

"pcmpeq");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

859

Rep = Builder.CreateSExt(Rep, CI->getType(), "");

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

860

} else if (IsX86 && (Name.startswith("sse2.pcmpgt.") ||

861

Name.startswith("avx2.pcmpgt."))) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

862

Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),

863

"pcmpgt");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

864

Rep = Builder.CreateSExt(Rep, CI->getType(), "");

Craig Topper

6910fa0

2016-11-16 05:24:10 +0000

[diff] [blame]

865

} else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {

866

Type *I32Ty = Type::getInt32Ty(C);

867

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

868

ConstantInt::get(I32Ty, 0));

869

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

870

ConstantInt::get(I32Ty, 0));

871

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

872

Builder.CreateFAdd(Elt0, Elt1),

873

ConstantInt::get(I32Ty, 0));

874

} else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) {

875

Type *I32Ty = Type::getInt32Ty(C);

876

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

877

ConstantInt::get(I32Ty, 0));

878

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

879

ConstantInt::get(I32Ty, 0));

880

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

881

Builder.CreateFSub(Elt0, Elt1),

882

ConstantInt::get(I32Ty, 0));

883

} else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) {

884

Type *I32Ty = Type::getInt32Ty(C);

885

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

886

ConstantInt::get(I32Ty, 0));

887

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

888

ConstantInt::get(I32Ty, 0));

889

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

890

Builder.CreateFMul(Elt0, Elt1),

891

ConstantInt::get(I32Ty, 0));

892

} else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) {

893

Type *I32Ty = Type::getInt32Ty(C);

894

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

895

ConstantInt::get(I32Ty, 0));

896

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

897

ConstantInt::get(I32Ty, 0));

898

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

899

Builder.CreateFDiv(Elt0, Elt1),

900

ConstantInt::get(I32Ty, 0));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

901

} else if (IsX86 && Name.startswith("avx512.mask.pcmpeq.")) {

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

902

Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_EQ);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

903

} else if (IsX86 && Name.startswith("avx512.mask.pcmpgt.")) {

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

904

Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_SGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

905

} else if (IsX86 && (Name == "sse41.pmaxsb" ||

906

Name == "sse2.pmaxs.w" ||

907

Name == "sse41.pmaxsd" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

908

Name.startswith("avx2.pmaxs") ||

909

Name.startswith("avx512.mask.pmaxs"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

910

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

911

} else if (IsX86 && (Name == "sse2.pmaxu.b" ||

912

Name == "sse41.pmaxuw" ||

913

Name == "sse41.pmaxud" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

914

Name.startswith("avx2.pmaxu") ||

915

Name.startswith("avx512.mask.pmaxu"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

916

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

917

} else if (IsX86 && (Name == "sse41.pminsb" ||

918

Name == "sse2.pmins.w" ||

919

Name == "sse41.pminsd" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

920

Name.startswith("avx2.pmins") ||

921

Name.startswith("avx512.mask.pmins"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

922

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

923

} else if (IsX86 && (Name == "sse2.pminu.b" ||

924

Name == "sse41.pminuw" ||

925

Name == "sse41.pminud" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

926

Name.startswith("avx2.pminu") ||

927

Name.startswith("avx512.mask.pminu"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

928

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

929

} else if (IsX86 && (Name == "sse2.cvtdq2pd" ||

930

Name == "sse2.cvtps2pd" ||

931

Name == "avx.cvtdq2.pd.256" ||

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

932

Name == "avx.cvt.ps2.pd.256" ||

933

Name.startswith("avx512.mask.cvtdq2pd.") ||

934

Name.startswith("avx512.mask.cvtudq2pd."))) {

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

935

// Lossless i32/float to double conversion.

936

// Extract the bottom elements if necessary and convert to double vector.

937

Value *Src = CI->getArgOperand(0);

938

VectorType *SrcTy = cast<VectorType>(Src->getType());

939

VectorType *DstTy = cast<VectorType>(CI->getType());

940

Rep = CI->getArgOperand(0);

941

942

unsigned NumDstElts = DstTy->getNumElements();

943

if (NumDstElts < SrcTy->getNumElements()) {

944

assert(NumDstElts == 2 && "Unexpected vector size");

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

945

uint32_t ShuffleMask[2] = { 0, 1 };

946

Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy),

947

ShuffleMask);

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

948

}

949

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

950

bool SInt2Double = (StringRef::npos != Name.find("cvtdq2"));

951

bool UInt2Double = (StringRef::npos != Name.find("cvtudq2"));

952

if (SInt2Double)

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

953

Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd");

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

954

else if (UInt2Double)

955

Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd");

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

956

else

957

Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

958

959

if (CI->getNumArgOperands() == 3)

960

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

961

CI->getArgOperand(1));

Craig Topper

a57d2ca

2016-09-03 23:55:13 +0000

[diff] [blame]

962

} else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

963

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

964

CI->getArgOperand(1), CI->getArgOperand(2),

965

/*Aligned*/false);

Craig Topper

a57d2ca

2016-09-03 23:55:13 +0000

[diff] [blame]

966

} else if (IsX86 && (Name.startswith("avx512.mask.load."))) {

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

967

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

968

CI->getArgOperand(1),CI->getArgOperand(2),

969

/*Aligned*/true);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

970

} else if (IsX86 && Name.startswith("xop.vpcom")) {

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

971

Intrinsic::ID intID;

972

if (Name.endswith("ub"))

973

intID = Intrinsic::x86_xop_vpcomub;

974

else if (Name.endswith("uw"))

975

intID = Intrinsic::x86_xop_vpcomuw;

976

else if (Name.endswith("ud"))

977

intID = Intrinsic::x86_xop_vpcomud;

978

else if (Name.endswith("uq"))

979

intID = Intrinsic::x86_xop_vpcomuq;

980

else if (Name.endswith("b"))

981

intID = Intrinsic::x86_xop_vpcomb;

982

else if (Name.endswith("w"))

983

intID = Intrinsic::x86_xop_vpcomw;

984

else if (Name.endswith("d"))

985

intID = Intrinsic::x86_xop_vpcomd;

986

else if (Name.endswith("q"))

987

intID = Intrinsic::x86_xop_vpcomq;

988

else

989

llvm_unreachable("Unknown suffix");

990

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

991

Name = Name.substr(9); // strip off "xop.vpcom"

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

992

unsigned Imm;

993

if (Name.startswith("lt"))

994

Imm = 0;

995

else if (Name.startswith("le"))

996

Imm = 1;

997

else if (Name.startswith("gt"))

998

Imm = 2;

999

else if (Name.startswith("ge"))

1000

Imm = 3;

1001

else if (Name.startswith("eq"))

1002

Imm = 4;

1003

else if (Name.startswith("ne"))

1004

Imm = 5;

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

1005

else if (Name.startswith("false"))

Craig Topper

e32546d

2015-02-13 07:42:15 +0000

[diff] [blame]

1006

Imm = 6;

1007

else if (Name.startswith("true"))

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

1008

Imm = 7;

1009

else

1010

llvm_unreachable("Unknown condition");

1011

1012

Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1013

Rep =

1014

Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),

1015

Builder.getInt8(Imm)});

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1016

} else if (IsX86 && Name == "xop.vpcmov") {

Simon Pilgrim

e88dc04

2015-11-03 20:27:01 +0000

[diff] [blame]

1017

Value *Arg0 = CI->getArgOperand(0);

1018

Value *Arg1 = CI->getArgOperand(1);

1019

Value *Sel = CI->getArgOperand(2);

1020

unsigned NumElts = CI->getType()->getVectorNumElements();

1021

Constant *MinusOne = ConstantVector::getSplat(NumElts, Builder.getInt64(-1));

1022

Value *NotSel = Builder.CreateXor(Sel, MinusOne);

1023

Value *Sel0 = Builder.CreateAnd(Arg0, Sel);

1024

Value *Sel1 = Builder.CreateAnd(Arg1, NotSel);

1025

Rep = Builder.CreateOr(Sel0, Sel1);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1026

} else if (IsX86 && Name == "sse42.crc32.64.8") {

Craig Topper

ef9e993

2013-10-15 05:20:47 +0000

[diff] [blame]

1027

Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),

1028

Intrinsic::x86_sse42_crc32_32_8);

1029

Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1030

Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});

Craig Topper

ef9e993

2013-10-15 05:20:47 +0000

[diff] [blame]

1031

Rep = Builder.CreateZExt(Rep, CI->getType(), "");

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1032

} else if (IsX86 && Name.startswith("avx.vbroadcast.s")) {

Adam Nemet

3906680

2014-05-29 23:35:33 +0000

[diff] [blame]

1033

// Replace broadcasts with a series of insertelements.

1034

Type *VecTy = CI->getType();

1035

Type *EltTy = VecTy->getVectorElementType();

1036

unsigned EltNum = VecTy->getVectorNumElements();

1037

Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),

1038

EltTy->getPointerTo());

David Blaikie

0c28fd7

2015-05-20 21:46:30 +0000

[diff] [blame]

1039

Value *Load = Builder.CreateLoad(EltTy, Cast);

Adam Nemet

3906680

2014-05-29 23:35:33 +0000

[diff] [blame]

1040

Type *I32Ty = Type::getInt32Ty(C);

1041

Rep = UndefValue::get(VecTy);

1042

for (unsigned I = 0; I < EltNum; ++I)

1043

Rep = Builder.CreateInsertElement(Rep, Load,

1044

ConstantInt::get(I32Ty, I));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1045

} else if (IsX86 && (Name.startswith("sse41.pmovsx") ||

1046

Name.startswith("sse41.pmovzx") ||

1047

Name.startswith("avx2.pmovsx") ||

Craig Topper

b110e04

2016-11-07 02:12:57 +0000

[diff] [blame]

1048

Name.startswith("avx2.pmovzx") ||

1049

Name.startswith("avx512.mask.pmovsx") ||

1050

Name.startswith("avx512.mask.pmovzx"))) {

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1051

VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());

1052

VectorType *DstTy = cast<VectorType>(CI->getType());

1053

unsigned NumDstElts = DstTy->getNumElements();

1054

Simon Pilgrim

9602d67

2016-05-28 18:03:41 +0000

[diff] [blame]

1055

// Extract a subvector of the first NumDstElts lanes and sign/zero extend.

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1056

SmallVector<uint32_t, 8> ShuffleMask(NumDstElts);

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

1057

for (unsigned i = 0; i != NumDstElts; ++i)

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1058

ShuffleMask[i] = i;

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1059

1060

Value *SV = Builder.CreateShuffleVector(

1061

CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);

Simon Pilgrim

9602d67

2016-05-28 18:03:41 +0000

[diff] [blame]

1062

1063

bool DoSext = (StringRef::npos != Name.find("pmovsx"));

1064

Rep = DoSext ? Builder.CreateSExt(SV, DstTy)

1065

: Builder.CreateZExt(SV, DstTy);

Craig Topper

b110e04

2016-11-07 02:12:57 +0000

[diff] [blame]

1066

// If there are 3 arguments, it's a masked intrinsic so we need a select.

1067

if (CI->getNumArgOperands() == 3)

1068

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1069

CI->getArgOperand(1));

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1070

} else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||

1071

Name == "avx2.vbroadcasti128")) {

1072

// Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.

1073

Type *EltTy = CI->getType()->getVectorElementType();

1074

unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();

1075

Type *VT = VectorType::get(EltTy, NumSrcElts);

David Blaikie

0c28fd7

2015-05-20 21:46:30 +0000

[diff] [blame]

1076

Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),

1077

PointerType::getUnqual(VT));

Chandler Carruth

0215e76

2016-08-10 07:41:26 +0000

[diff] [blame]

1078

Value *Load = Builder.CreateAlignedLoad(Op, 1);

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1079

if (NumSrcElts == 2)

1080

Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),

1081

{ 0, 1, 0, 1 });

1082

else

1083

Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),

1084

{ 0, 1, 2, 3, 0, 1, 2, 3 });

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1085

} else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||

Simon Pilgrim

4e96fbf

2016-07-05 13:58:47 +0000

[diff] [blame]

1086

Name.startswith("avx2.vbroadcast") ||

1087

Name.startswith("avx512.pbroadcast") ||

1088

Name.startswith("avx512.mask.broadcast.s"))) {

Ahmed Bougacha

1a498705

2015-08-20 20:36:19 +0000

[diff] [blame]

1089

// Replace vp?broadcasts with a vector shuffle.

1090

Value *Op = CI->getArgOperand(0);

1091

unsigned NumElts = CI->getType()->getVectorNumElements();

1092

Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts);

1093

Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),

1094

Constant::getNullValue(MaskTy));

Simon Pilgrim

4e96fbf

2016-07-05 13:58:47 +0000

[diff] [blame]

1095

1096

if (CI->getNumArgOperands() == 3)

1097

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1098

CI->getArgOperand(1));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1099

} else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

1100

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

1101

CI->getArgOperand(1),

1102

CI->getArgOperand(2),

1103

CI->getArgOperand(3),

1104

CI->getArgOperand(4),

1105

false);

1106

} else if (IsX86 && Name.startswith("avx512.mask.valign.")) {

1107

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

1108

CI->getArgOperand(1),

1109

CI->getArgOperand(2),

1110

CI->getArgOperand(3),

1111

CI->getArgOperand(4),

1112

true);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1113

} else if (IsX86 && (Name == "sse2.psll.dq" ||

1114

Name == "avx2.psll.dq")) {

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

1115

// 128/256-bit shift left specified in bits.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1116

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1117

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1118

Shift / 8); // Shift is in bits.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1119

} else if (IsX86 && (Name == "sse2.psrl.dq" ||

1120

Name == "avx2.psrl.dq")) {

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

1121

// 128/256-bit shift right specified in bits.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1122

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1123

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1124

Shift / 8); // Shift is in bits.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1125

} else if (IsX86 && (Name == "sse2.psll.dq.bs" ||

1126

Name == "avx2.psll.dq.bs" ||

1127

Name == "avx512.psll.dq.512")) {

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

1128

// 128/256/512-bit shift left specified in bytes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1129

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1130

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1131

} else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||

1132

Name == "avx2.psrl.dq.bs" ||

1133

Name == "avx512.psrl.dq.512")) {

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

1134

// 128/256/512-bit shift right specified in bytes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1135

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1136

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1137

} else if (IsX86 && (Name == "sse41.pblendw" ||

1138

Name.startswith("sse41.blendp") ||

1139

Name.startswith("avx.blend.p") ||

1140

Name == "avx2.pblendw" ||

1141

Name.startswith("avx2.pblendd."))) {

Craig Topper

782d620

2015-02-28 19:33:17 +0000

[diff] [blame]

1142

Value *Op0 = CI->getArgOperand(0);

1143

Value *Op1 = CI->getArgOperand(1);

1144

unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1145

VectorType *VecTy = cast<VectorType>(CI->getType());

1146

unsigned NumElts = VecTy->getNumElements();

1147

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1148

SmallVector<uint32_t, 16> Idxs(NumElts);

1149

for (unsigned i = 0; i != NumElts; ++i)

1150

Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;

Craig Topper

782d620

2015-02-28 19:33:17 +0000

[diff] [blame]

1151

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1152

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1153

} else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||

1154

Name == "avx2.vinserti128")) {

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1155

Value *Op0 = CI->getArgOperand(0);

1156

Value *Op1 = CI->getArgOperand(1);

1157

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1158

VectorType *VecTy = cast<VectorType>(CI->getType());

1159

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1160

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1161

// Mask off the high bits of the immediate value; hardware ignores those.

1162

Imm = Imm & 1;

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1163

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1164

// Extend the second operand into a vector that is twice as big.

1165

Value *UndefV = UndefValue::get(Op1->getType());

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1166

SmallVector<uint32_t, 8> Idxs(NumElts);

1167

for (unsigned i = 0; i != NumElts; ++i)

1168

Idxs[i] = i;

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1169

Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1170

1171

// Insert the second operand into the first operand.

1172

1173

// Note that there is no guarantee that instruction lowering will actually

1174

// produce a vinsertf128 instruction for the created shuffles. In

1175

// particular, the 0 immediate case involves no lane changes, so it can

1176

// be handled as a blend.

1177

1178

// Example of shuffle mask for 32-bit elements:

1179

// Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>

1180

// Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >

1181

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1182

// The low half of the result is either the low half of the 1st operand

1183

// or the low half of the 2nd operand (the inserted vector).

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1184

for (unsigned i = 0; i != NumElts / 2; ++i)

1185

Idxs[i] = Imm ? i : (i + NumElts);

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1186

// The high half of the result is either the low half of the 2nd operand

1187

// (the inserted vector) or the high half of the 1st operand.

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1188

for (unsigned i = NumElts / 2; i != NumElts; ++i)

1189

Idxs[i] = Imm ? (i + NumElts / 2) : i;

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1190

Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1191

} else if (IsX86 && (Name.startswith("avx.vextractf128.") ||

1192

Name == "avx2.vextracti128")) {

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1193

Value *Op0 = CI->getArgOperand(0);

1194

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1195

VectorType *VecTy = cast<VectorType>(CI->getType());

1196

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1197

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1198

// Mask off the high bits of the immediate value; hardware ignores those.

1199

Imm = Imm & 1;

1200

1201

// Get indexes for either the high half or low half of the input vector.

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1202

SmallVector<uint32_t, 4> Idxs(NumElts);

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1203

for (unsigned i = 0; i != NumElts; ++i) {

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1204

Idxs[i] = Imm ? (i + NumElts) : i;

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1205

}

1206

1207

Value *UndefV = UndefValue::get(Op0->getType());

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1208

Rep = Builder.CreateShuffleVector(Op0, UndefV, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1209

} else if (!IsX86 && Name == "stackprotectorcheck") {

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

1210

Rep = nullptr;

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1211

} else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||

1212

Name.startswith("avx512.mask.perm.di."))) {

Simon Pilgrim

02d435d

2016-07-04 14:19:05 +0000

[diff] [blame]

1213

Value *Op0 = CI->getArgOperand(0);

1214

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1215

VectorType *VecTy = cast<VectorType>(CI->getType());

1216

unsigned NumElts = VecTy->getNumElements();

1217

1218

SmallVector<uint32_t, 8> Idxs(NumElts);

1219

for (unsigned i = 0; i != NumElts; ++i)

1220

Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);

1221

1222

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

1223

1224

if (CI->getNumArgOperands() == 4)

1225

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1226

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1227

} else if (IsX86 && (Name.startswith("avx.vpermil.") ||

1228

Name == "sse2.pshuf.d" ||

1229

Name.startswith("avx512.mask.vpermil.p") ||

1230

Name.startswith("avx512.mask.pshuf.d."))) {

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1231

Value *Op0 = CI->getArgOperand(0);

1232

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1233

VectorType *VecTy = cast<VectorType>(CI->getType());

1234

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

9fca300

2016-07-04 12:40:54 +0000

[diff] [blame]

1235

// Calculate the size of each index in the immediate.

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1236

unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();

1237

unsigned IdxMask = ((1 << IdxSize) - 1);

1238

1239

SmallVector<uint32_t, 8> Idxs(NumElts);

1240

// Lookup the bits for this element, wrapping around the immediate every

1241

// 8-bits. Elements are grouped into sets of 2 or 4 elements so we need

1242

// to offset by the first index of each group.

1243

for (unsigned i = 0; i != NumElts; ++i)

1244

Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);

1245

1246

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1247

1248

if (CI->getNumArgOperands() == 4)

1249

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1250

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1251

} else if (IsX86 && (Name == "sse2.pshufl.w" ||

1252

Name.startswith("avx512.mask.pshufl.w."))) {

Craig Topper

1067986

2016-06-12 14:11:32 +0000

[diff] [blame]

1253

Value *Op0 = CI->getArgOperand(0);

1254

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1255

unsigned NumElts = CI->getType()->getVectorNumElements();

1256

1257

SmallVector<uint32_t, 16> Idxs(NumElts);

1258

for (unsigned l = 0; l != NumElts; l += 8) {

1259

for (unsigned i = 0; i != 4; ++i)

1260

Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;

1261

for (unsigned i = 4; i != 8; ++i)

Idxs[i + l] = i + l;

}

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1266

1267

if (CI->getNumArgOperands() == 4)

1268

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1269

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1270

} else if (IsX86 && (Name == "sse2.pshufh.w" ||

1271

Name.startswith("avx512.mask.pshufh.w."))) {

Craig Topper

1067986

2016-06-12 14:11:32 +0000

[diff] [blame]

1272

Value *Op0 = CI->getArgOperand(0);

1273

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1274

unsigned NumElts = CI->getType()->getVectorNumElements();

1275

1276

SmallVector<uint32_t, 16> Idxs(NumElts);

1277

for (unsigned l = 0; l != NumElts; l += 8) {

1278

for (unsigned i = 0; i != 4; ++i)

1279

Idxs[i + l] = i + l;

1280

for (unsigned i = 0; i != 4; ++i)

1281

Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;

1282

}

1283

1284

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1285

1286

if (CI->getNumArgOperands() == 4)

1287

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1288

CI->getArgOperand(2));

Craig Topper

4619c9e

2016-09-13 07:40:53 +0000

[diff] [blame]

1289

} else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {

1290

Value *Op0 = CI->getArgOperand(0);

1291

Value *Op1 = CI->getArgOperand(1);

1292

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1293

unsigned NumElts = CI->getType()->getVectorNumElements();

1294

1295

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1296

unsigned HalfLaneElts = NumLaneElts / 2;

1297

1298

SmallVector<uint32_t, 16> Idxs(NumElts);

1299

for (unsigned i = 0; i != NumElts; ++i) {

1300

// Base index is the starting element of the lane.

1301

Idxs[i] = i - (i % NumLaneElts);

1302

// If we are half way through the lane switch to the other source.

1303

if ((i % NumLaneElts) >= HalfLaneElts)

1304

Idxs[i] += NumElts;

1305

// Now select the specific element. By adding HalfLaneElts bits from

1306

// the immediate. Wrapping around the immediate every 8-bits.

1307

Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);

1308

}

1309

1310

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1311

1312

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,

1313

CI->getArgOperand(3));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1314

} else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||

1315

Name.startswith("avx512.mask.movshdup") ||

1316

Name.startswith("avx512.mask.movsldup"))) {

Simon Pilgrim

19adee9

2016-07-02 14:42:35 +0000

[diff] [blame]

1317

Value *Op0 = CI->getArgOperand(0);

1318

unsigned NumElts = CI->getType()->getVectorNumElements();

1319

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1320

1321

unsigned Offset = 0;

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1322

if (Name.startswith("avx512.mask.movshdup."))

Simon Pilgrim

19adee9

2016-07-02 14:42:35 +0000

[diff] [blame]

1323

Offset = 1;

1324

1325

SmallVector<uint32_t, 16> Idxs(NumElts);

1326

for (unsigned l = 0; l != NumElts; l += NumLaneElts)

1327

for (unsigned i = 0; i != NumLaneElts; i += 2) {

1328

Idxs[i + l + 0] = i + l + Offset;

1329

Idxs[i + l + 1] = i + l + Offset;

1330

}

1331

1332

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

1333

1334

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1335

CI->getArgOperand(1));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1336

} else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||

1337

Name.startswith("avx512.mask.unpckl."))) {

Craig Topper

597aa42

2016-06-23 07:37:33 +0000

[diff] [blame]

1338

Value *Op0 = CI->getArgOperand(0);

1339

Value *Op1 = CI->getArgOperand(1);

1340

int NumElts = CI->getType()->getVectorNumElements();

1341

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1342

1343

SmallVector<uint32_t, 64> Idxs(NumElts);

1344

for (int l = 0; l != NumElts; l += NumLaneElts)

1345

for (int i = 0; i != NumLaneElts; ++i)

1346

Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);

1347

1348

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1349

1350

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1351

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1352

} else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||

1353

Name.startswith("avx512.mask.unpckh."))) {

Craig Topper

597aa42

2016-06-23 07:37:33 +0000

[diff] [blame]

1354

Value *Op0 = CI->getArgOperand(0);

1355

Value *Op1 = CI->getArgOperand(1);

1356

int NumElts = CI->getType()->getVectorNumElements();

1357

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1358

1359

SmallVector<uint32_t, 64> Idxs(NumElts);

1360

for (int l = 0; l != NumElts; l += NumLaneElts)

1361

for (int i = 0; i != NumLaneElts; ++i)

1362

Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);

1363

1364

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1365

1366

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1367

CI->getArgOperand(2));

Craig Topper

a6e6feb

2016-07-12 05:27:53 +0000

[diff] [blame]

1368

} else if (IsX86 && Name.startswith("avx512.mask.pand.")) {

1369

Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1));

1370

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1371

CI->getArgOperand(2));

1372

} else if (IsX86 && Name.startswith("avx512.mask.pandn.")) {

1373

Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)),

1374

CI->getArgOperand(1));

1375

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1376

CI->getArgOperand(2));

1377

} else if (IsX86 && Name.startswith("avx512.mask.por.")) {

1378

Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1));

1379

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1380

CI->getArgOperand(2));

1381

} else if (IsX86 && Name.startswith("avx512.mask.pxor.")) {

1382

Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1));

1383

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1384

CI->getArgOperand(2));

Craig Topper

e75c495

2016-09-02 05:29:17 +0000

[diff] [blame]

1385

} else if (IsX86 && Name.startswith("avx512.mask.and.")) {

1386

VectorType *FTy = cast<VectorType>(CI->getType());

1387

VectorType *ITy = VectorType::getInteger(FTy);

1388

Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1389

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1390

Rep = Builder.CreateBitCast(Rep, FTy);

1391

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1392

CI->getArgOperand(2));

1393

} else if (IsX86 && Name.startswith("avx512.mask.andn.")) {

1394

VectorType *FTy = cast<VectorType>(CI->getType());

1395

VectorType *ITy = VectorType::getInteger(FTy);

1396

Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));

1397

Rep = Builder.CreateAnd(Rep,

1398

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1399

Rep = Builder.CreateBitCast(Rep, FTy);

1400

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1401

CI->getArgOperand(2));

1402

} else if (IsX86 && Name.startswith("avx512.mask.or.")) {

1403

VectorType *FTy = cast<VectorType>(CI->getType());

1404

VectorType *ITy = VectorType::getInteger(FTy);

1405

Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1406

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1407

Rep = Builder.CreateBitCast(Rep, FTy);

1408

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1409

CI->getArgOperand(2));

1410

} else if (IsX86 && Name.startswith("avx512.mask.xor.")) {

1411

VectorType *FTy = cast<VectorType>(CI->getType());

1412

VectorType *ITy = VectorType::getInteger(FTy);

1413

Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1414

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1415

Rep = Builder.CreateBitCast(Rep, FTy);

1416

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1417

CI->getArgOperand(2));

Craig Topper

af0d63d

2016-09-04 02:09:53 +0000

[diff] [blame]

1418

} else if (IsX86 && Name.startswith("avx512.mask.padd.")) {

1419

Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));

1420

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1421

CI->getArgOperand(2));

1422

} else if (IsX86 && Name.startswith("avx512.mask.psub.")) {

1423

Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));

1424

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1425

CI->getArgOperand(2));

1426

} else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {

1427

Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));

1428

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1429

CI->getArgOperand(2));

Craig Topper

8f354a7

2016-11-05 05:35:23 +0000

[diff] [blame]

1430

} else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1431

Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));

1432

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1433

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1434

} else if (IsX86 && Name.startswith("avx512.mask.div.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1435

Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));

1436

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1437

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1438

} else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1439

Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));

1440

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1441

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1442

} else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1443

Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));

1444

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1445

CI->getArgOperand(2));

Craig Topper

7e54533

2016-11-07 00:13:39 +0000

[diff] [blame]

1446

} else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) {

1447

VectorType *VecTy = cast<VectorType>(CI->getType());

1448

Intrinsic::ID IID;

1449

if (VecTy->getPrimitiveSizeInBits() == 128)

1450

IID = Intrinsic::x86_ssse3_pshuf_b_128;

1451

else if (VecTy->getPrimitiveSizeInBits() == 256)

1452

IID = Intrinsic::x86_avx2_pshuf_b;

Craig Topper

edab02b

2016-12-10 23:09:43 +0000

[diff] [blame^]

1453

else if (VecTy->getPrimitiveSizeInBits() == 512)

1454

IID = Intrinsic::x86_avx512_pshuf_b_512;

Craig Topper

7e54533

2016-11-07 00:13:39 +0000

[diff] [blame]

1455

else

1456

llvm_unreachable("Unexpected intrinsic");

1457

1458

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

1459

{ CI->getArgOperand(0), CI->getArgOperand(1) });

1460

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1461

CI->getArgOperand(2));

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1462

} else if (IsX86 && Name.startswith("avx512.mask.psll")) {

1463

bool IsImmediate = Name[16] == 'i' ||

1464

(Name.size() > 18 && Name[18] == 'i');

1465

bool IsVariable = Name[16] == 'v';

1466

char Size = Name[16] == '.' ? Name[17] :

1467

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1468

Name[18] == '.' ? Name[19] :

1469

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1470

1471

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1472

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1473

if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di

1474

IID = Intrinsic::x86_avx2_psllv_q;

1475

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di

1476

IID = Intrinsic::x86_avx2_psllv_q_256;

1477

else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si

1478

IID = Intrinsic::x86_avx2_psllv_d;

1479

else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si

1480

IID = Intrinsic::x86_avx2_psllv_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1481

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi

1482

IID = Intrinsic::x86_avx512_psllv_w_128;

1483

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi

1484

IID = Intrinsic::x86_avx512_psllv_w_256;

1485

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi

1486

IID = Intrinsic::x86_avx512_psllv_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1487

else

1488

llvm_unreachable("Unexpected size");

1489

} else if (Name.endswith(".128")) {

1490

if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128

1491

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d

1492

: Intrinsic::x86_sse2_psll_d;

1493

else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128

1494

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q

1495

: Intrinsic::x86_sse2_psll_q;

1496

else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128

1497

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w

1498

: Intrinsic::x86_sse2_psll_w;

1499

else

1500

llvm_unreachable("Unexpected size");

1501

} else if (Name.endswith(".256")) {

1502

if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256

1503

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d

1504

: Intrinsic::x86_avx2_psll_d;

1505

else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256

1506

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q

1507

: Intrinsic::x86_avx2_psll_q;

1508

else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256

1509

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w

1510

: Intrinsic::x86_avx2_psll_w;

1511

else

1512

llvm_unreachable("Unexpected size");

1513

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1514

if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512

1515

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :

1516

IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :

1517

Intrinsic::x86_avx512_psll_d_512;

1518

else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512

1519

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :

1520

IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :

1521

Intrinsic::x86_avx512_psll_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1522

else if (Size == 'w') // psll.wi.512, pslli.w, psll.w

1523

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512

1524

: Intrinsic::x86_avx512_psll_w_512;

1525

else

1526

llvm_unreachable("Unexpected size");

1527

}

1528

1529

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

1530

} else if (IsX86 && Name.startswith("avx512.mask.psrl")) {

1531

bool IsImmediate = Name[16] == 'i' ||

1532

(Name.size() > 18 && Name[18] == 'i');

1533

bool IsVariable = Name[16] == 'v';

1534

char Size = Name[16] == '.' ? Name[17] :

1535

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1536

Name[18] == '.' ? Name[19] :

1537

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1538

1539

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1540

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1541

if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di

1542

IID = Intrinsic::x86_avx2_psrlv_q;

1543

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di

1544

IID = Intrinsic::x86_avx2_psrlv_q_256;

1545

else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si

1546

IID = Intrinsic::x86_avx2_psrlv_d;

1547

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si

1548

IID = Intrinsic::x86_avx2_psrlv_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1549

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi

1550

IID = Intrinsic::x86_avx512_psrlv_w_128;

1551

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi

1552

IID = Intrinsic::x86_avx512_psrlv_w_256;

1553

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi

1554

IID = Intrinsic::x86_avx512_psrlv_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1555

else

1556

llvm_unreachable("Unexpected size");

1557

} else if (Name.endswith(".128")) {

1558

if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128

1559

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d

1560

: Intrinsic::x86_sse2_psrl_d;

1561

else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128

1562

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q

1563

: Intrinsic::x86_sse2_psrl_q;

1564

else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128

1565

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w

1566

: Intrinsic::x86_sse2_psrl_w;

1567

else

1568

llvm_unreachable("Unexpected size");

1569

} else if (Name.endswith(".256")) {

1570

if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256

1571

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d

1572

: Intrinsic::x86_avx2_psrl_d;

1573

else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256

1574

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q

1575

: Intrinsic::x86_avx2_psrl_q;

1576

else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256

1577

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w

1578

: Intrinsic::x86_avx2_psrl_w;

1579

else

1580

llvm_unreachable("Unexpected size");

1581

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1582

if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512

1583

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :

1584

IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :

1585

Intrinsic::x86_avx512_psrl_d_512;

1586

else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512

1587

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :

1588

IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :

1589

Intrinsic::x86_avx512_psrl_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1590

else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)

1591

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512

1592

: Intrinsic::x86_avx512_psrl_w_512;

1593

else

1594

llvm_unreachable("Unexpected size");

1595

}

1596

1597

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

1598

} else if (IsX86 && Name.startswith("avx512.mask.psra")) {

1599

bool IsImmediate = Name[16] == 'i' ||

1600

(Name.size() > 18 && Name[18] == 'i');

1601

bool IsVariable = Name[16] == 'v';

1602

char Size = Name[16] == '.' ? Name[17] :

1603

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1604

Name[18] == '.' ? Name[19] :

1605

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1606

1607

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1608

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1609

if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si

1610

IID = Intrinsic::x86_avx2_psrav_d;

1611

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si

1612

IID = Intrinsic::x86_avx2_psrav_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1613

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi

1614

IID = Intrinsic::x86_avx512_psrav_w_128;

1615

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi

1616

IID = Intrinsic::x86_avx512_psrav_w_256;

1617

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi

1618

IID = Intrinsic::x86_avx512_psrav_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1619

else

1620

llvm_unreachable("Unexpected size");

1621

} else if (Name.endswith(".128")) {

1622

if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128

1623

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d

1624

: Intrinsic::x86_sse2_psra_d;

1625

else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1626

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :

1627

IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :

1628

Intrinsic::x86_avx512_psra_q_128;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1629

else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128

1630

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w

1631

: Intrinsic::x86_sse2_psra_w;

1632

else

1633

llvm_unreachable("Unexpected size");

1634

} else if (Name.endswith(".256")) {

1635

if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256

1636

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d

1637

: Intrinsic::x86_avx2_psra_d;

1638

else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1639

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :

1640

IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :

1641

Intrinsic::x86_avx512_psra_q_256;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1642

else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256

1643

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w

1644

: Intrinsic::x86_avx2_psra_w;

1645

else

1646

llvm_unreachable("Unexpected size");

1647

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1648

if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512

1649

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :

1650

IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :

1651

Intrinsic::x86_avx512_psra_d_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1652

else if (Size == 'q') // psra.qi.512, psrai.q, psra.q

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1653

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :

1654

IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :

1655

Intrinsic::x86_avx512_psra_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1656

else if (Size == 'w') // psra.wi.512, psrai.w, psra.w

1657

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512

1658

: Intrinsic::x86_avx512_psra_w_512;

1659

else

1660

llvm_unreachable("Unexpected size");

1661

}

1662

1663

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

1664

} else if (IsX86 && Name.startswith("avx512.mask.move.s")) {

1665

Rep = upgradeMaskedMove(Builder, *CI);

Craig Topper

abe7c5b

2016-12-10 21:15:52 +0000

[diff] [blame]

1666

} else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) {

1667

Intrinsic::ID IID;

1668

if (Name.endswith("ps.128"))

1669

IID = Intrinsic::x86_avx_vpermilvar_ps;

1670

else if (Name.endswith("pd.128"))

1671

IID = Intrinsic::x86_avx_vpermilvar_pd;

1672

else if (Name.endswith("ps.256"))

1673

IID = Intrinsic::x86_avx_vpermilvar_ps_256;

1674

else if (Name.endswith("pd.256"))

1675

IID = Intrinsic::x86_avx_vpermilvar_pd_256;

1676

else

1677

llvm_unreachable("Unexpected vpermilvar intrinsic");

1678

1679

Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);

1680

Rep = Builder.CreateCall(Intrin,

1681

{ CI->getArgOperand(0), CI->getArgOperand(1) });

1682

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1683

CI->getArgOperand(2));

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1684

} else {

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1685

llvm_unreachable("Unknown function for CallInst upgrade.");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1686

}

1687

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

1688

if (Rep)

1689

CI->replaceAllUsesWith(Rep);

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1690

CI->eraseFromParent();

return;

}

Yaron Keren

2015-03-30 16:10:39 +0000

[diff] [blame]

1694

std::string Name = CI->getName();

Adrian Prantl

87b7eb9

2014-10-01 18:55:02 +0000

[diff] [blame]

1695

if (!Name.empty())

1696

CI->setName(Name + ".old");

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1697

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1698

switch (NewFn->getIntrinsicID()) {

1699

default:

Chris Lattner

0bcbde4

2011-11-27 08:42:07 +0000

[diff] [blame]

1700

llvm_unreachable("Unknown function for CallInst upgrade.");

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1701

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

1702

case Intrinsic::arm_neon_vld1:

1703

case Intrinsic::arm_neon_vld2:

1704

case Intrinsic::arm_neon_vld3:

1705

case Intrinsic::arm_neon_vld4:

1706

case Intrinsic::arm_neon_vld2lane:

1707

case Intrinsic::arm_neon_vld3lane:

1708

case Intrinsic::arm_neon_vld4lane:

1709

case Intrinsic::arm_neon_vst1:

1710

case Intrinsic::arm_neon_vst2:

1711

case Intrinsic::arm_neon_vst3:

1712

case Intrinsic::arm_neon_vst4:

1713

case Intrinsic::arm_neon_vst2lane:

1714

case Intrinsic::arm_neon_vst3lane:

1715

case Intrinsic::arm_neon_vst4lane: {

1716

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1717

CI->arg_operands().end());

1718

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args));

1719

CI->eraseFromParent();

return;

}

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1723

case Intrinsic::ctlz:

Nuno Lopes

ad40c0a

2012-05-22 15:25:31 +0000

[diff] [blame]

1724

case Intrinsic::cttz:

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1725

assert(CI->getNumArgOperands() == 1 &&

1726

"Mismatch between function args and call args");

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1727

CI->replaceAllUsesWith(Builder.CreateCall(

1728

NewFn, {CI->getArgOperand(0), Builder.getFalse()}, Name));

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1729

CI->eraseFromParent();

1730

return;

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1731

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

1732

case Intrinsic::objectsize:

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1733

CI->replaceAllUsesWith(Builder.CreateCall(

1734

NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)}, Name));

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

1735

CI->eraseFromParent();

1736

return;

1737

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

1738

case Intrinsic::ctpop: {

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1739

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {CI->getArgOperand(0)}));

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

1740

CI->eraseFromParent();

1741

return;

1742

}

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

1743

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1744

case Intrinsic::x86_xop_vfrcz_ss:

1745

case Intrinsic::x86_xop_vfrcz_sd:

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1746

CI->replaceAllUsesWith(

1747

Builder.CreateCall(NewFn, {CI->getArgOperand(1)}, Name));

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1748

CI->eraseFromParent();

1749

return;

1750

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

1751

case Intrinsic::x86_xop_vpermil2pd:

1752

case Intrinsic::x86_xop_vpermil2ps:

1753

case Intrinsic::x86_xop_vpermil2pd_256:

1754

case Intrinsic::x86_xop_vpermil2ps_256: {

1755

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1756

CI->arg_operands().end());

1757

VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());

1758

VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);

1759

Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);

1760

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args, Name));

1761

CI->eraseFromParent();

return;

}

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1765

case Intrinsic::x86_sse41_ptestc:

1766

case Intrinsic::x86_sse41_ptestz:

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1767

case Intrinsic::x86_sse41_ptestnzc: {

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1768

// The arguments for these intrinsics used to be v4f32, and changed

1769

// to v2i64. This is purely a nop, since those are bitwise intrinsics.

1770

// So, the only thing required is a bitcast for both arguments.

1771

// First, check the arguments have the old type.

1772

Value *Arg0 = CI->getArgOperand(0);

1773

if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))

1774

return;

1775

1776

// Old intrinsic, add bitcasts

1777

Value *Arg1 = CI->getArgOperand(1);

1778

David Blaikie

5bacf37

2015-04-24 21:16:07 +0000

[diff] [blame]

1779

Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1780

David Blaikie

5bacf37

2015-04-24 21:16:07 +0000

[diff] [blame]

1781

Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");

1782

Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

1783

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1784

CallInst *NewCall = Builder.CreateCall(NewFn, {BC0, BC1}, Name);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1785

CI->replaceAllUsesWith(NewCall);

1786

CI->eraseFromParent();

1787

return;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

1788

}

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1789

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1790

case Intrinsic::x86_sse41_insertps:

1791

case Intrinsic::x86_sse41_dppd:

1792

case Intrinsic::x86_sse41_dpps:

1793

case Intrinsic::x86_sse41_mpsadbw:

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1794

case Intrinsic::x86_avx_dp_ps_256:

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1795

case Intrinsic::x86_avx2_mpsadbw: {

1796

// Need to truncate the last argument from i32 to i8 -- this argument models

1797

// an inherently 8-bit immediate operand to these x86 instructions.

1798

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1799

CI->arg_operands().end());

1800

1801

// Replace the last argument with a trunc.

1802

Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");

1803

1804

CallInst *NewCall = Builder.CreateCall(NewFn, Args);

1805

CI->replaceAllUsesWith(NewCall);

1806

CI->eraseFromParent();

1807

return;

1808

}

Marcin Koscielnicki

3fdc257

2016-04-19 20:51:05 +0000

[diff] [blame]

1809

1810

case Intrinsic::thread_pointer: {

1811

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {}));

1812

CI->eraseFromParent();

1813

return;

1814

}

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

1815

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

1816

case Intrinsic::invariant_start:

1817

case Intrinsic::invariant_end:

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

1818

case Intrinsic::masked_load:

1819

case Intrinsic::masked_store: {

1820

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1821

CI->arg_operands().end());

1822

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args));

1823

CI->eraseFromParent();

1824

return;

1825

}

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1826

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1827

}

1828

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1829

void llvm::UpgradeCallsToIntrinsic(Function *F) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1830

assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");

1831

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1832

// Check if this function should be upgraded and get the replacement function

1833

// if there is one.

Chris Lattner

80ed9dc

2011-06-18 06:05:24 +0000

[diff] [blame]

1834

Function *NewFn;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

1835

if (UpgradeIntrinsicFunction(F, NewFn)) {

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1836

// Replace all users of the old function with the new function or new

1837

// instructions. This is not a range loop because the call is deleted.

1838

for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; )

Duncan P. N. Exon Smith

93f53c4

2016-04-17 03:59:37 +0000

[diff] [blame]

1839

if (CallInst *CI = dyn_cast<CallInst>(*UI++))

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

1840

UpgradeIntrinsicCall(CI, NewFn);

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1841

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

1842

// Remove old function, no longer used, from the module.

1843

F->eraseFromParent();

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1844

}

1845

}

Devang Patel

80ae349

2009-08-28 23:24:31 +0000

[diff] [blame]

1846

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1847

MDNode *llvm::UpgradeTBAANode(MDNode &MD) {

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1848

// Check if the tag uses struct-path aware TBAA format.

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1849

if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)

1850

return &MD;

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1851

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1852

auto &Context = MD.getContext();

1853

if (MD.getNumOperands() == 3) {

1854

Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};

1855

MDNode *ScalarType = MDNode::get(Context, Elts);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1856

// Create a MDNode <ScalarType, ScalarType, offset 0, const>

Duncan P. N. Exon Smith

5bf8fef

2014-12-09 18:38:53 +0000

[diff] [blame]

1857

Metadata *Elts2[] = {ScalarType, ScalarType,

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1858

ConstantAsMetadata::get(

1859

Constant::getNullValue(Type::getInt64Ty(Context))),

1860

MD.getOperand(2)};

1861

return MDNode::get(Context, Elts2);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1862

}

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1863

// Create a MDNode <MD, MD, offset 0>

1864

Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(

1865

Type::getInt64Ty(Context)))};

1866

return MDNode::get(Context, Elts);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1867

}

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1868

1869

Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,

1870

Instruction *&Temp) {

1871

if (Opc != Instruction::BitCast)

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1872

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1873

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1874

Temp = nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1875

Type *SrcTy = V->getType();

1876

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

1877

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

1878

LLVMContext &Context = V->getContext();

1879

1880

// We have no information about target data layout, so we assume that

1881

// the maximum pointer size is 64bit.

1882

Type *MidTy = Type::getInt64Ty(Context);

1883

Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);

1884

1885

return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);

1886

}

1887

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1888

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1889

}

1890

1891

Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {

1892

if (Opc != Instruction::BitCast)

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1893

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1894

1895

Type *SrcTy = C->getType();

1896

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

1897

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

1898

LLVMContext &Context = C->getContext();

1899

1900

// We have no information about target data layout, so we assume that

1901

// the maximum pointer size is 64bit.

1902

Type *MidTy = Type::getInt64Ty(Context);

1903

1904

return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),

DestTy);

}

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1908

return nullptr;

Matt Arsenault