Blame - llvm/lib/IR/AutoUpgrade.cpp - toolchain/llvm-project

2007-08-04 01:51:18 +0000

[diff] [blame]

1

//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//

2

//

3

// The LLVM Compiler Infrastructure

4

//

Chris Lattner

f3ebc3f

2007-12-29 20:36:04 +0000

[diff] [blame]

5

// This file is distributed under the University of Illinois Open Source

6

// License. See LICENSE.TXT for details.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

7

//

8

//===----------------------------------------------------------------------===//

9

//

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

10

// This file implements the auto-upgrade helper functions.

11

// This is where deprecated IR intrinsics and other IR features are updated to

12

// current specifications.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

13

//

14

//===----------------------------------------------------------------------===//

15

Chandler Carruth

9106521

2014-03-05 10:34:14 +0000

[diff] [blame]

16

#include "llvm/IR/AutoUpgrade.h"

Chandler Carruth

1305dc3

2014-03-04 11:45:46 +0000

[diff] [blame]

17

#include "llvm/IR/CFG.h"

Chandler Carruth

219b89b

2014-03-04 11:01:28 +0000

[diff] [blame]

18

#include "llvm/IR/CallSite.h"

Chandler Carruth

9fb823b

2013-01-02 11:36:10 +0000

[diff] [blame]

19

#include "llvm/IR/Constants.h"

Chandler Carruth

d990388

2015-01-14 11:23:27 +0000

[diff] [blame]

20

#include "llvm/IR/DIBuilder.h"

Chandler Carruth

9a4c9e5

2014-03-06 00:46:21 +0000

[diff] [blame]

21

#include "llvm/IR/DebugInfo.h"

Manman Ren

2ebfb42

2014-01-16 01:51:12 +0000

[diff] [blame]

22

#include "llvm/IR/DiagnosticInfo.h"

Chandler Carruth

9fb823b

2013-01-02 11:36:10 +0000

[diff] [blame]

23

#include "llvm/IR/Function.h"

24

#include "llvm/IR/IRBuilder.h"

25

#include "llvm/IR/Instruction.h"

26

#include "llvm/IR/IntrinsicInst.h"

27

#include "llvm/IR/LLVMContext.h"

28

#include "llvm/IR/Module.h"

Torok Edwin

56d0659

2009-07-11 20:10:48 +0000

[diff] [blame]

29

#include "llvm/Support/ErrorHandling.h"

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

30

#include "llvm/Support/Regex.h"

Anton Korobeynikov

579f071

2008-02-20 11:08:44 +0000

[diff] [blame]

31

#include <cstring>

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

32

using namespace llvm;

33

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

34

static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }

35

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

36

// Upgrade the declarations of the SSE4.1 functions whose arguments have

37

// changed their type from v4f32 to v2i64.

38

static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,

39

Function *&NewFn) {

40

// Check whether this is an old version of the function, which received

41

// v4f32 arguments.

42

Type *Arg0Type = F->getFunctionType()->getParamType(0);

43

if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))

44

return false;

45

46

// Yes, it's old, replace it with new version.

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

47

rename(F);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

48

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

49

return true;

50

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

51

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

52

// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask

53

// arguments have changed their type from i32 to i8.

54

static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,

55

Function *&NewFn) {

56

// Check that the last argument is an i32.

57

Type *LastArgType = F->getFunctionType()->getParamType(

58

F->getFunctionType()->getNumParams() - 1);

59

if (!LastArgType->isIntegerTy(32))

60

return false;

61

62

// Move this function aside and map down.

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

63

rename(F);

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

64

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

return true;

}

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

68

static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

69

assert(F && "Illegal to upgrade a non-existent Function.");

70

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

71

// Quickly eliminate it, if it's not a candidate.

Eric Liu

c9c6817

2016-07-08 16:09:51 +0000

[diff] [blame]

72

StringRef Name = F->getName();

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

73

if (Name.size() <= 8 || !Name.startswith("llvm."))

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

74

return false;

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

75

Name = Name.substr(5); // Strip off "llvm."

Chris Lattner

0bcbde4

2011-11-27 08:42:07 +0000

[diff] [blame]

76

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

77

switch (Name[0]) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

78

default: break;

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

79

case 'a': {

80

if (Name.startswith("arm.neon.vclz")) {

81

Type* args[2] = {

Matt Arsenault

c4c9226

2013-07-20 17:46:00 +0000

[diff] [blame]

82

F->arg_begin()->getType(),

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

83

Type::getInt1Ty(F->getContext())

84

};

85

// Can't use Intrinsic::getDeclaration here as it adds a ".i1" to

86

// the end of the name. Change name from llvm.arm.neon.vclz.* to

87

// llvm.ctlz.*

88

FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);

Matt Arsenault

c4c9226

2013-07-20 17:46:00 +0000

[diff] [blame]

89

NewFn = Function::Create(fType, F->getLinkage(),

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

90

"llvm.ctlz." + Name.substr(14), F->getParent());

91

return true;

92

}

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

93

if (Name.startswith("arm.neon.vcnt")) {

94

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,

95

F->arg_begin()->getType());

96

return true;

97

}

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

98

Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");

99

if (vldRegex.match(Name)) {

100

auto fArgs = F->getFunctionType()->params();

101

SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());

102

// Can't use Intrinsic::getDeclaration here as the return types might

103

// then only be structurally equal.

104

FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);

105

NewFn = Function::Create(fType, F->getLinkage(),

106

"llvm." + Name + ".p0i8", F->getParent());

107

return true;

108

}

109

Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");

110

if (vstRegex.match(Name)) {

Craig Topper

2626094

2015-10-18 05:15:34 +0000

[diff] [blame]

111

static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,

112

Intrinsic::arm_neon_vst2,

113

Intrinsic::arm_neon_vst3,

114

Intrinsic::arm_neon_vst4};

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

115

Craig Topper

2626094

2015-10-18 05:15:34 +0000

[diff] [blame]

116

static const Intrinsic::ID StoreLaneInts[] = {

117

Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,

118

Intrinsic::arm_neon_vst4lane

119

};

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

120

121

auto fArgs = F->getFunctionType()->params();

122

Type *Tys[] = {fArgs[0], fArgs[1]};

123

if (Name.find("lane") == StringRef::npos)

124

NewFn = Intrinsic::getDeclaration(F->getParent(),

125

StoreInts[fArgs.size() - 3], Tys);

126

else

127

NewFn = Intrinsic::getDeclaration(F->getParent(),

128

StoreLaneInts[fArgs.size() - 5], Tys);

129

return true;

130

}

Marcin Koscielnicki

3fdc257

2016-04-19 20:51:05 +0000

[diff] [blame]

131

if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {

132

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);

133

return true;

134

}

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

135

break;

136

}

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

137

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

138

case 'c': {

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

139

if (Name.startswith("ctlz.") && F->arg_size() == 1) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

140

rename(F);

Chandler Carruth

d4a0240

2011-12-12 10:57:20 +0000

[diff] [blame]

141

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,

142

F->arg_begin()->getType());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

143

return true;

144

}

145

if (Name.startswith("cttz.") && F->arg_size() == 1) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

146

rename(F);

Chandler Carruth

d4a0240

2011-12-12 10:57:20 +0000

[diff] [blame]

147

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,

148

F->arg_begin()->getType());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

return true;

}

break;

}

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

153

case 'i': {

154

if (Name.startswith("invariant.start")) {

155

auto Args = F->getFunctionType()->params();

156

Type* ObjectPtr[1] = {Args[1]};

157

if (F->getName() !=

158

Intrinsic::getName(Intrinsic::invariant_start, ObjectPtr)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

159

rename(F);

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

160

NewFn = Intrinsic::getDeclaration(

161

F->getParent(), Intrinsic::invariant_start, ObjectPtr);

return true;

}

}

if (Name.startswith("invariant.end")) {

166

auto Args = F->getFunctionType()->params();

167

Type* ObjectPtr[1] = {Args[2]};

168

if (F->getName() !=

169

Intrinsic::getName(Intrinsic::invariant_end, ObjectPtr)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

170

rename(F);

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

171

NewFn = Intrinsic::getDeclaration(F->getParent(),

172

Intrinsic::invariant_end, ObjectPtr);

return true;

}

}

break;

}

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

178

case 'm': {

179

if (Name.startswith("masked.load.")) {

180

Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };

181

if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

182

rename(F);

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

183

NewFn = Intrinsic::getDeclaration(F->getParent(),

184

Intrinsic::masked_load,

Tys);

return true;

}

}

if (Name.startswith("masked.store.")) {

190

auto Args = F->getFunctionType()->params();

191

Type *Tys[] = { Args[0], Args[1] };

192

if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

193

rename(F);

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

194

NewFn = Intrinsic::getDeclaration(F->getParent(),

195

Intrinsic::masked_store,

Tys);

return true;

}

}

break;

}

Matt Arsenault

2013-10-07 18:06:48 +0000

[diff] [blame]

203

case 'o':

204

// We only need to change the name to match the mangling including the

205

// address space.

206

if (F->arg_size() == 2 && Name.startswith("objectsize.")) {

207

Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };

208

if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

209

rename(F);

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

210

NewFn = Intrinsic::getDeclaration(F->getParent(),

211

Intrinsic::objectsize, Tys);

return true;

}

}

break;

Tim Shen

2016-04-08 21:26:31 +0000

[diff] [blame]

217

case 's':

218

if (Name == "stackprotectorcheck") {

219

NewFn = nullptr;

220

return true;

221

}

George Burgess IV

63b06e1

2016-11-08 04:01:50 +0000

[diff] [blame]

222

break;

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

223

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

224

case 'x': {

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

225

bool IsX86 = Name.startswith("x86.");

226

if (IsX86)

227

Name = Name.substr(4);

228

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

229

// All of the intrinsics matches below should be marked with which llvm

230

// version started autoupgrading them. At some point in the future we would

231

// like to use this information to remove upgrade code for some older

232

// intrinsics. It is currently undecided how we will determine that future

233

// point.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

234

if (IsX86 &&

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

235

(Name.startswith("sse2.pcmpeq.") || // Added in 3.1

236

Name.startswith("sse2.pcmpgt.") || // Added in 3.1

237

Name.startswith("avx2.pcmpeq.") || // Added in 3.1

238

Name.startswith("avx2.pcmpgt.") || // Added in 3.1

239

Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9

240

Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9

Craig Topper

6910fa0

2016-11-16 05:24:10 +0000

[diff] [blame]

241

Name == "sse.add.ss" || // Added in 4.0

242

Name == "sse2.add.sd" || // Added in 4.0

243

Name == "sse.sub.ss" || // Added in 4.0

244

Name == "sse2.sub.sd" || // Added in 4.0

245

Name == "sse.mul.ss" || // Added in 4.0

246

Name == "sse2.mul.sd" || // Added in 4.0

247

Name == "sse.div.ss" || // Added in 4.0

248

Name == "sse2.div.sd" || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

249

Name == "sse41.pmaxsb" || // Added in 3.9

250

Name == "sse2.pmaxs.w" || // Added in 3.9

251

Name == "sse41.pmaxsd" || // Added in 3.9

252

Name == "sse2.pmaxu.b" || // Added in 3.9

253

Name == "sse41.pmaxuw" || // Added in 3.9

254

Name == "sse41.pmaxud" || // Added in 3.9

255

Name == "sse41.pminsb" || // Added in 3.9

256

Name == "sse2.pmins.w" || // Added in 3.9

257

Name == "sse41.pminsd" || // Added in 3.9

258

Name == "sse2.pminu.b" || // Added in 3.9

259

Name == "sse41.pminuw" || // Added in 3.9

260

Name == "sse41.pminud" || // Added in 3.9

261

Name == "avx512.mask.pshuf.b.128" || // Added in 4.0

262

Name == "avx512.mask.pshuf.b.256" || // Added in 4.0

263

Name.startswith("avx2.pmax") || // Added in 3.9

264

Name.startswith("avx2.pmin") || // Added in 3.9

265

Name.startswith("avx512.mask.pmax") || // Added in 4.0

266

Name.startswith("avx512.mask.pmin") || // Added in 4.0

267

Name.startswith("avx2.vbroadcast") || // Added in 3.8

268

Name.startswith("avx2.pbroadcast") || // Added in 3.8

269

Name.startswith("avx.vpermil.") || // Added in 3.1

270

Name.startswith("sse2.pshuf") || // Added in 3.9

271

Name.startswith("avx512.pbroadcast") || // Added in 3.9

272

Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9

273

Name.startswith("avx512.mask.movddup") || // Added in 3.9

274

Name.startswith("avx512.mask.movshdup") || // Added in 3.9

275

Name.startswith("avx512.mask.movsldup") || // Added in 3.9

276

Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9

277

Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9

278

Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9

279

Name.startswith("avx512.mask.shuf.p") || // Added in 4.0

280

Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9

281

Name.startswith("avx512.mask.perm.df.") || // Added in 3.9

282

Name.startswith("avx512.mask.perm.di.") || // Added in 3.9

283

Name.startswith("avx512.mask.punpckl") || // Added in 3.9

284

Name.startswith("avx512.mask.punpckh") || // Added in 3.9

285

Name.startswith("avx512.mask.unpckl.") || // Added in 3.9

286

Name.startswith("avx512.mask.unpckh.") || // Added in 3.9

287

Name.startswith("avx512.mask.pand.") || // Added in 3.9

288

Name.startswith("avx512.mask.pandn.") || // Added in 3.9

289

Name.startswith("avx512.mask.por.") || // Added in 3.9

290

Name.startswith("avx512.mask.pxor.") || // Added in 3.9

291

Name.startswith("avx512.mask.and.") || // Added in 3.9

292

Name.startswith("avx512.mask.andn.") || // Added in 3.9

293

Name.startswith("avx512.mask.or.") || // Added in 3.9

294

Name.startswith("avx512.mask.xor.") || // Added in 3.9

295

Name.startswith("avx512.mask.padd.") || // Added in 4.0

296

Name.startswith("avx512.mask.psub.") || // Added in 4.0

297

Name.startswith("avx512.mask.pmull.") || // Added in 4.0

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

298

Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0

299

Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

300

Name == "avx512.mask.add.pd.128" || // Added in 4.0

301

Name == "avx512.mask.add.pd.256" || // Added in 4.0

302

Name == "avx512.mask.add.ps.128" || // Added in 4.0

303

Name == "avx512.mask.add.ps.256" || // Added in 4.0

304

Name == "avx512.mask.div.pd.128" || // Added in 4.0

305

Name == "avx512.mask.div.pd.256" || // Added in 4.0

306

Name == "avx512.mask.div.ps.128" || // Added in 4.0

307

Name == "avx512.mask.div.ps.256" || // Added in 4.0

308

Name == "avx512.mask.mul.pd.128" || // Added in 4.0

309

Name == "avx512.mask.mul.pd.256" || // Added in 4.0

310

Name == "avx512.mask.mul.ps.128" || // Added in 4.0

311

Name == "avx512.mask.mul.ps.256" || // Added in 4.0

312

Name == "avx512.mask.sub.pd.128" || // Added in 4.0

313

Name == "avx512.mask.sub.pd.256" || // Added in 4.0

314

Name == "avx512.mask.sub.ps.128" || // Added in 4.0

315

Name == "avx512.mask.sub.ps.256" || // Added in 4.0

Craig Topper

abe7c5b

2016-12-10 21:15:52 +0000

[diff] [blame]

316

Name == "avx512.mask.vpermilvar.ps.128" || // Added in 4.0

317

Name == "avx512.mask.vpermilvar.ps.256" || // Added in 4.0

318

Name == "avx512.mask.vpermilvar.pd.128" || // Added in 4.0

319

Name == "avx512.mask.vpermilvar.pd.256" || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

320

Name.startswith("avx512.mask.psll.d") || // Added in 4.0

321

Name.startswith("avx512.mask.psll.q") || // Added in 4.0

322

Name.startswith("avx512.mask.psll.w") || // Added in 4.0

323

Name.startswith("avx512.mask.psra.d") || // Added in 4.0

324

Name.startswith("avx512.mask.psra.q") || // Added in 4.0

325

Name.startswith("avx512.mask.psra.w") || // Added in 4.0

326

Name.startswith("avx512.mask.psrl.d") || // Added in 4.0

327

Name.startswith("avx512.mask.psrl.q") || // Added in 4.0

328

Name.startswith("avx512.mask.psrl.w") || // Added in 4.0

329

Name.startswith("avx512.mask.pslli") || // Added in 4.0

330

Name.startswith("avx512.mask.psrai") || // Added in 4.0

331

Name.startswith("avx512.mask.psrli") || // Added in 4.0

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

332

Name.startswith("avx512.mask.psllv") || // Added in 4.0

333

Name.startswith("avx512.mask.psrav") || // Added in 4.0

334

Name.startswith("avx512.mask.psrlv") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

335

Name.startswith("sse41.pmovsx") || // Added in 3.8

336

Name.startswith("sse41.pmovzx") || // Added in 3.9

337

Name.startswith("avx2.pmovsx") || // Added in 3.9

338

Name.startswith("avx2.pmovzx") || // Added in 3.9

339

Name.startswith("avx512.mask.pmovsx") || // Added in 4.0

340

Name.startswith("avx512.mask.pmovzx") || // Added in 4.0

341

Name == "sse2.cvtdq2pd" || // Added in 3.9

342

Name == "sse2.cvtps2pd" || // Added in 3.9

343

Name == "avx.cvtdq2.pd.256" || // Added in 3.9

344

Name == "avx.cvt.ps2.pd.256" || // Added in 3.9

345

Name.startswith("avx.vinsertf128.") || // Added in 3.7

346

Name == "avx2.vinserti128" || // Added in 3.7

347

Name.startswith("avx.vextractf128.") || // Added in 3.7

348

Name == "avx2.vextracti128" || // Added in 3.7

349

Name.startswith("sse4a.movnt.") || // Added in 3.9

350

Name.startswith("avx.movnt.") || // Added in 3.2

351

Name.startswith("avx512.storent.") || // Added in 3.9

352

Name == "sse2.storel.dq" || // Added in 3.9

353

Name.startswith("sse.storeu.") || // Added in 3.9

354

Name.startswith("sse2.storeu.") || // Added in 3.9

355

Name.startswith("avx.storeu.") || // Added in 3.9

356

Name.startswith("avx512.mask.storeu.") || // Added in 3.9

357

Name.startswith("avx512.mask.store.p") || // Added in 3.9

358

Name.startswith("avx512.mask.store.b.") || // Added in 3.9

359

Name.startswith("avx512.mask.store.w.") || // Added in 3.9

360

Name.startswith("avx512.mask.store.d.") || // Added in 3.9

361

Name.startswith("avx512.mask.store.q.") || // Added in 3.9

362

Name.startswith("avx512.mask.loadu.") || // Added in 3.9

363

Name.startswith("avx512.mask.load.") || // Added in 3.9

364

Name == "sse42.crc32.64.8" || // Added in 3.4

365

Name.startswith("avx.vbroadcast.s") || // Added in 3.5

366

Name.startswith("avx512.mask.palignr.") || // Added in 3.9

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

367

Name.startswith("avx512.mask.valign.") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

368

Name.startswith("sse2.psll.dq") || // Added in 3.7

369

Name.startswith("sse2.psrl.dq") || // Added in 3.7

370

Name.startswith("avx2.psll.dq") || // Added in 3.7

371

Name.startswith("avx2.psrl.dq") || // Added in 3.7

372

Name.startswith("avx512.psll.dq") || // Added in 3.9

373

Name.startswith("avx512.psrl.dq") || // Added in 3.9

374

Name == "sse41.pblendw" || // Added in 3.7

375

Name.startswith("sse41.blendp") || // Added in 3.7

376

Name.startswith("avx.blend.p") || // Added in 3.7

377

Name == "avx2.pblendw" || // Added in 3.7

378

Name.startswith("avx2.pblendd.") || // Added in 3.7

379

Name.startswith("avx.vbroadcastf128") || // Added in 4.0

380

Name == "avx2.vbroadcasti128" || // Added in 3.7

381

Name == "xop.vpcmov" || // Added in 3.8

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

382

Name.startswith("avx512.mask.move.s") || // Added in 4.0

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

383

(Name.startswith("xop.vpcom") && // Added in 3.2

384

F->arg_size() == 2))) {

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

385

NewFn = nullptr;

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

386

return true;

387

}

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

388

// SSE4.1 ptest functions may have an old signature.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

389

if (IsX86 && Name.startswith("sse41.ptest")) { // Added in 3.2

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

390

if (Name.substr(11) == "c")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

391

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

392

if (Name.substr(11) == "z")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

393

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

394

if (Name.substr(11) == "nzc")

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

395

return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);

396

}

Sanjay Patel

1c3eaec

2015-02-28 22:25:06 +0000

[diff] [blame]

397

// Several blend and other instructions with masks used the wrong number of

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

398

// bits.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

399

if (IsX86 && Name == "sse41.insertps") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

400

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,

401

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

402

if (IsX86 && Name == "sse41.dppd") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

403

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,

404

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

405

if (IsX86 && Name == "sse41.dpps") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

406

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,

407

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

408

if (IsX86 && Name == "sse41.mpsadbw") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

409

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,

410

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

411

if (IsX86 && Name == "avx.dp.ps.256") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

412

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,

413

NewFn);

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

414

if (IsX86 && Name == "avx2.mpsadbw") // Added in 3.6

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

415

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,

416

NewFn);

Craig Topper

29f2e95

2015-01-25 23:26:02 +0000

[diff] [blame]

417

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

418

// frcz.ss/sd may need to have an argument dropped. Added in 3.2

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

419

if (IsX86 && Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

420

rename(F);

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

421

NewFn = Intrinsic::getDeclaration(F->getParent(),

422

Intrinsic::x86_xop_vfrcz_ss);

423

return true;

424

}

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

425

if (IsX86 && Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

426

rename(F);

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

427

NewFn = Intrinsic::getDeclaration(F->getParent(),

428

Intrinsic::x86_xop_vfrcz_sd);

429

return true;

430

}

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

431

// Upgrade any XOP PERMIL2 index operand still using a float/double vector.

Craig Topper

2016-11-15 05:04:51 +0000

[diff] [blame]

432

if (IsX86 && Name.startswith("xop.vpermil2")) { // Added in 3.9

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

433

auto Params = F->getFunctionType()->params();

434

auto Idx = Params[2];

435

if (Idx->getScalarType()->isFloatingPointTy()) {

Rafael Espindola

2016-10-03 15:51:42 +0000

[diff] [blame]

436

rename(F);

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

437

unsigned IdxSize = Idx->getPrimitiveSizeInBits();

438

unsigned EltSize = Idx->getScalarSizeInBits();

439

Intrinsic::ID Permil2ID;

440

if (EltSize == 64 && IdxSize == 128)

441

Permil2ID = Intrinsic::x86_xop_vpermil2pd;

442

else if (EltSize == 32 && IdxSize == 128)

443

Permil2ID = Intrinsic::x86_xop_vpermil2ps;

444

else if (EltSize == 64 && IdxSize == 256)

445

Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;

446

else

447

Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;

448

NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);

449

return true;

450

}

451

}

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

452

break;

453

}

Chris Lattner

2011-06-18 18:56:39 +0000

[diff] [blame]

454

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

455

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

456

// This may not belong here. This function is effectively being overloaded

457

// to both detect an intrinsic which needs upgrading, and to provide the

458

// upgraded form of the intrinsic. We should perhaps have two separate

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

459

// functions for this.

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

460

return false;

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

461

}

462

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

463

bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

464

NewFn = nullptr;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

465

bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

466

assert(F != NewFn && "Intrinsic function upgraded to the same function");

Duncan Sands

38ef3a8

2007-12-03 20:06:50 +0000

[diff] [blame]

467

468

// Upgrade intrinsic attributes. This does not change the function.

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

469

if (NewFn)

470

F = NewFn;

Pete Cooper

9e1d335

2015-05-20 17:16:39 +0000

[diff] [blame]

471

if (Intrinsic::ID id = F->getIntrinsicID())

472

F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));

Duncan Sands

38ef3a8

2007-12-03 20:06:50 +0000

[diff] [blame]

return Upgraded;

}

Bill Wendling

2010-09-10 18:51:56 +0000

[diff] [blame]

476

bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {

Chris Lattner

80ed9dc

2011-06-18 06:05:24 +0000

[diff] [blame]

477

// Nothing to do yet.

Bill Wendling

e26fffc

2010-09-10 18:51:56 +0000

[diff] [blame]

return false;

}

Simon Pilgrim

2016-06-09 21:09:03 +0000

[diff] [blame]

481

// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

482

// to byte shuffles.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

483

static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

484

Value *Op, unsigned Shift) {

485

Type *ResultTy = Op->getType();

486

unsigned NumElts = ResultTy->getVectorNumElements() * 8;

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

487

488

// Bitcast from a 64-bit element type to a byte element type.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

489

Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

490

Op = Builder.CreateBitCast(Op, VecTy, "cast");

491

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

492

// We'll be shuffling in zeroes.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

493

Value *Res = Constant::getNullValue(VecTy);

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

494

495

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

496

// we'll just return the zero vector.

497

if (Shift < 16) {

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

498

uint32_t Idxs[64];

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

499

// 256/512-bit version is split into 2/4 16-byte lanes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

500

for (unsigned l = 0; l != NumElts; l += 16)

501

for (unsigned i = 0; i != 16; ++i) {

502

unsigned Idx = NumElts + i - Shift;

503

if (Idx < NumElts)

504

Idx -= NumElts - 16; // end of lane, switch operand.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

505

Idxs[l + i] = Idx + l;

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

506

}

507

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

508

Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

509

}

510

511

// Bitcast back to a 64-bit element type.

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

512

return Builder.CreateBitCast(Res, ResultTy, "cast");

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

513

}

514

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

515

// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them

516

// to byte shuffles.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

517

static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

518

unsigned Shift) {

519

Type *ResultTy = Op->getType();

520

unsigned NumElts = ResultTy->getVectorNumElements() * 8;

521

522

// Bitcast from a 64-bit element type to a byte element type.

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

523

Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts);

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

524

Op = Builder.CreateBitCast(Op, VecTy, "cast");

525

526

// We'll be shuffling in zeroes.

527

Value *Res = Constant::getNullValue(VecTy);

528

529

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

530

// we'll just return the zero vector.

531

if (Shift < 16) {

532

uint32_t Idxs[64];

533

// 256/512-bit version is split into 2/4 16-byte lanes.

534

for (unsigned l = 0; l != NumElts; l += 16)

535

for (unsigned i = 0; i != 16; ++i) {

536

unsigned Idx = i + Shift;

537

if (Idx >= 16)

538

Idx += NumElts - 16; // end of lane, switch operand.

539

Idxs[l + i] = Idx + l;

540

}

541

542

Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));

543

}

544

545

// Bitcast back to a 64-bit element type.

546

return Builder.CreateBitCast(Res, ResultTy, "cast");

547

}

548

549

static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,

550

unsigned NumElts) {

551

llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(),

552

cast<IntegerType>(Mask->getType())->getBitWidth());

553

Mask = Builder.CreateBitCast(Mask, MaskTy);

554

555

// If we have less than 8 elements, then the starting mask was an i8 and

556

// we need to extract down to the right number of elements.

557

if (NumElts < 8) {

558

uint32_t Indices[4];

559

for (unsigned i = 0; i != NumElts; ++i)

560

Indices[i] = i;

561

Mask = Builder.CreateShuffleVector(Mask, Mask,

562

makeArrayRef(Indices, NumElts),

"extract");

}

return Mask;

}

static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,

570

Value *Op0, Value *Op1) {

571

// If the mask is all ones just emit the align operation.

572

if (const auto *C = dyn_cast<Constant>(Mask))

573

if (C->isAllOnesValue())

574

return Op0;

575

576

Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements());

577

return Builder.CreateSelect(Mask, Op0, Op1);

578

}

579

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

580

// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.

581

// PALIGNR handles large immediates by shifting while VALIGN masks the immediate

582

// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.

583

static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,

584

Value *Op1, Value *Shift,

585

Value *Passthru, Value *Mask,

586

bool IsVALIGN) {

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

587

unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();

588

589

unsigned NumElts = Op0->getType()->getVectorNumElements();

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

590

assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");

591

assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");

592

assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");

593

594

// Mask the immediate for VALIGN.

595

if (IsVALIGN)

596

ShiftVal &= (NumElts - 1);

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

597

598

// If palignr is shifting the pair of vectors more than the size of two

599

// lanes, emit zero.

600

if (ShiftVal >= 32)

601

return llvm::Constant::getNullValue(Op0->getType());

602

603

// If palignr is shifting the pair of input vectors more than one lane,

604

// but less than two lanes, convert to shifting in zeroes.

if (ShiftVal > 16) {

ShiftVal -= 16;

Op1 = Op0;

Op0 = llvm::Constant::getNullValue(Op0->getType());

609

}

610

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

611

uint32_t Indices[64];

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

612

// 256-bit palignr operates on 128-bit lanes so we need to handle that

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

613

for (unsigned l = 0; l < NumElts; l += 16) {

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

614

for (unsigned i = 0; i != 16; ++i) {

615

unsigned Idx = ShiftVal + i;

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

616

if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.

Craig Topper

2016-06-06 06:12:54 +0000

[diff] [blame]

617

Idx += NumElts - 16; // End of lane, switch operand.

618

Indices[l + i] = Idx + l;

}

}

Value *Align = Builder.CreateShuffleVector(Op1, Op0,

623

makeArrayRef(Indices, NumElts),

624

"palignr");

625

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

626

return EmitX86Select(Builder, Mask, Align, Passthru);

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

627

}

628

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

629

static Value *UpgradeMaskedStore(IRBuilder<> &Builder,

Craig Topper

50f85c2

2016-05-31 01:50:02 +0000

[diff] [blame]

630

Value *Ptr, Value *Data, Value *Mask,

631

bool Aligned) {

632

// Cast the pointer to the right type.

633

Ptr = Builder.CreateBitCast(Ptr,

634

llvm::PointerType::getUnqual(Data->getType()));

635

unsigned Align =

636

Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1;

637

638

// If the mask is all ones just emit a regular store.

639

if (const auto *C = dyn_cast<Constant>(Mask))

640

if (C->isAllOnesValue())

641

return Builder.CreateAlignedStore(Data, Ptr, Align);

642

643

// Convert the mask from an integer type to a vector of i1.

644

unsigned NumElts = Data->getType()->getVectorNumElements();

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

645

Mask = getX86MaskVec(Builder, Mask, NumElts);

Craig Topper

50f85c2

2016-05-31 01:50:02 +0000

[diff] [blame]

646

return Builder.CreateMaskedStore(Data, Ptr, Align, Mask);

647

}

648

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

649

static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

650

Value *Ptr, Value *Passthru, Value *Mask,

651

bool Aligned) {

652

// Cast the pointer to the right type.

653

Ptr = Builder.CreateBitCast(Ptr,

654

llvm::PointerType::getUnqual(Passthru->getType()));

655

unsigned Align =

656

Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;

657

658

// If the mask is all ones just emit a regular store.

659

if (const auto *C = dyn_cast<Constant>(Mask))

660

if (C->isAllOnesValue())

661

return Builder.CreateAlignedLoad(Ptr, Align);

662

663

// Convert the mask from an integer type to a vector of i1.

664

unsigned NumElts = Passthru->getType()->getVectorNumElements();

Craig Topper

2016-06-13 02:36:42 +0000

[diff] [blame]

665

Mask = getX86MaskVec(Builder, Mask, NumElts);

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

666

return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru);

667

}

668

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

669

static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,

670

ICmpInst::Predicate Pred) {

671

Value *Op0 = CI.getArgOperand(0);

672

Value *Op1 = CI.getArgOperand(1);

673

Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1);

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

674

Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);

675

676

if (CI.getNumArgOperands() == 4)

677

Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));

678

679

return Res;

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

680

}

681

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

682

static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,

683

ICmpInst::Predicate Pred) {

684

Value *Op0 = CI.getArgOperand(0);

685

unsigned NumElts = Op0->getType()->getVectorNumElements();

686

Value *Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));

687

688

Value *Mask = CI.getArgOperand(2);

689

const auto *C = dyn_cast<Constant>(Mask);

690

if (!C || !C->isAllOnesValue())

691

Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts));

if (NumElts < 8) {

uint32_t Indices[8];

for (unsigned i = 0; i != NumElts; ++i)

696

Indices[i] = i;

697

for (unsigned i = NumElts; i != 8; ++i)

Craig Topper

d5d2a35

2016-07-07 06:11:07 +0000

[diff] [blame]

698

Indices[i] = NumElts + i % NumElts;

699

Cmp = Builder.CreateShuffleVector(Cmp,

700

Constant::getNullValue(Cmp->getType()),

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

701

Indices);

702

}

703

return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(),

704

std::max(NumElts, 8U)));

705

}

706

Craig Topper

c9467ed

2016-11-06 16:29:08 +0000

[diff] [blame]

707

// Replace a masked intrinsic with an older unmasked intrinsic.

708

static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,

709

Intrinsic::ID IID) {

710

Function *F = CI.getCalledFunction();

711

Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);

712

Value *Rep = Builder.CreateCall(Intrin,

713

{ CI.getArgOperand(0), CI.getArgOperand(1) });

714

return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));

715

}

716

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

717

static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {

718

Value* A = CI.getArgOperand(0);

719

Value* B = CI.getArgOperand(1);

720

Value* Src = CI.getArgOperand(2);

721

Value* Mask = CI.getArgOperand(3);

722

723

Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));

724

Value* Cmp = Builder.CreateIsNotNull(AndNode);

725

Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);

726

Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);

727

Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);

728

return Builder.CreateInsertElement(A, Select, (uint64_t)0);

729

}

Craig Topper

c9467ed

2016-11-06 16:29:08 +0000

[diff] [blame]

730

Sanjay Patel

595098f

2016-06-15 22:01:28 +0000

[diff] [blame]

731

/// Upgrade a call to an old intrinsic. All argument and return casting must be

732

/// provided to seamlessly integrate with existing context.

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

733

void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

734

Function *F = CI->getCalledFunction();

Nick Lewycky

2eb3ade

2011-12-12 22:59:34 +0000

[diff] [blame]

735

LLVMContext &C = CI->getContext();

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

736

IRBuilder<> Builder(C);

Duncan P. N. Exon Smith

52888a6

2015-10-08 23:49:46 +0000

[diff] [blame]

737

Builder.SetInsertPoint(CI->getParent(), CI->getIterator());

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

738

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

739

assert(F && "Intrinsic call is not direct?");

740

741

if (!NewFn) {

742

// Get the Function's name.

743

StringRef Name = F->getName();

744

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

745

assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");

746

Name = Name.substr(5);

747

748

bool IsX86 = Name.startswith("x86.");

749

if (IsX86)

750

Name = Name.substr(4);

751

Craig Topper

a4744d1

2016-12-10 21:15:48 +0000

[diff] [blame]

752

if (IsX86 && Name.startswith("sse4a.movnt.")) {

753

Module *M = F->getParent();

754

SmallVector<Metadata *, 1> Elts;

755

Elts.push_back(

756

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

757

MDNode *Node = MDNode::get(C, Elts);

758

759

Value *Arg0 = CI->getArgOperand(0);

760

Value *Arg1 = CI->getArgOperand(1);

761

762

// Nontemporal (unaligned) store of the 0'th element of the float/double

763

// vector.

764

Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();

765

PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);

766

Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");

767

Value *Extract =

768

Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");

769

770

StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1);

771

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

772

773

// Remove intrinsic.

774

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx.movnt.") ||

779

Name.startswith("avx512.storent."))) {

780

Module *M = F->getParent();

781

SmallVector<Metadata *, 1> Elts;

782

Elts.push_back(

783

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

784

MDNode *Node = MDNode::get(C, Elts);

785

786

Value *Arg0 = CI->getArgOperand(0);

787

Value *Arg1 = CI->getArgOperand(1);

788

789

// Convert the type of the pointer to a pointer to the stored type.

790

Value *BC = Builder.CreateBitCast(Arg0,

791

PointerType::getUnqual(Arg1->getType()),

792

"cast");

793

VectorType *VTy = cast<VectorType>(Arg1->getType());

794

StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC,

795

VTy->getBitWidth() / 8);

796

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

797

798

// Remove intrinsic.

799

CI->eraseFromParent();

return;

}

if (IsX86 && Name == "sse2.storel.dq") {

804

Value *Arg0 = CI->getArgOperand(0);

805

Value *Arg1 = CI->getArgOperand(1);

806

807

Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);

808

Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

809

Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);

810

Value *BC = Builder.CreateBitCast(Arg0,

811

PointerType::getUnqual(Elt->getType()),

812

"cast");

813

Builder.CreateAlignedStore(Elt, BC, 1);

814

815

// Remove intrinsic.

816

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("sse.storeu.") ||

821

Name.startswith("sse2.storeu.") ||

822

Name.startswith("avx.storeu."))) {

823

Value *Arg0 = CI->getArgOperand(0);

824

Value *Arg1 = CI->getArgOperand(1);

825

826

Arg0 = Builder.CreateBitCast(Arg0,

827

PointerType::getUnqual(Arg1->getType()),

828

"cast");

829

Builder.CreateAlignedStore(Arg1, Arg0, 1);

830

831

// Remove intrinsic.

832

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx512.mask.storeu."))) {

837

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

838

CI->getArgOperand(2), /*Aligned*/false);

839

840

// Remove intrinsic.

841

CI->eraseFromParent();

return;

}

if (IsX86 && (Name.startswith("avx512.mask.store."))) {

846

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

847

CI->getArgOperand(2), /*Aligned*/true);

848

849

// Remove intrinsic.

850

CI->eraseFromParent();

return;

}

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

854

Value *Rep;

Sanjay Patel

595098f

2016-06-15 22:01:28 +0000

[diff] [blame]

855

// Upgrade packed integer vector compare intrinsics to compare instructions.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

856

if (IsX86 && (Name.startswith("sse2.pcmpeq.") ||

857

Name.startswith("avx2.pcmpeq."))) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

858

Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),

859

"pcmpeq");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

860

Rep = Builder.CreateSExt(Rep, CI->getType(), "");

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

861

} else if (IsX86 && (Name.startswith("sse2.pcmpgt.") ||

862

Name.startswith("avx2.pcmpgt."))) {

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

863

Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),

864

"pcmpgt");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

865

Rep = Builder.CreateSExt(Rep, CI->getType(), "");

Craig Topper

6910fa0

2016-11-16 05:24:10 +0000

[diff] [blame]

866

} else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {

867

Type *I32Ty = Type::getInt32Ty(C);

868

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

869

ConstantInt::get(I32Ty, 0));

870

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

871

ConstantInt::get(I32Ty, 0));

872

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

873

Builder.CreateFAdd(Elt0, Elt1),

874

ConstantInt::get(I32Ty, 0));

875

} else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) {

876

Type *I32Ty = Type::getInt32Ty(C);

877

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

878

ConstantInt::get(I32Ty, 0));

879

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

880

ConstantInt::get(I32Ty, 0));

881

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

882

Builder.CreateFSub(Elt0, Elt1),

883

ConstantInt::get(I32Ty, 0));

884

} else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) {

885

Type *I32Ty = Type::getInt32Ty(C);

886

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

887

ConstantInt::get(I32Ty, 0));

888

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

889

ConstantInt::get(I32Ty, 0));

890

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

891

Builder.CreateFMul(Elt0, Elt1),

892

ConstantInt::get(I32Ty, 0));

893

} else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) {

894

Type *I32Ty = Type::getInt32Ty(C);

895

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

896

ConstantInt::get(I32Ty, 0));

897

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

898

ConstantInt::get(I32Ty, 0));

899

Rep = Builder.CreateInsertElement(CI->getArgOperand(0),

900

Builder.CreateFDiv(Elt0, Elt1),

901

ConstantInt::get(I32Ty, 0));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

902

} else if (IsX86 && Name.startswith("avx512.mask.pcmpeq.")) {

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

903

Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_EQ);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

904

} else if (IsX86 && Name.startswith("avx512.mask.pcmpgt.")) {

Craig Topper

2016-06-21 03:53:24 +0000

[diff] [blame]

905

Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_SGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

906

} else if (IsX86 && (Name == "sse41.pmaxsb" ||

907

Name == "sse2.pmaxs.w" ||

908

Name == "sse41.pmaxsd" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

909

Name.startswith("avx2.pmaxs") ||

910

Name.startswith("avx512.mask.pmaxs"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

911

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

912

} else if (IsX86 && (Name == "sse2.pmaxu.b" ||

913

Name == "sse41.pmaxuw" ||

914

Name == "sse41.pmaxud" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

915

Name.startswith("avx2.pmaxu") ||

916

Name.startswith("avx512.mask.pmaxu"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

917

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

918

} else if (IsX86 && (Name == "sse41.pminsb" ||

919

Name == "sse2.pmins.w" ||

920

Name == "sse41.pminsd" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

921

Name.startswith("avx2.pmins") ||

922

Name.startswith("avx512.mask.pmins"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

923

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

924

} else if (IsX86 && (Name == "sse2.pminu.b" ||

925

Name == "sse41.pminuw" ||

926

Name == "sse41.pminud" ||

Craig Topper

2016-10-24 04:04:16 +0000

[diff] [blame]

927

Name.startswith("avx2.pminu") ||

928

Name.startswith("avx512.mask.pminu"))) {

Sanjay Patel

2016-06-16 15:48:30 +0000

[diff] [blame]

929

Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

930

} else if (IsX86 && (Name == "sse2.cvtdq2pd" ||

931

Name == "sse2.cvtps2pd" ||

932

Name == "avx.cvtdq2.pd.256" ||

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

933

Name == "avx.cvt.ps2.pd.256" ||

934

Name.startswith("avx512.mask.cvtdq2pd.") ||

935

Name.startswith("avx512.mask.cvtudq2pd."))) {

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

936

// Lossless i32/float to double conversion.

937

// Extract the bottom elements if necessary and convert to double vector.

938

Value *Src = CI->getArgOperand(0);

939

VectorType *SrcTy = cast<VectorType>(Src->getType());

940

VectorType *DstTy = cast<VectorType>(CI->getType());

941

Rep = CI->getArgOperand(0);

942

943

unsigned NumDstElts = DstTy->getNumElements();

944

if (NumDstElts < SrcTy->getNumElements()) {

945

assert(NumDstElts == 2 && "Unexpected vector size");

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

946

uint32_t ShuffleMask[2] = { 0, 1 };

947

Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy),

948

ShuffleMask);

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

949

}

950

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

951

bool SInt2Double = (StringRef::npos != Name.find("cvtdq2"));

952

bool UInt2Double = (StringRef::npos != Name.find("cvtudq2"));

953

if (SInt2Double)

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

954

Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd");

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

955

else if (UInt2Double)

956

Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd");

Simon Pilgrim

2016-05-25 08:59:18 +0000

[diff] [blame]

957

else

958

Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");

Simon Pilgrim

2016-11-16 14:48:32 +0000

[diff] [blame]

959

960

if (CI->getNumArgOperands() == 3)

961

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

962

CI->getArgOperand(1));

Craig Topper

a57d2ca

2016-09-03 23:55:13 +0000

[diff] [blame]

963

} else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

964

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

965

CI->getArgOperand(1), CI->getArgOperand(2),

966

/*Aligned*/false);

Craig Topper

a57d2ca

2016-09-03 23:55:13 +0000

[diff] [blame]

967

} else if (IsX86 && (Name.startswith("avx512.mask.load."))) {

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

968

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

Craig Topper

2016-06-02 04:19:36 +0000

[diff] [blame]

969

CI->getArgOperand(1),CI->getArgOperand(2),

970

/*Aligned*/true);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

971

} else if (IsX86 && Name.startswith("xop.vpcom")) {

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

972

Intrinsic::ID intID;

973

if (Name.endswith("ub"))

974

intID = Intrinsic::x86_xop_vpcomub;

975

else if (Name.endswith("uw"))

976

intID = Intrinsic::x86_xop_vpcomuw;

977

else if (Name.endswith("ud"))

978

intID = Intrinsic::x86_xop_vpcomud;

979

else if (Name.endswith("uq"))

980

intID = Intrinsic::x86_xop_vpcomuq;

981

else if (Name.endswith("b"))

982

intID = Intrinsic::x86_xop_vpcomb;

983

else if (Name.endswith("w"))

984

intID = Intrinsic::x86_xop_vpcomw;

985

else if (Name.endswith("d"))

986

intID = Intrinsic::x86_xop_vpcomd;

987

else if (Name.endswith("q"))

988

intID = Intrinsic::x86_xop_vpcomq;

989

else

990

llvm_unreachable("Unknown suffix");

991

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

992

Name = Name.substr(9); // strip off "xop.vpcom"

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

993

unsigned Imm;

994

if (Name.startswith("lt"))

995

Imm = 0;

996

else if (Name.startswith("le"))

997

Imm = 1;

998

else if (Name.startswith("gt"))

999

Imm = 2;

1000

else if (Name.startswith("ge"))

1001

Imm = 3;

1002

else if (Name.startswith("eq"))

1003

Imm = 4;

1004

else if (Name.startswith("ne"))

1005

Imm = 5;

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

1006

else if (Name.startswith("false"))

Craig Topper

e32546d

2015-02-13 07:42:15 +0000

[diff] [blame]

1007

Imm = 6;

1008

else if (Name.startswith("true"))

Craig Topper

2012-06-09 16:46:13 +0000

[diff] [blame]

1009

Imm = 7;

1010

else

1011

llvm_unreachable("Unknown condition");

1012

1013

Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1014

Rep =

1015

Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),

1016

Builder.getInt8(Imm)});

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1017

} else if (IsX86 && Name == "xop.vpcmov") {

Simon Pilgrim

e88dc04

2015-11-03 20:27:01 +0000

[diff] [blame]

1018

Value *Arg0 = CI->getArgOperand(0);

1019

Value *Arg1 = CI->getArgOperand(1);

1020

Value *Sel = CI->getArgOperand(2);

1021

unsigned NumElts = CI->getType()->getVectorNumElements();

1022

Constant *MinusOne = ConstantVector::getSplat(NumElts, Builder.getInt64(-1));

1023

Value *NotSel = Builder.CreateXor(Sel, MinusOne);

1024

Value *Sel0 = Builder.CreateAnd(Arg0, Sel);

1025

Value *Sel1 = Builder.CreateAnd(Arg1, NotSel);

1026

Rep = Builder.CreateOr(Sel0, Sel1);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1027

} else if (IsX86 && Name == "sse42.crc32.64.8") {

Craig Topper

ef9e993

2013-10-15 05:20:47 +0000

[diff] [blame]

1028

Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),

1029

Intrinsic::x86_sse42_crc32_32_8);

1030

Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1031

Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});

Craig Topper

ef9e993

2013-10-15 05:20:47 +0000

[diff] [blame]

1032

Rep = Builder.CreateZExt(Rep, CI->getType(), "");

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1033

} else if (IsX86 && Name.startswith("avx.vbroadcast.s")) {

Adam Nemet

3906680

2014-05-29 23:35:33 +0000

[diff] [blame]

1034

// Replace broadcasts with a series of insertelements.

1035

Type *VecTy = CI->getType();

1036

Type *EltTy = VecTy->getVectorElementType();

1037

unsigned EltNum = VecTy->getVectorNumElements();

1038

Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),

1039

EltTy->getPointerTo());

David Blaikie

0c28fd7

2015-05-20 21:46:30 +0000

[diff] [blame]

1040

Value *Load = Builder.CreateLoad(EltTy, Cast);

Adam Nemet

3906680

2014-05-29 23:35:33 +0000

[diff] [blame]

1041

Type *I32Ty = Type::getInt32Ty(C);

1042

Rep = UndefValue::get(VecTy);

1043

for (unsigned I = 0; I < EltNum; ++I)

1044

Rep = Builder.CreateInsertElement(Rep, Load,

1045

ConstantInt::get(I32Ty, I));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1046

} else if (IsX86 && (Name.startswith("sse41.pmovsx") ||

1047

Name.startswith("sse41.pmovzx") ||

1048

Name.startswith("avx2.pmovsx") ||

Craig Topper

b110e04

2016-11-07 02:12:57 +0000

[diff] [blame]

1049

Name.startswith("avx2.pmovzx") ||

1050

Name.startswith("avx512.mask.pmovsx") ||

1051

Name.startswith("avx512.mask.pmovzx"))) {

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1052

VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());

1053

VectorType *DstTy = cast<VectorType>(CI->getType());

1054

unsigned NumDstElts = DstTy->getNumElements();

1055

Simon Pilgrim

9602d67

2016-05-28 18:03:41 +0000

[diff] [blame]

1056

// Extract a subvector of the first NumDstElts lanes and sign/zero extend.

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1057

SmallVector<uint32_t, 8> ShuffleMask(NumDstElts);

Craig Topper

2016-06-12 00:41:19 +0000

[diff] [blame]

1058

for (unsigned i = 0; i != NumDstElts; ++i)

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1059

ShuffleMask[i] = i;

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1060

1061

Value *SV = Builder.CreateShuffleVector(

1062

CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);

Simon Pilgrim

9602d67

2016-05-28 18:03:41 +0000

[diff] [blame]

1063

1064

bool DoSext = (StringRef::npos != Name.find("pmovsx"));

1065

Rep = DoSext ? Builder.CreateSExt(SV, DstTy)

1066

: Builder.CreateZExt(SV, DstTy);

Craig Topper

b110e04

2016-11-07 02:12:57 +0000

[diff] [blame]

1067

// If there are 3 arguments, it's a masked intrinsic so we need a select.

1068

if (CI->getNumArgOperands() == 3)

1069

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1070

CI->getArgOperand(1));

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1071

} else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||

1072

Name == "avx2.vbroadcasti128")) {

1073

// Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.

1074

Type *EltTy = CI->getType()->getVectorElementType();

1075

unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();

1076

Type *VT = VectorType::get(EltTy, NumSrcElts);

David Blaikie

0c28fd7

2015-05-20 21:46:30 +0000

[diff] [blame]

1077

Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),

1078

PointerType::getUnqual(VT));

Chandler Carruth

0215e76

2016-08-10 07:41:26 +0000

[diff] [blame]

1079

Value *Load = Builder.CreateAlignedLoad(Op, 1);

Simon Pilgrim

ea0d4f9

2016-07-22 13:58:44 +0000

[diff] [blame]

1080

if (NumSrcElts == 2)

1081

Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),

1082

{ 0, 1, 0, 1 });

1083

else

1084

Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),

1085

{ 0, 1, 2, 3, 0, 1, 2, 3 });

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1086

} else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||

Simon Pilgrim

4e96fbf

2016-07-05 13:58:47 +0000

[diff] [blame]

1087

Name.startswith("avx2.vbroadcast") ||

1088

Name.startswith("avx512.pbroadcast") ||

1089

Name.startswith("avx512.mask.broadcast.s"))) {

Ahmed Bougacha

1a498705

2015-08-20 20:36:19 +0000

[diff] [blame]

1090

// Replace vp?broadcasts with a vector shuffle.

1091

Value *Op = CI->getArgOperand(0);

1092

unsigned NumElts = CI->getType()->getVectorNumElements();

1093

Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts);

1094

Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),

1095

Constant::getNullValue(MaskTy));

Simon Pilgrim

4e96fbf

2016-07-05 13:58:47 +0000

[diff] [blame]

1096

1097

if (CI->getNumArgOperands() == 3)

1098

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1099

CI->getArgOperand(1));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1100

} else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {

Craig Topper

2016-11-23 06:54:55 +0000

[diff] [blame]

1101

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

1102

CI->getArgOperand(1),

1103

CI->getArgOperand(2),

1104

CI->getArgOperand(3),

1105

CI->getArgOperand(4),

1106

false);

1107

} else if (IsX86 && Name.startswith("avx512.mask.valign.")) {

1108

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

1109

CI->getArgOperand(1),

1110

CI->getArgOperand(2),

1111

CI->getArgOperand(3),

1112

CI->getArgOperand(4),

1113

true);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1114

} else if (IsX86 && (Name == "sse2.psll.dq" ||

1115

Name == "avx2.psll.dq")) {

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

1116

// 128/256-bit shift left specified in bits.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1117

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1118

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1119

Shift / 8); // Shift is in bits.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1120

} else if (IsX86 && (Name == "sse2.psrl.dq" ||

1121

Name == "avx2.psrl.dq")) {

Craig Topper

2016-05-29 06:37:33 +0000

[diff] [blame]

1122

// 128/256-bit shift right specified in bits.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1123

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1124

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1125

Shift / 8); // Shift is in bits.

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1126

} else if (IsX86 && (Name == "sse2.psll.dq.bs" ||

1127

Name == "avx2.psll.dq.bs" ||

1128

Name == "avx512.psll.dq.512")) {

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

1129

// 128/256/512-bit shift left specified in bytes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1130

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1131

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1132

} else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||

1133

Name == "avx2.psrl.dq.bs" ||

1134

Name == "avx512.psrl.dq.512")) {

Simon Pilgrim

f718682

2016-06-09 21:09:03 +0000

[diff] [blame]

1135

// 128/256/512-bit shift right specified in bytes.

Craig Topper

2015-02-18 06:24:44 +0000

[diff] [blame]

1136

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

Craig Topper

2016-07-12 01:42:33 +0000

[diff] [blame]

1137

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1138

} else if (IsX86 && (Name == "sse41.pblendw" ||

1139

Name.startswith("sse41.blendp") ||

1140

Name.startswith("avx.blend.p") ||

1141

Name == "avx2.pblendw" ||

1142

Name.startswith("avx2.pblendd."))) {

Craig Topper

782d620

2015-02-28 19:33:17 +0000

[diff] [blame]

1143

Value *Op0 = CI->getArgOperand(0);

1144

Value *Op1 = CI->getArgOperand(1);

1145

unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1146

VectorType *VecTy = cast<VectorType>(CI->getType());

1147

unsigned NumElts = VecTy->getNumElements();

1148

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1149

SmallVector<uint32_t, 16> Idxs(NumElts);

1150

for (unsigned i = 0; i != NumElts; ++i)

1151

Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;

Craig Topper

782d620

2015-02-28 19:33:17 +0000

[diff] [blame]

1152

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1153

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1154

} else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||

1155

Name == "avx2.vinserti128")) {

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1156

Value *Op0 = CI->getArgOperand(0);

1157

Value *Op1 = CI->getArgOperand(1);

1158

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1159

VectorType *VecTy = cast<VectorType>(CI->getType());

1160

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1161

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1162

// Mask off the high bits of the immediate value; hardware ignores those.

1163

Imm = Imm & 1;

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1164

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1165

// Extend the second operand into a vector that is twice as big.

1166

Value *UndefV = UndefValue::get(Op1->getType());

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1167

SmallVector<uint32_t, 8> Idxs(NumElts);

1168

for (unsigned i = 0; i != NumElts; ++i)

1169

Idxs[i] = i;

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1170

Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1171

1172

// Insert the second operand into the first operand.

1173

1174

// Note that there is no guarantee that instruction lowering will actually

1175

// produce a vinsertf128 instruction for the created shuffles. In

1176

// particular, the 0 immediate case involves no lane changes, so it can

1177

// be handled as a blend.

1178

1179

// Example of shuffle mask for 32-bit elements:

1180

// Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>

1181

// Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >

1182

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1183

// The low half of the result is either the low half of the 1st operand

1184

// or the low half of the 2nd operand (the inserted vector).

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1185

for (unsigned i = 0; i != NumElts / 2; ++i)

1186

Idxs[i] = Imm ? i : (i + NumElts);

Sanjay Patel

2015-03-10 16:08:36 +0000

[diff] [blame]

1187

// The high half of the result is either the low half of the 2nd operand

1188

// (the inserted vector) or the high half of the 1st operand.

Craig Topper

2016-06-12 04:48:00 +0000

[diff] [blame]

1189

for (unsigned i = NumElts / 2; i != NumElts; ++i)

1190

Idxs[i] = Imm ? (i + NumElts / 2) : i;

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1191

Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1192

} else if (IsX86 && (Name.startswith("avx.vextractf128.") ||

1193

Name == "avx2.vextracti128")) {

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1194

Value *Op0 = CI->getArgOperand(0);

1195

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1196

VectorType *VecTy = cast<VectorType>(CI->getType());

1197

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

2015-09-23 08:48:33 +0000

[diff] [blame]

1198

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1199

// Mask off the high bits of the immediate value; hardware ignores those.

1200

Imm = Imm & 1;

1201

1202

// Get indexes for either the high half or low half of the input vector.

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1203

SmallVector<uint32_t, 4> Idxs(NumElts);

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1204

for (unsigned i = 0; i != NumElts; ++i) {

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1205

Idxs[i] = Imm ? (i + NumElts) : i;

Sanjay Patel

2015-03-12 15:15:19 +0000

[diff] [blame]

1206

}

1207

1208

Value *UndefV = UndefValue::get(Op0->getType());

Craig Topper

2016-06-12 01:05:59 +0000

[diff] [blame]

1209

Rep = Builder.CreateShuffleVector(Op0, UndefV, Idxs);

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1210

} else if (!IsX86 && Name == "stackprotectorcheck") {

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

1211

Rep = nullptr;

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1212

} else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||

1213

Name.startswith("avx512.mask.perm.di."))) {

Simon Pilgrim

02d435d

2016-07-04 14:19:05 +0000

[diff] [blame]

1214

Value *Op0 = CI->getArgOperand(0);

1215

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1216

VectorType *VecTy = cast<VectorType>(CI->getType());

1217

unsigned NumElts = VecTy->getNumElements();

1218

1219

SmallVector<uint32_t, 8> Idxs(NumElts);

1220

for (unsigned i = 0; i != NumElts; ++i)

1221

Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);

1222

1223

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

1224

1225

if (CI->getNumArgOperands() == 4)

1226

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1227

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1228

} else if (IsX86 && (Name.startswith("avx.vpermil.") ||

1229

Name == "sse2.pshuf.d" ||

1230

Name.startswith("avx512.mask.vpermil.p") ||

1231

Name.startswith("avx512.mask.pshuf.d."))) {

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1232

Value *Op0 = CI->getArgOperand(0);

1233

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1234

VectorType *VecTy = cast<VectorType>(CI->getType());

1235

unsigned NumElts = VecTy->getNumElements();

Simon Pilgrim

9fca300

2016-07-04 12:40:54 +0000

[diff] [blame]

1236

// Calculate the size of each index in the immediate.

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1237

unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();

1238

unsigned IdxMask = ((1 << IdxSize) - 1);

1239

1240

SmallVector<uint32_t, 8> Idxs(NumElts);

1241

// Lookup the bits for this element, wrapping around the immediate every

1242

// 8-bits. Elements are grouped into sets of 2 or 4 elements so we need

1243

// to offset by the first index of each group.

1244

for (unsigned i = 0; i != NumElts; ++i)

1245

Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);

1246

1247

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1248

1249

if (CI->getNumArgOperands() == 4)

1250

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1251

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1252

} else if (IsX86 && (Name == "sse2.pshufl.w" ||

1253

Name.startswith("avx512.mask.pshufl.w."))) {

Craig Topper

1067986

2016-06-12 14:11:32 +0000

[diff] [blame]

1254

Value *Op0 = CI->getArgOperand(0);

1255

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1256

unsigned NumElts = CI->getType()->getVectorNumElements();

1257

1258

SmallVector<uint32_t, 16> Idxs(NumElts);

1259

for (unsigned l = 0; l != NumElts; l += 8) {

1260

for (unsigned i = 0; i != 4; ++i)

1261

Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;

1262

for (unsigned i = 4; i != 8; ++i)

Idxs[i + l] = i + l;

}

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1267

1268

if (CI->getNumArgOperands() == 4)

1269

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1270

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1271

} else if (IsX86 && (Name == "sse2.pshufh.w" ||

1272

Name.startswith("avx512.mask.pshufh.w."))) {

Craig Topper

1067986

2016-06-12 14:11:32 +0000

[diff] [blame]

1273

Value *Op0 = CI->getArgOperand(0);

1274

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

1275

unsigned NumElts = CI->getType()->getVectorNumElements();

1276

1277

SmallVector<uint32_t, 16> Idxs(NumElts);

1278

for (unsigned l = 0; l != NumElts; l += 8) {

1279

for (unsigned i = 0; i != 4; ++i)

1280

Idxs[i + l] = i + l;

1281

for (unsigned i = 0; i != 4; ++i)

1282

Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;

1283

}

1284

1285

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

Craig Topper

13cf7ca

2016-06-13 02:36:48 +0000

[diff] [blame]

1286

1287

if (CI->getNumArgOperands() == 4)

1288

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1289

CI->getArgOperand(2));

Craig Topper

4619c9e

2016-09-13 07:40:53 +0000

[diff] [blame]

1290

} else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {

1291

Value *Op0 = CI->getArgOperand(0);

1292

Value *Op1 = CI->getArgOperand(1);

1293

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

1294

unsigned NumElts = CI->getType()->getVectorNumElements();

1295

1296

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1297

unsigned HalfLaneElts = NumLaneElts / 2;

1298

1299

SmallVector<uint32_t, 16> Idxs(NumElts);

1300

for (unsigned i = 0; i != NumElts; ++i) {

1301

// Base index is the starting element of the lane.

1302

Idxs[i] = i - (i % NumLaneElts);

1303

// If we are half way through the lane switch to the other source.

1304

if ((i % NumLaneElts) >= HalfLaneElts)

1305

Idxs[i] += NumElts;

1306

// Now select the specific element. By adding HalfLaneElts bits from

1307

// the immediate. Wrapping around the immediate every 8-bits.

1308

Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);

1309

}

1310

1311

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1312

1313

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,

1314

CI->getArgOperand(3));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1315

} else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||

1316

Name.startswith("avx512.mask.movshdup") ||

1317

Name.startswith("avx512.mask.movsldup"))) {

Simon Pilgrim

19adee9

2016-07-02 14:42:35 +0000

[diff] [blame]

1318

Value *Op0 = CI->getArgOperand(0);

1319

unsigned NumElts = CI->getType()->getVectorNumElements();

1320

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1321

1322

unsigned Offset = 0;

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1323

if (Name.startswith("avx512.mask.movshdup."))

Simon Pilgrim

19adee9

2016-07-02 14:42:35 +0000

[diff] [blame]

1324

Offset = 1;

1325

1326

SmallVector<uint32_t, 16> Idxs(NumElts);

1327

for (unsigned l = 0; l != NumElts; l += NumLaneElts)

1328

for (unsigned i = 0; i != NumLaneElts; i += 2) {

1329

Idxs[i + l + 0] = i + l + Offset;

1330

Idxs[i + l + 1] = i + l + Offset;

1331

}

1332

1333

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

1334

1335

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1336

CI->getArgOperand(1));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1337

} else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||

1338

Name.startswith("avx512.mask.unpckl."))) {

Craig Topper

597aa42

2016-06-23 07:37:33 +0000

[diff] [blame]

1339

Value *Op0 = CI->getArgOperand(0);

1340

Value *Op1 = CI->getArgOperand(1);

1341

int NumElts = CI->getType()->getVectorNumElements();

1342

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1343

1344

SmallVector<uint32_t, 64> Idxs(NumElts);

1345

for (int l = 0; l != NumElts; l += NumLaneElts)

1346

for (int i = 0; i != NumLaneElts; ++i)

1347

Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);

1348

1349

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1350

1351

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1352

CI->getArgOperand(2));

Craig Topper

2016-07-04 20:56:38 +0000

[diff] [blame]

1353

} else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||

1354

Name.startswith("avx512.mask.unpckh."))) {

Craig Topper

597aa42

2016-06-23 07:37:33 +0000

[diff] [blame]

1355

Value *Op0 = CI->getArgOperand(0);

1356

Value *Op1 = CI->getArgOperand(1);

1357

int NumElts = CI->getType()->getVectorNumElements();

1358

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

1359

1360

SmallVector<uint32_t, 64> Idxs(NumElts);

1361

for (int l = 0; l != NumElts; l += NumLaneElts)

1362

for (int i = 0; i != NumLaneElts; ++i)

1363

Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);

1364

1365

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

1366

1367

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1368

CI->getArgOperand(2));

Craig Topper

a6e6feb

2016-07-12 05:27:53 +0000

[diff] [blame]

1369

} else if (IsX86 && Name.startswith("avx512.mask.pand.")) {

1370

Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1));

1371

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1372

CI->getArgOperand(2));

1373

} else if (IsX86 && Name.startswith("avx512.mask.pandn.")) {

1374

Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)),

1375

CI->getArgOperand(1));

1376

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1377

CI->getArgOperand(2));

1378

} else if (IsX86 && Name.startswith("avx512.mask.por.")) {

1379

Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1));

1380

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1381

CI->getArgOperand(2));

1382

} else if (IsX86 && Name.startswith("avx512.mask.pxor.")) {

1383

Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1));

1384

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1385

CI->getArgOperand(2));

Craig Topper

e75c495

2016-09-02 05:29:17 +0000

[diff] [blame]

1386

} else if (IsX86 && Name.startswith("avx512.mask.and.")) {

1387

VectorType *FTy = cast<VectorType>(CI->getType());

1388

VectorType *ITy = VectorType::getInteger(FTy);

1389

Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1390

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1391

Rep = Builder.CreateBitCast(Rep, FTy);

1392

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1393

CI->getArgOperand(2));

1394

} else if (IsX86 && Name.startswith("avx512.mask.andn.")) {

1395

VectorType *FTy = cast<VectorType>(CI->getType());

1396

VectorType *ITy = VectorType::getInteger(FTy);

1397

Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));

1398

Rep = Builder.CreateAnd(Rep,

1399

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1400

Rep = Builder.CreateBitCast(Rep, FTy);

1401

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1402

CI->getArgOperand(2));

1403

} else if (IsX86 && Name.startswith("avx512.mask.or.")) {

1404

VectorType *FTy = cast<VectorType>(CI->getType());

1405

VectorType *ITy = VectorType::getInteger(FTy);

1406

Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1407

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1408

Rep = Builder.CreateBitCast(Rep, FTy);

1409

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1410

CI->getArgOperand(2));

1411

} else if (IsX86 && Name.startswith("avx512.mask.xor.")) {

1412

VectorType *FTy = cast<VectorType>(CI->getType());

1413

VectorType *ITy = VectorType::getInteger(FTy);

1414

Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

1415

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

1416

Rep = Builder.CreateBitCast(Rep, FTy);

1417

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1418

CI->getArgOperand(2));

Craig Topper

af0d63d

2016-09-04 02:09:53 +0000

[diff] [blame]

1419

} else if (IsX86 && Name.startswith("avx512.mask.padd.")) {

1420

Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));

1421

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1422

CI->getArgOperand(2));

1423

} else if (IsX86 && Name.startswith("avx512.mask.psub.")) {

1424

Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));

1425

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1426

CI->getArgOperand(2));

1427

} else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {

1428

Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));

1429

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1430

CI->getArgOperand(2));

Craig Topper

8f354a7

2016-11-05 05:35:23 +0000

[diff] [blame]

1431

} else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1432

Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));

1433

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1434

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1435

} else if (IsX86 && Name.startswith("avx512.mask.div.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1436

Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));

1437

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1438

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1439

} else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1440

Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));

1441

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1442

CI->getArgOperand(2));

Craig Topper

96041c6

2016-11-07 00:13:42 +0000

[diff] [blame]

1443

} else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {

Craig Topper

2016-09-04 18:13:33 +0000

[diff] [blame]

1444

Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));

1445

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1446

CI->getArgOperand(2));

Craig Topper

7e54533

2016-11-07 00:13:39 +0000

[diff] [blame]

1447

} else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) {

1448

VectorType *VecTy = cast<VectorType>(CI->getType());

1449

Intrinsic::ID IID;

1450

if (VecTy->getPrimitiveSizeInBits() == 128)

1451

IID = Intrinsic::x86_ssse3_pshuf_b_128;

1452

else if (VecTy->getPrimitiveSizeInBits() == 256)

1453

IID = Intrinsic::x86_avx2_pshuf_b;

1454

else

1455

llvm_unreachable("Unexpected intrinsic");

1456

1457

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

1458

{ CI->getArgOperand(0), CI->getArgOperand(1) });

1459

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1460

CI->getArgOperand(2));

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1461

} else if (IsX86 && Name.startswith("avx512.mask.psll")) {

1462

bool IsImmediate = Name[16] == 'i' ||

1463

(Name.size() > 18 && Name[18] == 'i');

1464

bool IsVariable = Name[16] == 'v';

1465

char Size = Name[16] == '.' ? Name[17] :

1466

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1467

Name[18] == '.' ? Name[19] :

1468

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1469

1470

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1471

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1472

if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di

1473

IID = Intrinsic::x86_avx2_psllv_q;

1474

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di

1475

IID = Intrinsic::x86_avx2_psllv_q_256;

1476

else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si

1477

IID = Intrinsic::x86_avx2_psllv_d;

1478

else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si

1479

IID = Intrinsic::x86_avx2_psllv_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1480

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi

1481

IID = Intrinsic::x86_avx512_psllv_w_128;

1482

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi

1483

IID = Intrinsic::x86_avx512_psllv_w_256;

1484

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi

1485

IID = Intrinsic::x86_avx512_psllv_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1486

else

1487

llvm_unreachable("Unexpected size");

1488

} else if (Name.endswith(".128")) {

1489

if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128

1490

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d

1491

: Intrinsic::x86_sse2_psll_d;

1492

else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128

1493

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q

1494

: Intrinsic::x86_sse2_psll_q;

1495

else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128

1496

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w

1497

: Intrinsic::x86_sse2_psll_w;

1498

else

1499

llvm_unreachable("Unexpected size");

1500

} else if (Name.endswith(".256")) {

1501

if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256

1502

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d

1503

: Intrinsic::x86_avx2_psll_d;

1504

else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256

1505

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q

1506

: Intrinsic::x86_avx2_psll_q;

1507

else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256

1508

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w

1509

: Intrinsic::x86_avx2_psll_w;

1510

else

1511

llvm_unreachable("Unexpected size");

1512

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1513

if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512

1514

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :

1515

IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :

1516

Intrinsic::x86_avx512_psll_d_512;

1517

else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512

1518

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :

1519

IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :

1520

Intrinsic::x86_avx512_psll_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1521

else if (Size == 'w') // psll.wi.512, pslli.w, psll.w

1522

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512

1523

: Intrinsic::x86_avx512_psll_w_512;

1524

else

1525

llvm_unreachable("Unexpected size");

1526

}

1527

1528

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

1529

} else if (IsX86 && Name.startswith("avx512.mask.psrl")) {

1530

bool IsImmediate = Name[16] == 'i' ||

1531

(Name.size() > 18 && Name[18] == 'i');

1532

bool IsVariable = Name[16] == 'v';

1533

char Size = Name[16] == '.' ? Name[17] :

1534

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1535

Name[18] == '.' ? Name[19] :

1536

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1537

1538

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1539

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1540

if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di

1541

IID = Intrinsic::x86_avx2_psrlv_q;

1542

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di

1543

IID = Intrinsic::x86_avx2_psrlv_q_256;

1544

else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si

1545

IID = Intrinsic::x86_avx2_psrlv_d;

1546

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si

1547

IID = Intrinsic::x86_avx2_psrlv_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1548

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi

1549

IID = Intrinsic::x86_avx512_psrlv_w_128;

1550

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi

1551

IID = Intrinsic::x86_avx512_psrlv_w_256;

1552

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi

1553

IID = Intrinsic::x86_avx512_psrlv_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1554

else

1555

llvm_unreachable("Unexpected size");

1556

} else if (Name.endswith(".128")) {

1557

if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128

1558

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d

1559

: Intrinsic::x86_sse2_psrl_d;

1560

else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128

1561

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q

1562

: Intrinsic::x86_sse2_psrl_q;

1563

else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128

1564

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w

1565

: Intrinsic::x86_sse2_psrl_w;

1566

else

1567

llvm_unreachable("Unexpected size");

1568

} else if (Name.endswith(".256")) {

1569

if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256

1570

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d

1571

: Intrinsic::x86_avx2_psrl_d;

1572

else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256

1573

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q

1574

: Intrinsic::x86_avx2_psrl_q;

1575

else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256

1576

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w

1577

: Intrinsic::x86_avx2_psrl_w;

1578

else

1579

llvm_unreachable("Unexpected size");

1580

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1581

if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512

1582

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :

1583

IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :

1584

Intrinsic::x86_avx512_psrl_d_512;

1585

else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512

1586

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :

1587

IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :

1588

Intrinsic::x86_avx512_psrl_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1589

else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)

1590

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512

1591

: Intrinsic::x86_avx512_psrl_w_512;

1592

else

1593

llvm_unreachable("Unexpected size");

1594

}

1595

1596

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

1597

} else if (IsX86 && Name.startswith("avx512.mask.psra")) {

1598

bool IsImmediate = Name[16] == 'i' ||

1599

(Name.size() > 18 && Name[18] == 'i');

1600

bool IsVariable = Name[16] == 'v';

1601

char Size = Name[16] == '.' ? Name[17] :

1602

Name[17] == '.' ? Name[18] :

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1603

Name[18] == '.' ? Name[19] :

1604

Name[20];

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1605

1606

Intrinsic::ID IID;

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1607

if (IsVariable && Name[17] != '.') {

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1608

if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si

1609

IID = Intrinsic::x86_avx2_psrav_d;

1610

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si

1611

IID = Intrinsic::x86_avx2_psrav_d_256;

Craig Topper

2016-11-18 05:04:44 +0000

[diff] [blame]

1612

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi

1613

IID = Intrinsic::x86_avx512_psrav_w_128;

1614

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi

1615

IID = Intrinsic::x86_avx512_psrav_w_256;

1616

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi

1617

IID = Intrinsic::x86_avx512_psrav_w_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1618

else

1619

llvm_unreachable("Unexpected size");

1620

} else if (Name.endswith(".128")) {

1621

if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128

1622

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d

1623

: Intrinsic::x86_sse2_psra_d;

1624

else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1625

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :

1626

IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :

1627

Intrinsic::x86_avx512_psra_q_128;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1628

else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128

1629

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w

1630

: Intrinsic::x86_sse2_psra_w;

1631

else

1632

llvm_unreachable("Unexpected size");

1633

} else if (Name.endswith(".256")) {

1634

if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256

1635

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d

1636

: Intrinsic::x86_avx2_psra_d;

1637

else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1638

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :

1639

IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :

1640

Intrinsic::x86_avx512_psra_q_256;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1641

else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256

1642

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w

1643

: Intrinsic::x86_avx2_psra_w;

1644

else

1645

llvm_unreachable("Unexpected size");

1646

} else {

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1647

if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512

1648

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :

1649

IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :

1650

Intrinsic::x86_avx512_psra_d_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1651

else if (Size == 'q') // psra.qi.512, psrai.q, psra.q

Craig Topper

2016-11-14 01:53:22 +0000

[diff] [blame]

1652

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :

1653

IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :

1654

Intrinsic::x86_avx512_psra_q_512;

Craig Topper

2016-11-13 19:09:56 +0000

[diff] [blame]

1655

else if (Size == 'w') // psra.wi.512, psrai.w, psra.w

1656

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512

1657

: Intrinsic::x86_avx512_psra_w_512;

1658

else

1659

llvm_unreachable("Unexpected size");

1660

}

1661

1662

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

Ayman Musa

4d60243

2016-11-16 09:00:28 +0000

[diff] [blame]

1663

} else if (IsX86 && Name.startswith("avx512.mask.move.s")) {

1664

Rep = upgradeMaskedMove(Builder, *CI);

Craig Topper

abe7c5b

2016-12-10 21:15:52 +0000

[diff] [blame]

1665

} else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) {

1666

Intrinsic::ID IID;

1667

if (Name.endswith("ps.128"))

1668

IID = Intrinsic::x86_avx_vpermilvar_ps;

1669

else if (Name.endswith("pd.128"))

1670

IID = Intrinsic::x86_avx_vpermilvar_pd;

1671

else if (Name.endswith("ps.256"))

1672

IID = Intrinsic::x86_avx_vpermilvar_ps_256;

1673

else if (Name.endswith("pd.256"))

1674

IID = Intrinsic::x86_avx_vpermilvar_pd_256;

1675

else

1676

llvm_unreachable("Unexpected vpermilvar intrinsic");

1677

1678

Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);

1679

Rep = Builder.CreateCall(Intrin,

1680

{ CI->getArgOperand(0), CI->getArgOperand(1) });

1681

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

1682

CI->getArgOperand(2));

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1683

} else {

Craig Topper

8a10505

2016-06-12 03:10:47 +0000

[diff] [blame]

1684

llvm_unreachable("Unknown function for CallInst upgrade.");

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1685

}

1686

Tim Shen

0012756

2016-04-08 21:26:31 +0000

[diff] [blame]

1687

if (Rep)

1688

CI->replaceAllUsesWith(Rep);

Craig Topper

2012-02-03 06:10:55 +0000

[diff] [blame]

1689

CI->eraseFromParent();

return;

}

Yaron Keren

2015-03-30 16:10:39 +0000

[diff] [blame]

1693

std::string Name = CI->getName();

Adrian Prantl

87b7eb9

2014-10-01 18:55:02 +0000

[diff] [blame]

1694

if (!Name.empty())

1695

CI->setName(Name + ".old");

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1696

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1697

switch (NewFn->getIntrinsicID()) {

1698

default:

Chris Lattner

0bcbde4

2011-11-27 08:42:07 +0000

[diff] [blame]

1699

llvm_unreachable("Unknown function for CallInst upgrade.");

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1700

Jeroen Ketema

2015-09-30 10:56:37 +0000

[diff] [blame]

1701

case Intrinsic::arm_neon_vld1:

1702

case Intrinsic::arm_neon_vld2:

1703

case Intrinsic::arm_neon_vld3:

1704

case Intrinsic::arm_neon_vld4:

1705

case Intrinsic::arm_neon_vld2lane:

1706

case Intrinsic::arm_neon_vld3lane:

1707

case Intrinsic::arm_neon_vld4lane:

1708

case Intrinsic::arm_neon_vst1:

1709

case Intrinsic::arm_neon_vst2:

1710

case Intrinsic::arm_neon_vst3:

1711

case Intrinsic::arm_neon_vst4:

1712

case Intrinsic::arm_neon_vst2lane:

1713

case Intrinsic::arm_neon_vst3lane:

1714

case Intrinsic::arm_neon_vst4lane: {

1715

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1716

CI->arg_operands().end());

1717

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args));

1718

CI->eraseFromParent();

return;

}

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1722

case Intrinsic::ctlz:

Nuno Lopes

ad40c0a

2012-05-22 15:25:31 +0000

[diff] [blame]

1723

case Intrinsic::cttz:

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1724

assert(CI->getNumArgOperands() == 1 &&

1725

"Mismatch between function args and call args");

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1726

CI->replaceAllUsesWith(Builder.CreateCall(

1727

NewFn, {CI->getArgOperand(0), Builder.getFalse()}, Name));

Chandler Carruth

2011-12-12 04:26:04 +0000

[diff] [blame]

1728

CI->eraseFromParent();

1729

return;

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1730

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

1731

case Intrinsic::objectsize:

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1732

CI->replaceAllUsesWith(Builder.CreateCall(

1733

NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)}, Name));

Matt Arsenault

fbcbce4

2013-10-07 18:06:48 +0000

[diff] [blame]

1734

CI->eraseFromParent();

1735

return;

1736

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

1737

case Intrinsic::ctpop: {

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1738

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {CI->getArgOperand(0)}));

Joel Jones

b84f7be

2012-07-18 00:02:16 +0000

[diff] [blame]

1739

CI->eraseFromParent();

1740

return;

1741

}

Joel Jones

2012-07-13 23:25:25 +0000

[diff] [blame]

1742

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1743

case Intrinsic::x86_xop_vfrcz_ss:

1744

case Intrinsic::x86_xop_vfrcz_sd:

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1745

CI->replaceAllUsesWith(

1746

Builder.CreateCall(NewFn, {CI->getArgOperand(1)}, Name));

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1747

CI->eraseFromParent();

1748

return;

1749

Simon Pilgrim

2016-06-03 08:06:03 +0000

[diff] [blame]

1750

case Intrinsic::x86_xop_vpermil2pd:

1751

case Intrinsic::x86_xop_vpermil2ps:

1752

case Intrinsic::x86_xop_vpermil2pd_256:

1753

case Intrinsic::x86_xop_vpermil2ps_256: {

1754

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1755

CI->arg_operands().end());

1756

VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());

1757

VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);

1758

Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);

1759

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args, Name));

1760

CI->eraseFromParent();

return;

}

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1764

case Intrinsic::x86_sse41_ptestc:

1765

case Intrinsic::x86_sse41_ptestz:

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1766

case Intrinsic::x86_sse41_ptestnzc: {

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1767

// The arguments for these intrinsics used to be v4f32, and changed

1768

// to v2i64. This is purely a nop, since those are bitwise intrinsics.

1769

// So, the only thing required is a bitcast for both arguments.

1770

// First, check the arguments have the old type.

1771

Value *Arg0 = CI->getArgOperand(0);

1772

if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))

1773

return;

1774

1775

// Old intrinsic, add bitcasts

1776

Value *Arg1 = CI->getArgOperand(1);

1777

David Blaikie

5bacf37

2015-04-24 21:16:07 +0000

[diff] [blame]

1778

Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1779

David Blaikie

5bacf37

2015-04-24 21:16:07 +0000

[diff] [blame]

1780

Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");

1781

Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

1782

David Blaikie

2015-05-18 22:13:54 +0000

[diff] [blame]

1783

CallInst *NewCall = Builder.CreateCall(NewFn, {BC0, BC1}, Name);

Nadav Rotem

2012-06-10 18:42:51 +0000

[diff] [blame]

1784

CI->replaceAllUsesWith(NewCall);

1785

CI->eraseFromParent();

1786

return;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

1787

}

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1788

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1789

case Intrinsic::x86_sse41_insertps:

1790

case Intrinsic::x86_sse41_dppd:

1791

case Intrinsic::x86_sse41_dpps:

1792

case Intrinsic::x86_sse41_mpsadbw:

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1793

case Intrinsic::x86_avx_dp_ps_256:

Chandler Carruth

2014-09-06 10:00:01 +0000

[diff] [blame]

1794

case Intrinsic::x86_avx2_mpsadbw: {

1795

// Need to truncate the last argument from i32 to i8 -- this argument models

1796

// an inherently 8-bit immediate operand to these x86 instructions.

1797

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1798

CI->arg_operands().end());

1799

1800

// Replace the last argument with a trunc.

1801

Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");

1802

1803

CallInst *NewCall = Builder.CreateCall(NewFn, Args);

1804

CI->replaceAllUsesWith(NewCall);

1805

CI->eraseFromParent();

1806

return;

1807

}

Marcin Koscielnicki

3fdc257

2016-04-19 20:51:05 +0000

[diff] [blame]

1808

1809

case Intrinsic::thread_pointer: {

1810

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {}));

1811

CI->eraseFromParent();

1812

return;

1813

}

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

1814

Mehdi Amini

2016-08-13 23:31:24 +0000

[diff] [blame]

1815

case Intrinsic::invariant_start:

1816

case Intrinsic::invariant_end:

Artur Pilipenko

2016-06-28 18:27:25 +0000

[diff] [blame]

1817

case Intrinsic::masked_load:

1818

case Intrinsic::masked_store: {

1819

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

1820

CI->arg_operands().end());

1821

CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args));

1822

CI->eraseFromParent();

1823

return;

1824

}

Craig Topper

2012-06-13 07:18:53 +0000

[diff] [blame]

1825

}

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1826

}

1827

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1828

void llvm::UpgradeCallsToIntrinsic(Function *F) {

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1829

assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");

1830

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1831

// Check if this function should be upgraded and get the replacement function

1832

// if there is one.

Chris Lattner

80ed9dc

2011-06-18 06:05:24 +0000

[diff] [blame]

1833

Function *NewFn;

Evan Cheng

2007-12-17 22:33:23 +0000

[diff] [blame]

1834

if (UpgradeIntrinsicFunction(F, NewFn)) {

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1835

// Replace all users of the old function with the new function or new

1836

// instructions. This is not a range loop because the call is deleted.

1837

for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; )

Duncan P. N. Exon Smith

93f53c4

2016-04-17 03:59:37 +0000

[diff] [blame]

1838

if (CallInst *CI = dyn_cast<CallInst>(*UI++))

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

1839

UpgradeIntrinsicCall(CI, NewFn);

Sanjay Patel

2016-04-18 19:11:57 +0000

[diff] [blame]

1840

Filipe Cabecinhas

0011c58

2015-07-03 20:12:01 +0000

[diff] [blame]

1841

// Remove old function, no longer used, from the module.

1842

F->eraseFromParent();

Chandler Carruth

2007-08-04 01:51:18 +0000

[diff] [blame]

1843

}

1844

}

Devang Patel

80ae349

2009-08-28 23:24:31 +0000

[diff] [blame]

1845

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1846

MDNode *llvm::UpgradeTBAANode(MDNode &MD) {

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1847

// Check if the tag uses struct-path aware TBAA format.

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1848

if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)

1849

return &MD;

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1850

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1851

auto &Context = MD.getContext();

1852

if (MD.getNumOperands() == 3) {

1853

Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};

1854

MDNode *ScalarType = MDNode::get(Context, Elts);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1855

// Create a MDNode <ScalarType, ScalarType, offset 0, const>

Duncan P. N. Exon Smith

5bf8fef

2014-12-09 18:38:53 +0000

[diff] [blame]

1856

Metadata *Elts2[] = {ScalarType, ScalarType,

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1857

ConstantAsMetadata::get(

1858

Constant::getNullValue(Type::getInt64Ty(Context))),

1859

MD.getOperand(2)};

1860

return MDNode::get(Context, Elts2);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1861

}

Mehdi Amini

2016-09-14 22:29:59 +0000

[diff] [blame]

1862

// Create a MDNode <MD, MD, offset 0>

1863

Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(

1864

Type::getInt64Ty(Context)))};

1865

return MDNode::get(Context, Elts);

Manman Ren

2013-09-28 00:22:27 +0000

[diff] [blame]

1866

}

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1867

1868

Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,

1869

Instruction *&Temp) {

1870

if (Opc != Instruction::BitCast)

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1871

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1872

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1873

Temp = nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1874

Type *SrcTy = V->getType();

1875

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

1876

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

1877

LLVMContext &Context = V->getContext();

1878

1879

// We have no information about target data layout, so we assume that

1880

// the maximum pointer size is 64bit.

1881

Type *MidTy = Type::getInt64Ty(Context);

1882

Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);

1883

1884

return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);

1885

}

1886

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1887

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1888

}

1889

1890

Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {

1891

if (Opc != Instruction::BitCast)

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1892

return nullptr;

Matt Arsenault

2013-11-15 01:34:59 +0000

[diff] [blame]

1893

1894

Type *SrcTy = C->getType();

1895

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

1896

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

1897

LLVMContext &Context = C->getContext();

1898

1899

// We have no information about target data layout, so we assume that

1900

// the maximum pointer size is 64bit.

1901

Type *MidTy = Type::getInt64Ty(Context);

1902

1903

return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),

DestTy);

}

Craig Topper

2014-04-09 06:08:46 +0000

[diff] [blame]

1907

return nullptr;

Matt Arsenault