Blame - src/qs8-igemm/gen/3x16c2-minmax-rndnu-neon-mull-ld4r.c - platform/external/XNNPACK

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

1

// Auto-generated file. Do not edit!

2

// Template: src/qs8-igemm/c2-neon-mull-dup.c.in

3

// Generator: tools/xngen

//

//

// This source code is licensed under the BSD-style license found in the

8

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <arm_neon.h>

13

14

#include <xnnpack/gemm.h>

15

#include <xnnpack/math.h>

16

17

void xnn_qs8_igemm_minmax_rndnu_ukernel_3x16c2__neon_mull_ld4r(

size_t mr,

size_t nc,

size_t kc,

size_t ks,

const int8_t** restrict a,

23

const void* restrict w,

int8_t* restrict c,

size_t cm_stride,

size_t cn_stride,

size_t a_offset,

const int8_t* zero,

const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN

{

assert(mr != 0);

assert(mr <= 3);

assert(nc != 0);

assert(kc != 0);

assert(ks != 0);

assert(ks % (3 * sizeof(void*)) == 0);

37

assert(a_offset % sizeof(int8_t) == 0);

assert(a != NULL);

assert(w != NULL);

assert(c != NULL);

kc = round_up_po2(kc, 2 * sizeof(int8_t));

43

int8_t* c0 = c;

44

int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);

45

if XNN_UNPREDICTABLE(mr < 2) {

46

c1 = c0;

47

}

48

int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);

49

if XNN_UNPREDICTABLE(mr <= 2) {

c2 = c1;

}

do {

int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

55

int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

56

int32x4_t vacc0x89AB = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

57

int32x4_t vacc0xCDEF = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

58

int32x4_t vacc1x0123 = vacc0x0123;

59

int32x4_t vacc1x4567 = vacc0x4567;

60

int32x4_t vacc1x89AB = vacc0x89AB;

61

int32x4_t vacc1xCDEF = vacc0xCDEF;

62

int32x4_t vacc2x0123 = vacc0x0123;

63

int32x4_t vacc2x4567 = vacc0x4567;

64

int32x4_t vacc2x89AB = vacc0x89AB;

65

int32x4_t vacc2xCDEF = vacc0xCDEF;

size_t p = ks;

do {

const int8_t* restrict a0 = a[0];

70

if XNN_UNPREDICTABLE(a0 != zero) {

71

a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);

72

}

73

const int8_t* restrict a1 = a[1];

74

if XNN_UNPREDICTABLE(a1 != zero) {

75

a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);

76

}

77

const int8_t* restrict a2 = a[2];

78

if XNN_UNPREDICTABLE(a2 != zero) {

79

a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);

}

a += 3;

size_t k = kc;

while (k >= 8 * sizeof(int8_t)) {

87

const int16x4x4_t va0 = vld4_dup_s16((const void*)a0); a0 += 8;

88

const int16x4x4_t va1 = vld4_dup_s16((const void*)a1); a1 += 8;

89

const int16x4x4_t va2 = vld4_dup_s16((const void*)a2); a2 += 8;

90

91

const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

92

const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

93

const int8x8_t vb89ABc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

94

const int8x8_t vbCDEFc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

95

const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

96

const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

97

const int8x8_t vb89ABc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

98

const int8x8_t vbCDEFc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

99

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

100

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

101

const int8x8_t vb89ABc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

102

const int8x8_t vbCDEFc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

103

const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

104

const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

105

const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

106

const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

107

108

const int8x8_t va0c0 = vreinterpret_s8_s16(va0.val[0]);

109

const int8x8_t va1c0 = vreinterpret_s8_s16(va1.val[0]);

110

const int8x8_t va2c0 = vreinterpret_s8_s16(va2.val[0]);

111

112

const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);

113

const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);

114

const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);

115

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

116

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

117

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);

118

const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);

119

const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);

120

const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);

121

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

122

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

123

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);

124

const int16x8_t vprod0x89ABc0 = vmull_s8(vb89ABc0, va0c0);

125

const int16x8_t vprod1x89ABc0 = vmull_s8(vb89ABc0, va1c0);

126

const int16x8_t vprod2x89ABc0 = vmull_s8(vb89ABc0, va2c0);

127

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc0);

128

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc0);

129

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc0);

130

const int16x8_t vprod0xCDEFc0 = vmull_s8(vbCDEFc0, va0c0);

131

const int16x8_t vprod1xCDEFc0 = vmull_s8(vbCDEFc0, va1c0);

132

const int16x8_t vprod2xCDEFc0 = vmull_s8(vbCDEFc0, va2c0);

133

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc0);

134

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc0);

135

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc0);

136

const int8x8_t va0c1 = vreinterpret_s8_s16(va0.val[1]);

137

const int8x8_t va1c1 = vreinterpret_s8_s16(va1.val[1]);

138

const int8x8_t va2c1 = vreinterpret_s8_s16(va2.val[1]);

139

140

const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);

141

const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);

142

const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);

143

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

144

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

145

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);

146

const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);

147

const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);

148

const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);

149

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

150

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

151

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);

152

const int16x8_t vprod0x89ABc1 = vmull_s8(vb89ABc1, va0c1);

153

const int16x8_t vprod1x89ABc1 = vmull_s8(vb89ABc1, va1c1);

154

const int16x8_t vprod2x89ABc1 = vmull_s8(vb89ABc1, va2c1);

155

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc1);

156

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc1);

157

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc1);

158

const int16x8_t vprod0xCDEFc1 = vmull_s8(vbCDEFc1, va0c1);

159

const int16x8_t vprod1xCDEFc1 = vmull_s8(vbCDEFc1, va1c1);

160

const int16x8_t vprod2xCDEFc1 = vmull_s8(vbCDEFc1, va2c1);

161

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc1);

162

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc1);

163

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc1);

164

const int8x8_t va0c2 = vreinterpret_s8_s16(va0.val[2]);

165

const int8x8_t va1c2 = vreinterpret_s8_s16(va1.val[2]);

166

const int8x8_t va2c2 = vreinterpret_s8_s16(va2.val[2]);

167

168

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);

169

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);

170

const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);

171

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

172

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

173

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);

174

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);

175

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);

176

const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);

177

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

178

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

179

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);

180

const int16x8_t vprod0x89ABc2 = vmull_s8(vb89ABc2, va0c2);

181

const int16x8_t vprod1x89ABc2 = vmull_s8(vb89ABc2, va1c2);

182

const int16x8_t vprod2x89ABc2 = vmull_s8(vb89ABc2, va2c2);

183

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);

184

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);

185

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);

186

const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, va0c2);

187

const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, va1c2);

188

const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, va2c2);

189

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);

190

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);

191

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);

192

const int8x8_t va0c3 = vreinterpret_s8_s16(va0.val[3]);

193

const int8x8_t va1c3 = vreinterpret_s8_s16(va1.val[3]);

194

const int8x8_t va2c3 = vreinterpret_s8_s16(va2.val[3]);

195

196

const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, va0c3);

197

const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, va1c3);

198

const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, va2c3);

199

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);

200

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);

201

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);

202

const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, va0c3);

203

const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, va1c3);

204

const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, va2c3);

205

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);

206

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);

207

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);

208

const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, va0c3);

209

const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, va1c3);

210

const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, va2c3);

211

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);

212

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);

213

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);

214

const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, va0c3);

215

const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, va1c3);

216

const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, va2c3);

217

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);

218

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);

219

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);

220

221

k -= 8 * sizeof(int8_t);

222

}

223

224

if XNN_UNLIKELY(k != 0) {

225

const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);

226

const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);

227

const int8x8_t va2 = vld1_s8(a2); a2 = (const int8_t*) ((uintptr_t) a2 + k);

228

229

const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

230

const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

231

const int8x8_t vb89ABc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

232

const int8x8_t vbCDEFc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

233

234

const int8x8_t va0c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0));

235

const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);

236

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

237

const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);

238

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

239

const int16x8_t vprod0x89ABc0 = vmull_s8(vb89ABc0, va0c0);

240

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc0);

241

const int16x8_t vprod0xCDEFc0 = vmull_s8(vbCDEFc0, va0c0);

242

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc0);

243

const int8x8_t va1c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0));

244

const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);

245

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

246

const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);

247

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

248

const int16x8_t vprod1x89ABc0 = vmull_s8(vb89ABc0, va1c0);

249

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc0);

250

const int16x8_t vprod1xCDEFc0 = vmull_s8(vbCDEFc0, va1c0);

251

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc0);

252

const int8x8_t va2c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 0));

253

const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);

254

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);

255

const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);

256

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);

257

const int16x8_t vprod2x89ABc0 = vmull_s8(vb89ABc0, va2c0);

258

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc0);

259

const int16x8_t vprod2xCDEFc0 = vmull_s8(vbCDEFc0, va2c0);

260

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc0);

261

262

if (k > 2 * sizeof(int8_t)) {

263

const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

264

const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

265

const int8x8_t vb89ABc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

266

const int8x8_t vbCDEFc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

267

268

const int8x8_t va0c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1));

269

const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);

270

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

271

const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);

272

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

273

const int16x8_t vprod0x89ABc1 = vmull_s8(vb89ABc1, va0c1);

274

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc1);

275

const int16x8_t vprod0xCDEFc1 = vmull_s8(vbCDEFc1, va0c1);

276

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc1);

277

const int8x8_t va1c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1));

278

const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);

279

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

280

const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);

281

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

282

const int16x8_t vprod1x89ABc1 = vmull_s8(vb89ABc1, va1c1);

283

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc1);

284

const int16x8_t vprod1xCDEFc1 = vmull_s8(vbCDEFc1, va1c1);

285

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc1);

286

const int8x8_t va2c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 1));

287

const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);

288

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);

289

const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);

290

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);

291

const int16x8_t vprod2x89ABc1 = vmull_s8(vb89ABc1, va2c1);

292

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc1);

293

const int16x8_t vprod2xCDEFc1 = vmull_s8(vbCDEFc1, va2c1);

294

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc1);

295

296

if (k > 4 * sizeof(int8_t)) {

297

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

298

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

299

const int8x8_t vb89ABc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

300

const int8x8_t vbCDEFc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

301

302

const int8x8_t va0c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2));

303

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);

304

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

305

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);

306

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

307

const int16x8_t vprod0x89ABc2 = vmull_s8(vb89ABc2, va0c2);

308

vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);

309

const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, va0c2);

310

vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);

311

const int8x8_t va1c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2));

312

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);

313

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

314

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);

315

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

316

const int16x8_t vprod1x89ABc2 = vmull_s8(vb89ABc2, va1c2);

317

vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);

318

const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, va1c2);

319

vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);

320

const int8x8_t va2c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2));

321

const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);

322

vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);

323

const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);

324

vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);

325

const int16x8_t vprod2x89ABc2 = vmull_s8(vb89ABc2, va2c2);

326

vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);

327

const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, va2c2);

328

vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);

}

}

}

p -= 3 * sizeof(void*);

333

} while (p != 0);

334

335

const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);

336

const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);

337

const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);

338

339

vacc0x0123 = vshlq_s32(vacc0x0123, vright_pre_shift);

340

vacc0x4567 = vshlq_s32(vacc0x4567, vright_pre_shift);

341

vacc0x89AB = vshlq_s32(vacc0x89AB, vright_pre_shift);

342

vacc0xCDEF = vshlq_s32(vacc0xCDEF, vright_pre_shift);

343

vacc1x0123 = vshlq_s32(vacc1x0123, vright_pre_shift);

344

vacc1x4567 = vshlq_s32(vacc1x4567, vright_pre_shift);

345

vacc1x89AB = vshlq_s32(vacc1x89AB, vright_pre_shift);

346

vacc1xCDEF = vshlq_s32(vacc1xCDEF, vright_pre_shift);

347

vacc2x0123 = vshlq_s32(vacc2x0123, vright_pre_shift);

348

vacc2x4567 = vshlq_s32(vacc2x4567, vright_pre_shift);

349

vacc2x89AB = vshlq_s32(vacc2x89AB, vright_pre_shift);

350

vacc2xCDEF = vshlq_s32(vacc2xCDEF, vright_pre_shift);

351

352

vacc0x0123 = vqdmulhq_s32(vacc0x0123, vmultiplier);

353

vacc0x4567 = vqdmulhq_s32(vacc0x4567, vmultiplier);

354

vacc0x89AB = vqdmulhq_s32(vacc0x89AB, vmultiplier);

355

vacc0xCDEF = vqdmulhq_s32(vacc0xCDEF, vmultiplier);

356

vacc1x0123 = vqdmulhq_s32(vacc1x0123, vmultiplier);

357

vacc1x4567 = vqdmulhq_s32(vacc1x4567, vmultiplier);

358

vacc1x89AB = vqdmulhq_s32(vacc1x89AB, vmultiplier);

359

vacc1xCDEF = vqdmulhq_s32(vacc1xCDEF, vmultiplier);

360

vacc2x0123 = vqdmulhq_s32(vacc2x0123, vmultiplier);

361

vacc2x4567 = vqdmulhq_s32(vacc2x4567, vmultiplier);

362

vacc2x89AB = vqdmulhq_s32(vacc2x89AB, vmultiplier);

363

vacc2xCDEF = vqdmulhq_s32(vacc2xCDEF, vmultiplier);

364

365

vacc0x0123 = vrshlq_s32(vacc0x0123, vright_post_shift);

366

vacc0x4567 = vrshlq_s32(vacc0x4567, vright_post_shift);

367

vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_post_shift);

368

vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_post_shift);

369

vacc1x0123 = vrshlq_s32(vacc1x0123, vright_post_shift);

370

vacc1x4567 = vrshlq_s32(vacc1x4567, vright_post_shift);

371

vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_post_shift);

372

vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_post_shift);

373

vacc2x0123 = vrshlq_s32(vacc2x0123, vright_post_shift);

374

vacc2x4567 = vrshlq_s32(vacc2x4567, vright_post_shift);

375

vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_post_shift);

376

vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_post_shift);

377

378

const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);

379

#if XNN_ARCH_ARM64

380

const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);

381

const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);

382

const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);

383

const int16x8_t vacc1x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x89AB), vacc1xCDEF), voutput_zero_point);

384

const int16x8_t vacc2x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567), voutput_zero_point);

385

const int16x8_t vacc2x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x89AB), vacc2xCDEF), voutput_zero_point);

386

387

int8x16_t vout0x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc0x89ABCDEF);

388

int8x16_t vout1x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc1x01234567), vacc1x89ABCDEF);

389

int8x16_t vout2x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc2x89ABCDEF);

390

#else

391

const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point);

392

const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x89AB), vqmovn_s32(vacc0xCDEF)), voutput_zero_point);

393

const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point);

394

const int16x8_t vacc1x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x89AB), vqmovn_s32(vacc1xCDEF)), voutput_zero_point);

395

const int16x8_t vacc2x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567)), voutput_zero_point);

396

const int16x8_t vacc2x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x89AB), vqmovn_s32(vacc2xCDEF)), voutput_zero_point);

397

398

int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));

399

int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));

400

int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));

401

#endif

402

const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);

403

const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);

404

405

vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);

406

vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);

407

vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);

408

409

vout2x0123456789ABCDEF = vminq_s8(vout2x0123456789ABCDEF, voutput_max);

410

vout1x0123456789ABCDEF = vminq_s8(vout1x0123456789ABCDEF, voutput_max);

411

vout0x0123456789ABCDEF = vminq_s8(vout0x0123456789ABCDEF, voutput_max);

412

413

if (nc >= 16) {

414

vst1q_s8(c2 + 0, vout2x0123456789ABCDEF);

415

vst1q_s8(c1 + 0, vout1x0123456789ABCDEF);

416

vst1q_s8(c0 + 0, vout0x0123456789ABCDEF);

417

418

c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);

419

c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);

420

c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);

421

422

a = (const int8_t**restrict) ((uintptr_t) a - ks);

nc -= 16;

} else {

int8x8_t vout2x01234567 = vget_low_s8(vout2x0123456789ABCDEF);

427

int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vget_low_s8(vout0x0123456789ABCDEF), vget_low_s8(vout1x0123456789ABCDEF));

428

if (nc & 8) {

429

vst1_s8(c2, vout2x01234567); c2 += 8;

430

vst1_s8(c1, vget_high_s8(vout0x01234567_1x01234567)); c1 += 8;

431

vst1_s8(c0, vget_low_s8(vout0x01234567_1x01234567)); c0 += 8;

432

vout2x01234567 = vget_high_s8(vout2x0123456789ABCDEF);

433

vout0x01234567_1x01234567 = vcombine_s8(vget_high_s8(vout0x0123456789ABCDEF), vget_high_s8(vout1x0123456789ABCDEF));

434

}

435

if (nc & 4) {

436

vst1_lane_u32(__builtin_assume_aligned(c2, 1), vreinterpret_u32_s8(vout2x01234567), 0); c2 += 4;

437

vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;

438

vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;

439

vout2x01234567 = vext_s8(vout2x01234567, vout2x01234567, 4);

440

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);

441

}

442

if (nc & 2) {

443

vst1_lane_u16(__builtin_assume_aligned(c2, 1), vreinterpret_u16_s8(vout2x01234567), 0); c2 += 2;

444

vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;

445

vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;

446

vout2x01234567 = vext_s8(vout2x01234567, vout2x01234567, 2);

447

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);

448

}

449

if (nc & 1) {

450

vst1_lane_s8(c2, vout2x01234567, 0);

451

vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);

452

vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);

}

nc = 0;

}

} while (nc != 0);

}