Blame - src/qs8-igemm/gen/2x8c2-minmax-fp32-neon-mlal-ld4r.c - platform/external/XNNPACK

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

1

// Auto-generated file. Do not edit!

2

// Template: src/qs8-igemm/c2-neon-mull-dup.c.in

3

// Generator: tools/xngen

//

//

// This source code is licensed under the BSD-style license found in the

8

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <arm_neon.h>

13

14

#include <xnnpack/gemm.h>

15

#include <xnnpack/math.h>

16

17

void xnn_qs8_igemm_minmax_fp32_ukernel_2x8c2__neon_mlal_ld4r(

size_t mr,

size_t nc,

size_t kc,

size_t ks,

const int8_t** restrict a,

23

const void* restrict w,

int8_t* restrict c,

size_t cm_stride,

size_t cn_stride,

size_t a_offset,

const int8_t* zero,

const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN

{

assert(mr != 0);

assert(mr <= 2);

assert(nc != 0);

assert(kc != 0);

assert(ks != 0);

assert(ks % (2 * sizeof(void*)) == 0);

37

assert(a_offset % sizeof(int8_t) == 0);

assert(a != NULL);

assert(w != NULL);

assert(c != NULL);

kc = round_up_po2(kc, 2 * sizeof(int8_t));

43

int8_t* c0 = c;

44

int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);

45

if XNN_UNPREDICTABLE(mr != 2) {

c1 = c0;

}

do {

int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

51

int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

52

int32x4_t vacc1x0123 = vacc0x0123;

53

int32x4_t vacc1x4567 = vacc0x4567;

size_t p = ks;

do {

const int8_t* restrict a0 = a[0];

58

if XNN_UNPREDICTABLE(a0 != zero) {

59

a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);

60

}

61

const int8_t* restrict a1 = a[1];

62

if XNN_UNPREDICTABLE(a1 != zero) {

63

a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);

}

a += 2;

size_t k = kc;

while (k >= 16 * sizeof(int8_t)) {

70

const int16x4x4_t va0x0 = vld4_dup_s16((const void*)a0); a0 += 8;

71

const int16x4x4_t va0x1 = vld4_dup_s16((const void*)a0); a0 += 8;

72

const int16x4x4_t va1x0 = vld4_dup_s16((const void*)a1); a1 += 8;

73

const int16x4x4_t va1x1 = vld4_dup_s16((const void*)a1); a1 += 8;

74

75

const int8x8_t vb0123c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

76

const int8x8_t vb4567c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

77

const int8x8_t vb0123c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

78

const int8x8_t vb4567c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

79

const int8x8_t vb0123c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

80

const int8x8_t vb4567c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

81

const int8x8_t vb0123c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

82

const int8x8_t vb4567c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

83

84

const int8x8_t va0c0x0 = vreinterpret_s8_s16(va0x0.val[0]);

85

const int8x8_t va0c0x1 = vreinterpret_s8_s16(va0x1.val[0]);

86

const int8x8_t va1c0x0 = vreinterpret_s8_s16(va1x0.val[0]);

87

const int8x8_t va1c0x1 = vreinterpret_s8_s16(va1x1.val[0]);

Frank Barchard

15eec02

2021-11-17 13:26:20 -0800

[diff] [blame]

88

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

89

int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0x0, va0c0x0);

90

int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0x0, va1c0x0);

91

const int8x8_t vb0123c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

92

vprod0x0123c0 = vmlal_s8(vprod0x0123c0, vb0123c0x1, va0c0x1);

93

vprod1x0123c0 = vmlal_s8(vprod1x0123c0, vb0123c0x1, va1c0x1);

94

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

95

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

96

int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0x0, va0c0x0);

97

int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0x0, va1c0x0);

98

const int8x8_t vb4567c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

99

vprod0x4567c0 = vmlal_s8(vprod0x4567c0, vb4567c0x1, va0c0x1);

100

vprod1x4567c0 = vmlal_s8(vprod1x4567c0, vb4567c0x1, va1c0x1);

101

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

102

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

103

const int8x8_t va0c1x0 = vreinterpret_s8_s16(va0x0.val[1]);

104

const int8x8_t va0c1x1 = vreinterpret_s8_s16(va0x1.val[1]);

105

const int8x8_t va1c1x0 = vreinterpret_s8_s16(va1x0.val[1]);

106

const int8x8_t va1c1x1 = vreinterpret_s8_s16(va1x1.val[1]);

Frank Barchard

15eec02

2021-11-17 13:26:20 -0800

[diff] [blame]

107

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

108

int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1x0, va0c1x0);

109

int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1x0, va1c1x0);

110

const int8x8_t vb0123c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

111

vprod0x0123c1 = vmlal_s8(vprod0x0123c1, vb0123c1x1, va0c1x1);

112

vprod1x0123c1 = vmlal_s8(vprod1x0123c1, vb0123c1x1, va1c1x1);

113

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

114

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

115

int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1x0, va0c1x0);

116

int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1x0, va1c1x0);

117

const int8x8_t vb4567c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

118

vprod0x4567c1 = vmlal_s8(vprod0x4567c1, vb4567c1x1, va0c1x1);

119

vprod1x4567c1 = vmlal_s8(vprod1x4567c1, vb4567c1x1, va1c1x1);

120

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

121

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

122

const int8x8_t va0c2x0 = vreinterpret_s8_s16(va0x0.val[2]);

123

const int8x8_t va0c2x1 = vreinterpret_s8_s16(va0x1.val[2]);

124

const int8x8_t va1c2x0 = vreinterpret_s8_s16(va1x0.val[2]);

125

const int8x8_t va1c2x1 = vreinterpret_s8_s16(va1x1.val[2]);

Frank Barchard

15eec02

2021-11-17 13:26:20 -0800

[diff] [blame]

126

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

127

int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2x0, va0c2x0);

128

int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2x0, va1c2x0);

129

const int8x8_t vb0123c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

130

vprod0x0123c2 = vmlal_s8(vprod0x0123c2, vb0123c2x1, va0c2x1);

131

vprod1x0123c2 = vmlal_s8(vprod1x0123c2, vb0123c2x1, va1c2x1);

132

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

133

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

134

int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2x0, va0c2x0);

135

int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2x0, va1c2x0);

136

const int8x8_t vb4567c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

137

vprod0x4567c2 = vmlal_s8(vprod0x4567c2, vb4567c2x1, va0c2x1);

138

vprod1x4567c2 = vmlal_s8(vprod1x4567c2, vb4567c2x1, va1c2x1);

139

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

140

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

141

const int8x8_t va0c3x0 = vreinterpret_s8_s16(va0x0.val[3]);

142

const int8x8_t va0c3x1 = vreinterpret_s8_s16(va0x1.val[3]);

143

const int8x8_t va1c3x0 = vreinterpret_s8_s16(va1x0.val[3]);

144

const int8x8_t va1c3x1 = vreinterpret_s8_s16(va1x1.val[3]);

Frank Barchard

15eec02

2021-11-17 13:26:20 -0800

[diff] [blame]

145

Frank Barchard

42f5c50

2021-11-16 10:04:21 -0800

[diff] [blame]

146

int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3x0, va0c3x0);

147

int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3x0, va1c3x0);

148

const int8x8_t vb0123c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

149

vprod0x0123c3 = vmlal_s8(vprod0x0123c3, vb0123c3x1, va0c3x1);

150

vprod1x0123c3 = vmlal_s8(vprod1x0123c3, vb0123c3x1, va1c3x1);

151

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);

152

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);

153

int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3x0, va0c3x0);

154

int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3x0, va1c3x0);

155

const int8x8_t vb4567c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

156

vprod0x4567c3 = vmlal_s8(vprod0x4567c3, vb4567c3x1, va0c3x1);

157

vprod1x4567c3 = vmlal_s8(vprod1x4567c3, vb4567c3x1, va1c3x1);

158

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);

159

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);

160

161

k -= 16 * sizeof(int8_t);

162

}

163

164

if (k >= 8 * sizeof(int8_t)) {

165

const int16x4x4_t va0 = vld4_dup_s16((const void*)a0); a0 += 8;

166

const int16x4x4_t va1 = vld4_dup_s16((const void*)a1); a1 += 8;

167

168

const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

169

const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

170

const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

171

const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

172

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

173

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

174

const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

175

const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

176

177

const int8x8_t va0c0 = vreinterpret_s8_s16(va0.val[0]);

178

const int8x8_t va1c0 = vreinterpret_s8_s16(va1.val[0]);

179

180

const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);

181

const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);

182

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

183

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

184

const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);

185

const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);

186

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

187

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

188

const int8x8_t va0c1 = vreinterpret_s8_s16(va0.val[1]);

189

const int8x8_t va1c1 = vreinterpret_s8_s16(va1.val[1]);

190

191

const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);

192

const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);

193

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

194

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

195

const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);

196

const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);

197

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

198

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

199

const int8x8_t va0c2 = vreinterpret_s8_s16(va0.val[2]);

200

const int8x8_t va1c2 = vreinterpret_s8_s16(va1.val[2]);

201

202

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);

203

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);

204

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

205

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

206

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);

207

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);

208

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

209

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

210

const int8x8_t va0c3 = vreinterpret_s8_s16(va0.val[3]);

211

const int8x8_t va1c3 = vreinterpret_s8_s16(va1.val[3]);

212

213

const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, va0c3);

214

const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, va1c3);

215

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);

216

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);

217

const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, va0c3);

218

const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, va1c3);

219

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);

220

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);

221

222

k -= 8 * sizeof(int8_t);

223

}

224

225

if XNN_UNLIKELY(k != 0) {

226

const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);

227

const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);

228

229

const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

230

const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

231

232

const int8x8_t va0c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0));

233

const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);

234

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

235

const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);

236

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

237

const int8x8_t va1c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0));

238

const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);

239

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

240

const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);

241

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

242

243

if (k > 2 * sizeof(int8_t)) {

244

const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

245

const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

246

247

const int8x8_t va0c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1));

248

const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);

249

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

250

const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);

251

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

252

const int8x8_t va1c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1));

253

const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);

254

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

255

const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);

256

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

257

258

if (k > 4 * sizeof(int8_t)) {

259

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

260

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

261

262

const int8x8_t va0c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2));

263

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);

264

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

265

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);

266

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

267

const int8x8_t va1c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2));

268

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);

269

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

270

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);

271

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

}

}

}

p -= 2 * sizeof(void*);

276

} while (p != 0);

277

278

float32x4_t vfpacc0x0123 = vcvtq_f32_s32(vacc0x0123);

279

float32x4_t vfpacc0x4567 = vcvtq_f32_s32(vacc0x4567);

280

float32x4_t vfpacc1x0123 = vcvtq_f32_s32(vacc1x0123);

281

float32x4_t vfpacc1x4567 = vcvtq_f32_s32(vacc1x4567);

282

283

const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neon.scale);

284

vfpacc0x0123 = vmulq_f32(vfpacc0x0123, vscale);

285

vfpacc0x4567 = vmulq_f32(vfpacc0x4567, vscale);

286

vfpacc1x0123 = vmulq_f32(vfpacc1x0123, vscale);

287

vfpacc1x4567 = vmulq_f32(vfpacc1x4567, vscale);

288

289

const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(&params->fp32_neon.output_min_less_zero_point);

290

vfpacc0x0123 = vmaxq_f32(vfpacc0x0123, voutput_min_less_zero_point);

291

vfpacc0x4567 = vmaxq_f32(vfpacc0x4567, voutput_min_less_zero_point);

292

vfpacc1x0123 = vmaxq_f32(vfpacc1x0123, voutput_min_less_zero_point);

293

vfpacc1x4567 = vmaxq_f32(vfpacc1x4567, voutput_min_less_zero_point);

294

295

const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(&params->fp32_neon.output_max_less_zero_point);

296

vfpacc0x0123 = vminq_f32(vfpacc0x0123, voutput_max_less_zero_point);

297

vfpacc0x4567 = vminq_f32(vfpacc0x4567, voutput_max_less_zero_point);

298

vfpacc1x0123 = vminq_f32(vfpacc1x0123, voutput_max_less_zero_point);

299

vfpacc1x4567 = vminq_f32(vfpacc1x4567, voutput_max_less_zero_point);

300

301

const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);

302

vacc0x0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0x0123, vmagic_bias));

303

vacc0x4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0x4567, vmagic_bias));

304

vacc1x0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc1x0123, vmagic_bias));

305

vacc1x4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc1x4567, vmagic_bias));

306

307

const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_zero_point);

308

vacc0x0123 = vsubq_s32(vacc0x0123, vmagic_bias_less_zero_point);

309

vacc0x4567 = vsubq_s32(vacc0x4567, vmagic_bias_less_zero_point);

310

vacc1x0123 = vsubq_s32(vacc1x0123, vmagic_bias_less_zero_point);

311

vacc1x4567 = vsubq_s32(vacc1x4567, vmagic_bias_less_zero_point);

312

313

#if XNN_ARCH_ARM64

314

const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));

315

const int16x8_t vacc1x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x0123), vreinterpretq_s16_s32(vacc1x4567));

316

317

int8x16_t vout0x01234567_1x01234567 = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc1x01234567));

318

#else

319

const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));

320

const int16x8_t vacc1x01234567 = vcombine_s16(vmovn_s32(vacc1x0123), vmovn_s32(vacc1x4567));

321

322

int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vmovn_s16(vacc0x01234567), vmovn_s16(vacc1x01234567));

#endif

if (nc >= 8) {

vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567));

327

vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567));

328

329

c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);

330

c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);

331

332

a = (const int8_t**restrict) ((uintptr_t) a - ks);

nc -= 8;

} else {

if (nc & 4) {

vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;

338

vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;

339

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);

340

}

341

if (nc & 2) {

342

vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;

343

vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;

344

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);

345

}

346

if (nc & 1) {

347

vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);

348

vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);

}

nc = 0;

}

} while (nc != 0);

}