Blame - src/qs8-igemm/gen/2x8s4c2-minmax-rndnu-neon-mlal-padal.c - platform/external/XNNPACK

Frank Barchard

494cd2b

2021-10-28 17:36:37 -0700

[diff] [blame^]

1

// Auto-generated file. Do not edit!

2

// Template: src/qs8-igemm/c2-neon-mull-padal-shuffle.c.in

3

// Generator: tools/xngen

//

//

// This source code is licensed under the BSD-style license found in the

8

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <arm_neon.h>

13

14

#include <xnnpack/gemm.h>

15

#include <xnnpack/math.h>

16

17

18

void xnn_qs8_igemm_minmax_rndnu_ukernel_2x8s4c2__neon_mlal_padal(

size_t mr,

size_t nc,

size_t kc,

size_t ks,

const int8_t** restrict a,

24

const void* restrict w,

int8_t* restrict c,

size_t cm_stride,

size_t cn_stride,

size_t a_offset,

const int8_t* zero,

const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN

{

assert(mr != 0);

assert(mr <= 2);

assert(nc != 0);

assert(kc != 0);

assert(ks != 0);

assert(ks % (2 * sizeof(void*)) == 0);

38

assert(a_offset % sizeof(int8_t) == 0);

assert(a != NULL);

assert(w != NULL);

assert(c != NULL);

int8_t* c0 = c;

int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);

45

if XNN_UNPREDICTABLE(mr != 2) {

c1 = c0;

}

do {

int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

51

int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));

52

int32x4_t vacc1x0123 = vacc0x0123;

53

int32x4_t vacc1x4567 = vacc0x4567;

size_t p = ks;

do {

const int8_t* restrict a0 = a[0];

58

if XNN_UNPREDICTABLE(a0 != zero) {

59

a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);

60

}

61

const int8_t* restrict a1 = a[1];

62

if XNN_UNPREDICTABLE(a1 != zero) {

63

a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);

}

a += 2;

size_t k = kc;

while (k >= 16 * sizeof(int8_t)) {

70

int8x8_t va0x0 = vld1_s8(a0); a0 += 8;

71

int8x8_t va0x1 = vld1_s8(a0); a0 += 8;

72

int8x8_t va1x0 = vld1_s8(a1); a1 += 8;

73

int8x8_t va1x1 = vld1_s8(a1); a1 += 8;

74

75

const int8x8_t vb0123c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

76

const int8x8_t vb4567c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

77

const int8x8_t vb0123c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

78

const int8x8_t vb4567c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

79

const int8x8_t vb0123c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

80

const int8x8_t vb4567c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

81

const int8x8_t vb0123c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

82

const int8x8_t vb4567c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

83

84

int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0x0, va0x0);

85

int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0x0, va1x0);

86

const int8x8_t vb0123c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

87

vprod0x0123c0 = vmlal_s8(vprod0x0123c0, vb0123c0x1, va0x1);

88

vprod1x0123c0 = vmlal_s8(vprod1x0123c0, vb0123c0x1, va1x1);

89

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

90

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

91

int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0x0, va0x0);

92

int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0x0, va1x0);

93

const int8x8_t vb4567c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

94

vprod0x4567c0 = vmlal_s8(vprod0x4567c0, vb4567c0x1, va0x1);

95

vprod1x4567c0 = vmlal_s8(vprod1x4567c0, vb4567c0x1, va1x1);

96

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

97

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

98

va0x0 = vext_s8(va0x0, va0x0, 2);

99

va0x1 = vext_s8(va0x1, va0x1, 2);

100

va1x0 = vext_s8(va1x0, va1x0, 2);

101

va1x1 = vext_s8(va1x1, va1x1, 2);

102

int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1x0, va0x0);

103

int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1x0, va1x0);

104

const int8x8_t vb0123c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

105

vprod0x0123c1 = vmlal_s8(vprod0x0123c1, vb0123c1x1, va0x1);

106

vprod1x0123c1 = vmlal_s8(vprod1x0123c1, vb0123c1x1, va1x1);

107

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

108

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

109

int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1x0, va0x0);

110

int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1x0, va1x0);

111

const int8x8_t vb4567c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

112

vprod0x4567c1 = vmlal_s8(vprod0x4567c1, vb4567c1x1, va0x1);

113

vprod1x4567c1 = vmlal_s8(vprod1x4567c1, vb4567c1x1, va1x1);

114

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

115

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

116

va0x0 = vext_s8(va0x0, va0x0, 2);

117

va0x1 = vext_s8(va0x1, va0x1, 2);

118

va1x0 = vext_s8(va1x0, va1x0, 2);

119

va1x1 = vext_s8(va1x1, va1x1, 2);

120

int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2x0, va0x0);

121

int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2x0, va1x0);

122

const int8x8_t vb0123c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

123

vprod0x0123c2 = vmlal_s8(vprod0x0123c2, vb0123c2x1, va0x1);

124

vprod1x0123c2 = vmlal_s8(vprod1x0123c2, vb0123c2x1, va1x1);

125

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

126

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

127

int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2x0, va0x0);

128

int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2x0, va1x0);

129

const int8x8_t vb4567c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

130

vprod0x4567c2 = vmlal_s8(vprod0x4567c2, vb4567c2x1, va0x1);

131

vprod1x4567c2 = vmlal_s8(vprod1x4567c2, vb4567c2x1, va1x1);

132

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

133

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

134

va0x0 = vext_s8(va0x0, va0x0, 2);

135

va0x1 = vext_s8(va0x1, va0x1, 2);

136

va1x0 = vext_s8(va1x0, va1x0, 2);

137

va1x1 = vext_s8(va1x1, va1x1, 2);

138

int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3x0, va0x0);

139

int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3x0, va1x0);

140

const int8x8_t vb0123c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

141

vprod0x0123c3 = vmlal_s8(vprod0x0123c3, vb0123c3x1, va0x1);

142

vprod1x0123c3 = vmlal_s8(vprod1x0123c3, vb0123c3x1, va1x1);

143

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);

144

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);

145

int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3x0, va0x0);

146

int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3x0, va1x0);

147

const int8x8_t vb4567c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

148

vprod0x4567c3 = vmlal_s8(vprod0x4567c3, vb4567c3x1, va0x1);

149

vprod1x4567c3 = vmlal_s8(vprod1x4567c3, vb4567c3x1, va1x1);

150

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);

151

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);

152

153

k -= 16 * sizeof(int8_t);

154

}

155

156

if (k >= 8 * sizeof(int8_t)) {

157

int8x8_t va0x0 = vld1_s8(a0); a0 += 8;

158

int8x8_t va1x0 = vld1_s8(a1); a1 += 8;

159

160

const int8x8_t vb0123c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

161

const int8x8_t vb4567c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

162

const int8x8_t vb0123c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

163

const int8x8_t vb4567c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

164

const int8x8_t vb0123c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

165

const int8x8_t vb4567c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

166

const int8x8_t vb0123c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

167

const int8x8_t vb4567c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

168

169

int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0x0, va0x0);

170

int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0x0, va1x0);

171

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

172

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

173

int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0x0, va0x0);

174

int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0x0, va1x0);

175

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

176

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

177

va0x0 = vext_s8(va0x0, va0x0, 2);

178

va1x0 = vext_s8(va1x0, va1x0, 2);

179

int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1x0, va0x0);

180

int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1x0, va1x0);

181

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

182

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

183

int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1x0, va0x0);

184

int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1x0, va1x0);

185

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

186

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

187

va0x0 = vext_s8(va0x0, va0x0, 2);

188

va1x0 = vext_s8(va1x0, va1x0, 2);

189

int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2x0, va0x0);

190

int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2x0, va1x0);

191

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

192

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

193

int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2x0, va0x0);

194

int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2x0, va1x0);

195

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

196

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

197

va0x0 = vext_s8(va0x0, va0x0, 2);

198

va1x0 = vext_s8(va1x0, va1x0, 2);

199

int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3x0, va0x0);

200

int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3x0, va1x0);

201

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);

202

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);

203

int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3x0, va0x0);

204

int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3x0, va1x0);

205

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);

206

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);

207

208

k -= 8 * sizeof(int8_t);

209

}

210

211

if XNN_UNLIKELY(k != 0) {

212

const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);

213

const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);

214

215

const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

216

const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

217

218

const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0)));

219

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);

220

const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0)));

221

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);

222

const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0)));

223

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);

224

const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0)));

225

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);

226

227

if (k > 2 * sizeof(int8_t)) {

228

const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

229

const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

230

231

const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1)));

232

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);

233

const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1)));

234

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);

235

const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1)));

236

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);

237

const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1)));

238

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);

239

240

if (k > 4 * sizeof(int8_t)) {

241

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

242

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

243

244

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));

245

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

246

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2)));

247

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

248

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));

249

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

250

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2)));

251

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

252

253

if (k > 6 * sizeof(int8_t)) {

254

const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

255

const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));

256

257

const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));

258

vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);

259

const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3)));

260

vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);

261

const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));

262

vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);

263

const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3)));

264

vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);

}

}

}

}

p -= 2 * sizeof(void*);

270

} while (p != 0);

271

272

const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);

273

const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);

274

const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);

275

276

vacc0x0123 = vshlq_s32(vacc0x0123, vright_pre_shift);

277

vacc0x4567 = vshlq_s32(vacc0x4567, vright_pre_shift);

278

vacc1x0123 = vshlq_s32(vacc1x0123, vright_pre_shift);

279

vacc1x4567 = vshlq_s32(vacc1x4567, vright_pre_shift);

280

281

vacc0x0123 = vqdmulhq_s32(vacc0x0123, vmultiplier);

282

vacc0x4567 = vqdmulhq_s32(vacc0x4567, vmultiplier);

283

vacc1x0123 = vqdmulhq_s32(vacc1x0123, vmultiplier);

284

vacc1x4567 = vqdmulhq_s32(vacc1x4567, vmultiplier);

285

286

vacc0x0123 = vrshlq_s32(vacc0x0123, vright_post_shift);

287

vacc0x4567 = vrshlq_s32(vacc0x4567, vright_post_shift);

288

vacc1x0123 = vrshlq_s32(vacc1x0123, vright_post_shift);

289

vacc1x4567 = vrshlq_s32(vacc1x4567, vright_post_shift);

290

291

const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);

292

#if XNN_ARCH_ARM64

293

const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);

294

const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);

295

296

int8x16_t vout0x01234567_1x01234567 = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc1x01234567);

297

#else

298

const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point);

299

const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point);

300

301

int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));

302

#endif

303

const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);

304

const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);

305

306

vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);

307

308

vout0x01234567_1x01234567 = vminq_s8(vout0x01234567_1x01234567, voutput_max);

309

310

if (nc >= 8) {

311

vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567));

312

vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567));

313

314

c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);

315

c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);

316

317

a = (const int8_t**restrict) ((uintptr_t) a - ks);

nc -= 8;

} else {

if (nc & 4) {

vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;

323

vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;

324

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);

325

}

326

if (nc & 2) {

327

vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;

328

vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;

329

vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);

330

}

331

if (nc & 1) {

332

vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);

333

vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);

}

nc = 0;

}

} while (nc != 0);

}