Blame - src/f32-conv-hwc/gen/3x3s2p0p1c3x8-neon-2x2.c - platform/external/XNNPACK

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

1

// Auto-generated file. Do not edit!

2

// Template: src/f32-conv-hwc/3x3s2p0p1c3-neon-x2.c.in

3

// Generator: tools/xngen

//

//

// This source code is licensed under the BSD-style license found in the

8

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <arm_neon.h>

14

15

#include <xnnpack/conv.h>

16

#include <xnnpack/math.h>

17

18

19

void xnn_f32_conv_hwc_ukernel_3x3s2p0p1c3x8__neon_2x2(

20

size_t input_height,

21

size_t input_width,

22

size_t output_y_start,

size_t output_y_end,

const float* input,

const float* zero,

const float* weights,

27

float* output,

28

size_t input_padding_top,

29

size_t output_channels,

30

size_t output_height_stride,

31

size_t output_width_stride,

32

const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])

33

{

34

assert(input_width != 0);

35

assert(output_y_end > output_y_start);

36

assert(input_padding_top <= 1);

37

assert(output_channels != 0);

38

39

const size_t input_height_stride = input_width * 3 /* channels */ * sizeof(float);

40

const size_t input_width_decrement = (4 + ((input_width - 1) & 1) * 2 + (round_down_po2(input_width - 1, 2) * 3 /* channels */)) * sizeof(float);

41

const size_t output_width = input_width / 2;

42

const size_t output_channel_decrement = output_width * output_width_stride - 8 * sizeof(float);

43

const size_t output_height_increment = output_height_stride * 2 - round_up_po2(output_channels, 8) * sizeof(float);

44

45

// Adjustment for padding processed below

46

const float* i0 = (const float*) ((uintptr_t) input +

47

input_height_stride * (output_y_start * 2 /* vertical stride */ - input_padding_top));

48

const float* i1 = (const float*) ((uintptr_t) i0 + input_height_stride);

49

const float* i2 = (const float*) ((uintptr_t) i1 + input_height_stride);

50

const float* i3 = (const float*) ((uintptr_t) i2 + input_height_stride);

51

const float* i4 = (const float*) ((uintptr_t) i3 + input_height_stride);

52

float* o0 = (float*) ((uintptr_t) output + output_height_stride * output_y_start);

53

float* o1 = (float*) ((uintptr_t) o0 + output_height_stride);

54

55

if XNN_UNPREDICTABLE(output_y_start < input_padding_top) {

i0 = zero;

}

Marat Dukhan

2020-05-17 21:46:44 -0700

[diff] [blame]

59

60

for (size_t output_y = output_y_start; output_y < output_y_end; output_y += 2) {

61

const size_t input_y2 = output_y * 2 + 2 - input_padding_top;

62

const size_t input_y4 = input_y2 + 2;

63

if XNN_UNPREDICTABLE(input_y2 > input_height) {

64

i1 = zero;

65

}

66

if XNN_UNPREDICTABLE(input_y2 >= input_height) {

67

i2 = zero;

68

}

69

if XNN_UNPREDICTABLE(input_y4 > input_height) {

70

i3 = zero;

71

}

72

if XNN_UNPREDICTABLE(input_y4 >= input_height) {

73

i4 = zero;

74

}

75

if XNN_UNPREDICTABLE(output_y + 2 > output_y_end) {

o1 = o0;

}

const float* w = weights;

80

size_t c = output_channels;

81

do {

82

// viMx0 = ( iM1c0, iM0c2, iM0c1, iM0c0 )

83

float32x4_t vi0x0 = vld1q_f32(i0); i0 += 4;

84

float32x4_t vi1x0 = vld1q_f32(i1); i1 += 4;

85

float32x4_t vi2x0 = vld1q_f32(i2); i2 += 4;

86

float32x4_t vi3x0 = vld1q_f32(i3); i3 += 4;

87

float32x4_t vi4x0 = vld1q_f32(i4); i4 += 4;

88

89

size_t iw = input_width - 1;

90

for (; iw >= 4; iw -= 4) {

91

float32x4_t vo0x0c0123 = vld1q_f32(w);

92

float32x4_t vo0x0c4567 = vld1q_f32(w + 4);

93

float32x4_t vo1x0c0123 = vo0x0c0123;

94

float32x4_t vo1x0c4567 = vo0x0c4567;

95

float32x4_t vo0x1c0123 = vo0x0c0123;

96

float32x4_t vo0x1c4567 = vo0x0c4567;

97

float32x4_t vo1x1c0123 = vo0x0c0123;

98

float32x4_t vo1x1c4567 = vo0x0c4567;

99

100

const float32x4_t vk00c0x0123 = vld1q_f32(w + 8);

101

const float32x4_t vk00c0x4567 = vld1q_f32(w + 12);

102

103

// viMx1 = ( iM2c1, iM2c0, iM1c2, iM1c1 )

104

const float32x4_t vi0x1 = vld1q_f32(i0); i0 += 4;

105

const float32x4_t vi1x1 = vld1q_f32(i1); i1 += 4;

106

const float32x4_t vi2x1 = vld1q_f32(i2); i2 += 4;

107

const float32x4_t vi3x1 = vld1q_f32(i3); i3 += 4;

108

const float32x4_t vi4x1 = vld1q_f32(i4); i4 += 4;

109

110

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk00c0x0123, vget_low_f32(vi0x0), 0);

111

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk00c0x0123, vget_low_f32(vi2x0), 0);

112

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk00c0x4567, vget_low_f32(vi0x0), 0);

113

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk00c0x4567, vget_low_f32(vi2x0), 0);

114

115

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk00c0x0123, vget_high_f32(vi0x1), 0);

116

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk00c0x0123, vget_high_f32(vi2x1), 0);

117

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk00c0x4567, vget_high_f32(vi0x1), 0);

118

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk00c0x4567, vget_high_f32(vi2x1), 0);

119

120

const float32x4_t vk10c0x0123 = vld1q_f32(w + 16);

121

const float32x4_t vk10c0x4567 = vld1q_f32(w + 20);

122

123

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk10c0x0123, vget_low_f32(vi1x0), 0);

124

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk10c0x0123, vget_low_f32(vi3x0), 0);

125

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk10c0x4567, vget_low_f32(vi1x0), 0);

126

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk10c0x4567, vget_low_f32(vi3x0), 0);

127

128

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk10c0x0123, vget_high_f32(vi1x1), 0);

129

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk10c0x0123, vget_high_f32(vi3x1), 0);

130

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk10c0x4567, vget_high_f32(vi1x1), 0);

131

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk10c0x4567, vget_high_f32(vi3x1), 0);

132

133

const float32x4_t vk20c0x0123 = vld1q_f32(w + 24);

134

const float32x4_t vk20c0x4567 = vld1q_f32(w + 28);

135

136

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk20c0x0123, vget_low_f32(vi2x0), 0);

137

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk20c0x0123, vget_low_f32(vi4x0), 0);

138

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk20c0x4567, vget_low_f32(vi2x0), 0);

139

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk20c0x4567, vget_low_f32(vi4x0), 0);

140

141

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk20c0x0123, vget_high_f32(vi2x1), 0);

142

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk20c0x0123, vget_high_f32(vi4x1), 0);

143

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk20c0x4567, vget_high_f32(vi2x1), 0);

144

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk20c0x4567, vget_high_f32(vi4x1), 0);

145

146

const float32x4_t vk00c1x0123 = vld1q_f32(w + 32);

147

const float32x4_t vk00c1x4567 = vld1q_f32(w + 36);

148

149

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk00c1x0123, vget_low_f32(vi0x0), 1);

150

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk00c1x0123, vget_low_f32(vi2x0), 1);

151

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk00c1x4567, vget_low_f32(vi0x0), 1);

152

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk00c1x4567, vget_low_f32(vi2x0), 1);

153

154

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk00c1x0123, vget_high_f32(vi0x1), 1);

155

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk00c1x0123, vget_high_f32(vi2x1), 1);

156

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk00c1x4567, vget_high_f32(vi0x1), 1);

157

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk00c1x4567, vget_high_f32(vi2x1), 1);

158

159

const float32x4_t vk10c1x0123 = vld1q_f32(w + 40);

160

const float32x4_t vk10c1x4567 = vld1q_f32(w + 44);

161

162

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk10c1x0123, vget_low_f32(vi1x0), 1);

163

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk10c1x0123, vget_low_f32(vi3x0), 1);

164

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk10c1x4567, vget_low_f32(vi1x0), 1);

165

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk10c1x4567, vget_low_f32(vi3x0), 1);

166

167

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk10c1x0123, vget_high_f32(vi1x1), 1);

168

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk10c1x0123, vget_high_f32(vi3x1), 1);

169

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk10c1x4567, vget_high_f32(vi1x1), 1);

170

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk10c1x4567, vget_high_f32(vi3x1), 1);

171

172

const float32x4_t vk20c1x0123 = vld1q_f32(w + 48);

173

const float32x4_t vk20c1x4567 = vld1q_f32(w + 52);

174

175

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk20c1x0123, vget_low_f32(vi2x0), 1);

176

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk20c1x0123, vget_low_f32(vi4x0), 1);

177

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk20c1x4567, vget_low_f32(vi2x0), 1);

178

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk20c1x4567, vget_low_f32(vi4x0), 1);

179

180

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk20c1x0123, vget_high_f32(vi2x1), 1);

181

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk20c1x0123, vget_high_f32(vi4x1), 1);

182

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk20c1x4567, vget_high_f32(vi2x1), 1);

183

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk20c1x4567, vget_high_f32(vi4x1), 1);

184

185

const float32x4_t vk00c2x0123 = vld1q_f32(w + 56);

186

const float32x4_t vk00c2x4567 = vld1q_f32(w + 60);

187

188

// viMx2 = ( iM3c2, iM3c1, iM3c0, iM2c2 )

189

const float32x4_t vi0x2 = vld1q_f32(i0); i0 += 4;

190

const float32x4_t vi1x2 = vld1q_f32(i1); i1 += 4;

191

const float32x4_t vi2x2 = vld1q_f32(i2); i2 += 4;

192

const float32x4_t vi3x2 = vld1q_f32(i3); i3 += 4;

193

const float32x4_t vi4x2 = vld1q_f32(i4); i4 += 4;

194

195

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk00c2x0123, vget_high_f32(vi0x0), 0);

196

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk00c2x0123, vget_high_f32(vi2x0), 0);

197

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk00c2x4567, vget_high_f32(vi0x0), 0);

198

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk00c2x4567, vget_high_f32(vi2x0), 0);

199

200

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk00c2x0123, vget_low_f32(vi0x2), 0);

201

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk00c2x0123, vget_low_f32(vi2x2), 0);

202

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk00c2x4567, vget_low_f32(vi0x2), 0);

203

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk00c2x4567, vget_low_f32(vi2x2), 0);

204

205

const float32x4_t vk10c2x0123 = vld1q_f32(w + 64);

206

const float32x4_t vk10c2x4567 = vld1q_f32(w + 68);

207

208

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk10c2x0123, vget_high_f32(vi1x0), 0);

209

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk10c2x0123, vget_high_f32(vi3x0), 0);

210

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk10c2x4567, vget_high_f32(vi1x0), 0);

211

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk10c2x4567, vget_high_f32(vi3x0), 0);

212

213

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk10c2x0123, vget_low_f32(vi1x2), 0);

214

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk10c2x0123, vget_low_f32(vi3x2), 0);

215

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk10c2x4567, vget_low_f32(vi1x2), 0);

216

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk10c2x4567, vget_low_f32(vi3x2), 0);

217

218

const float32x4_t vk20c2x0123 = vld1q_f32(w + 72);

219

const float32x4_t vk20c2x4567 = vld1q_f32(w + 76);

220

221

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk20c2x0123, vget_high_f32(vi2x0), 0);

222

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk20c2x0123, vget_high_f32(vi4x0), 0);

223

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk20c2x4567, vget_high_f32(vi2x0), 0);

224

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk20c2x4567, vget_high_f32(vi4x0), 0);

225

226

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk20c2x0123, vget_low_f32(vi2x2), 0);

227

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk20c2x0123, vget_low_f32(vi4x2), 0);

228

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk20c2x4567, vget_low_f32(vi2x2), 0);

229

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk20c2x4567, vget_low_f32(vi4x2), 0);

230

231

const float32x4_t vk01c0x0123 = vld1q_f32(w + 80);

232

const float32x4_t vk01c0x4567 = vld1q_f32(w + 84);

233

234

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk01c0x0123, vget_high_f32(vi0x0), 1);

235

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk01c0x0123, vget_high_f32(vi2x0), 1);

236

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk01c0x4567, vget_high_f32(vi0x0), 1);

237

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk01c0x4567, vget_high_f32(vi2x0), 1);

238

239

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk01c0x0123, vget_low_f32(vi0x2), 1);

240

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk01c0x0123, vget_low_f32(vi2x2), 1);

241

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk01c0x4567, vget_low_f32(vi0x2), 1);

242

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk01c0x4567, vget_low_f32(vi2x2), 1);

243

244

const float32x4_t vk11c0x0123 = vld1q_f32(w + 88);

245

const float32x4_t vk11c0x4567 = vld1q_f32(w + 92);

246

247

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk11c0x0123, vget_high_f32(vi1x0), 1);

248

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk11c0x0123, vget_high_f32(vi3x0), 1);

249

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk11c0x4567, vget_high_f32(vi1x0), 1);

250

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk11c0x4567, vget_high_f32(vi3x0), 1);

251

252

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk11c0x0123, vget_low_f32(vi1x2), 1);

253

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk11c0x0123, vget_low_f32(vi3x2), 1);

254

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk11c0x4567, vget_low_f32(vi1x2), 1);

255

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk11c0x4567, vget_low_f32(vi3x2), 1);

256

257

const float32x4_t vk21c0x0123 = vld1q_f32(w + 96);

258

const float32x4_t vk21c0x4567 = vld1q_f32(w + 100);

259

260

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk21c0x0123, vget_high_f32(vi2x0), 1);

261

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk21c0x0123, vget_high_f32(vi4x0), 1);

262

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk21c0x4567, vget_high_f32(vi2x0), 1);

263

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk21c0x4567, vget_high_f32(vi4x0), 1);

264

265

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk21c0x0123, vget_low_f32(vi2x2), 1);

266

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk21c0x0123, vget_low_f32(vi4x2), 1);

267

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk21c0x4567, vget_low_f32(vi2x2), 1);

268

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk21c0x4567, vget_low_f32(vi4x2), 1);

269

270

const float32x4_t vk01c1x0123 = vld1q_f32(w + 104);

271

const float32x4_t vk01c1x4567 = vld1q_f32(w + 108);

272

273

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk01c1x0123, vget_low_f32(vi0x1), 0);

274

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk01c1x0123, vget_low_f32(vi2x1), 0);

275

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk01c1x4567, vget_low_f32(vi0x1), 0);

276

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk01c1x4567, vget_low_f32(vi2x1), 0);

277

278

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk01c1x0123, vget_high_f32(vi0x2), 0);

279

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk01c1x0123, vget_high_f32(vi2x2), 0);

280

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk01c1x4567, vget_high_f32(vi0x2), 0);

281

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk01c1x4567, vget_high_f32(vi2x2), 0);

282

283

const float32x4_t vk11c1x0123 = vld1q_f32(w + 112);

284

const float32x4_t vk11c1x4567 = vld1q_f32(w + 116);

285

286

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk11c1x0123, vget_low_f32(vi1x1), 0);

287

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk11c1x0123, vget_low_f32(vi3x1), 0);

288

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk11c1x4567, vget_low_f32(vi1x1), 0);

289

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk11c1x4567, vget_low_f32(vi3x1), 0);

290

291

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk11c1x0123, vget_high_f32(vi1x2), 0);

292

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk11c1x0123, vget_high_f32(vi3x2), 0);

293

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk11c1x4567, vget_high_f32(vi1x2), 0);

294

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk11c1x4567, vget_high_f32(vi3x2), 0);

295

296

const float32x4_t vk21c1x0123 = vld1q_f32(w + 120);

297

const float32x4_t vk21c1x4567 = vld1q_f32(w + 124);

298

299

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk21c1x0123, vget_low_f32(vi2x1), 0);

300

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk21c1x0123, vget_low_f32(vi4x1), 0);

301

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk21c1x4567, vget_low_f32(vi2x1), 0);

302

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk21c1x4567, vget_low_f32(vi4x1), 0);

303

304

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk21c1x0123, vget_high_f32(vi2x2), 0);

305

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk21c1x0123, vget_high_f32(vi4x2), 0);

306

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk21c1x4567, vget_high_f32(vi2x2), 0);

307

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk21c1x4567, vget_high_f32(vi4x2), 0);

308

309

const float32x4_t vk01c2x0123 = vld1q_f32(w + 128);

310

const float32x4_t vk01c2x4567 = vld1q_f32(w + 132);

311

312

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk01c2x0123, vget_low_f32(vi0x1), 1);

313

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk01c2x0123, vget_low_f32(vi2x1), 1);

314

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk01c2x4567, vget_low_f32(vi0x1), 1);

315

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk01c2x4567, vget_low_f32(vi2x1), 1);

316

317

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk01c2x0123, vget_high_f32(vi0x2), 1);

318

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk01c2x0123, vget_high_f32(vi2x2), 1);

319

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk01c2x4567, vget_high_f32(vi0x2), 1);

320

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk01c2x4567, vget_high_f32(vi2x2), 1);

321

322

const float32x4_t vk11c2x0123 = vld1q_f32(w + 136);

323

const float32x4_t vk11c2x4567 = vld1q_f32(w + 140);

324

325

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk11c2x0123, vget_low_f32(vi1x1), 1);

326

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk11c2x0123, vget_low_f32(vi3x1), 1);

327

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk11c2x4567, vget_low_f32(vi1x1), 1);

328

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk11c2x4567, vget_low_f32(vi3x1), 1);

329

330

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk11c2x0123, vget_high_f32(vi1x2), 1);

331

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk11c2x0123, vget_high_f32(vi3x2), 1);

332

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk11c2x4567, vget_high_f32(vi1x2), 1);

333

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk11c2x4567, vget_high_f32(vi3x2), 1);

334

335

const float32x4_t vk21c2x0123 = vld1q_f32(w + 144);

336

const float32x4_t vk21c2x4567 = vld1q_f32(w + 148);

337

338

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk21c2x0123, vget_low_f32(vi2x1), 1);

339

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk21c2x0123, vget_low_f32(vi4x1), 1);

340

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk21c2x4567, vget_low_f32(vi2x1), 1);

341

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk21c2x4567, vget_low_f32(vi4x1), 1);

342

343

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk21c2x0123, vget_high_f32(vi2x2), 1);

344

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk21c2x0123, vget_high_f32(vi4x2), 1);

345

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk21c2x4567, vget_high_f32(vi2x2), 1);

346

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk21c2x4567, vget_high_f32(vi4x2), 1);

347

348

const float32x4_t vk02c0x0123 = vld1q_f32(w + 152);

349

const float32x4_t vk02c0x4567 = vld1q_f32(w + 156);

350

351

// viMx3 = ( iM5c0, iM4c2, iM4c1, iM4c0 )

352

const float32x4_t vi0x3 = vld1q_f32(i0); i0 += 4;

353

const float32x4_t vi1x3 = vld1q_f32(i1); i1 += 4;

354

const float32x4_t vi2x3 = vld1q_f32(i2); i2 += 4;

355

const float32x4_t vi3x3 = vld1q_f32(i3); i3 += 4;

356

const float32x4_t vi4x3 = vld1q_f32(i4); i4 += 4;

357

358

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk02c0x0123, vget_high_f32(vi0x1), 0);

359

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk02c0x0123, vget_high_f32(vi2x1), 0);

360

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk02c0x4567, vget_high_f32(vi0x1), 0);

361

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk02c0x4567, vget_high_f32(vi2x1), 0);

362

363

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk02c0x0123, vget_low_f32(vi0x3), 0);

364

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk02c0x0123, vget_low_f32(vi2x3), 0);

365

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk02c0x4567, vget_low_f32(vi0x3), 0);

366

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk02c0x4567, vget_low_f32(vi2x3), 0);

367

368

const float32x4_t vk12c0x0123 = vld1q_f32(w + 160);

369

const float32x4_t vk12c0x4567 = vld1q_f32(w + 164);

370

371

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk12c0x0123, vget_high_f32(vi1x1), 0);

372

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk12c0x0123, vget_high_f32(vi3x1), 0);

373

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk12c0x4567, vget_high_f32(vi1x1), 0);

374

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk12c0x4567, vget_high_f32(vi3x1), 0);

375

376

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk12c0x0123, vget_low_f32(vi1x3), 0);

377

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk12c0x0123, vget_low_f32(vi3x3), 0);

378

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk12c0x4567, vget_low_f32(vi1x3), 0);

379

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk12c0x4567, vget_low_f32(vi3x3), 0);

380

381

const float32x4_t vk22c0x0123 = vld1q_f32(w + 168);

382

const float32x4_t vk22c0x4567 = vld1q_f32(w + 172);

383

384

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk22c0x0123, vget_high_f32(vi2x1), 0);

385

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk22c0x0123, vget_high_f32(vi4x1), 0);

386

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk22c0x4567, vget_high_f32(vi2x1), 0);

387

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk22c0x4567, vget_high_f32(vi4x1), 0);

388

389

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk22c0x0123, vget_low_f32(vi2x3), 0);

390

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk22c0x0123, vget_low_f32(vi4x3), 0);

391

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk22c0x4567, vget_low_f32(vi2x3), 0);

392

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk22c0x4567, vget_low_f32(vi4x3), 0);

393

394

const float32x4_t vk02c1x0123 = vld1q_f32(w + 176);

395

const float32x4_t vk02c1x4567 = vld1q_f32(w + 180);

396

397

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk02c1x0123, vget_high_f32(vi0x1), 1);

398

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk02c1x0123, vget_high_f32(vi2x1), 1);

399

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk02c1x4567, vget_high_f32(vi0x1), 1);

400

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk02c1x4567, vget_high_f32(vi2x1), 1);

401

402

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk02c1x0123, vget_low_f32(vi0x3), 1);

403

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk02c1x0123, vget_low_f32(vi2x3), 1);

404

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk02c1x4567, vget_low_f32(vi0x3), 1);

405

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk02c1x4567, vget_low_f32(vi2x3), 1);

406

407

const float32x4_t vk12c1x0123 = vld1q_f32(w + 184);

408

const float32x4_t vk12c1x4567 = vld1q_f32(w + 188);

409

410

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk12c1x0123, vget_high_f32(vi1x1), 1);

411

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk12c1x0123, vget_high_f32(vi3x1), 1);

412

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk12c1x4567, vget_high_f32(vi1x1), 1);

413

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk12c1x4567, vget_high_f32(vi3x1), 1);

414

415

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk12c1x0123, vget_low_f32(vi1x3), 1);

416

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk12c1x0123, vget_low_f32(vi3x3), 1);

417

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk12c1x4567, vget_low_f32(vi1x3), 1);

418

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk12c1x4567, vget_low_f32(vi3x3), 1);

419

420

const float32x4_t vk22c1x0123 = vld1q_f32(w + 192);

421

const float32x4_t vk22c1x4567 = vld1q_f32(w + 196);

422

423

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk22c1x0123, vget_high_f32(vi2x1), 1);

424

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk22c1x0123, vget_high_f32(vi4x1), 1);

425

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk22c1x4567, vget_high_f32(vi2x1), 1);

426

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk22c1x4567, vget_high_f32(vi4x1), 1);

427

428

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk22c1x0123, vget_low_f32(vi2x3), 1);

429

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk22c1x0123, vget_low_f32(vi4x3), 1);

430

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk22c1x4567, vget_low_f32(vi2x3), 1);

431

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk22c1x4567, vget_low_f32(vi4x3), 1);

432

433

const float32x4_t vk02c2x0123 = vld1q_f32(w + 200);

434

const float32x4_t vk02c2x4567 = vld1q_f32(w + 204);

435

436

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk02c2x0123, vget_low_f32(vi0x2), 0);

437

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk02c2x0123, vget_low_f32(vi2x2), 0);

438

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk02c2x4567, vget_low_f32(vi0x2), 0);

439

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk02c2x4567, vget_low_f32(vi2x2), 0);

440

441

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk02c2x0123, vget_high_f32(vi0x3), 0);

442

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk02c2x0123, vget_high_f32(vi2x3), 0);

443

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk02c2x4567, vget_high_f32(vi0x3), 0);

444

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk02c2x4567, vget_high_f32(vi2x3), 0);

445

446

const float32x4_t vk12c2x0123 = vld1q_f32(w + 208);

447

const float32x4_t vk12c2x4567 = vld1q_f32(w + 212);

448

449

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk12c2x0123, vget_low_f32(vi1x2), 0);

450

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk12c2x0123, vget_low_f32(vi3x2), 0);

451

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk12c2x4567, vget_low_f32(vi1x2), 0);

452

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk12c2x4567, vget_low_f32(vi3x2), 0);

453

454

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk12c2x0123, vget_high_f32(vi1x3), 0);

455

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk12c2x0123, vget_high_f32(vi3x3), 0);

456

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk12c2x4567, vget_high_f32(vi1x3), 0);

457

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk12c2x4567, vget_high_f32(vi3x3), 0);

458

459

const float32x4_t vk22c2x0123 = vld1q_f32(w + 216);

460

const float32x4_t vk22c2x4567 = vld1q_f32(w + 220);

461

462

vo0x0c0123 = vmlaq_lane_f32(vo0x0c0123, vk22c2x0123, vget_low_f32(vi2x2), 0);

463

vo1x0c0123 = vmlaq_lane_f32(vo1x0c0123, vk22c2x0123, vget_low_f32(vi4x2), 0);

464

vo0x0c4567 = vmlaq_lane_f32(vo0x0c4567, vk22c2x4567, vget_low_f32(vi2x2), 0);

465

vo1x0c4567 = vmlaq_lane_f32(vo1x0c4567, vk22c2x4567, vget_low_f32(vi4x2), 0);

466

467

vo0x1c0123 = vmlaq_lane_f32(vo0x1c0123, vk22c2x0123, vget_high_f32(vi2x3), 0);

468

vo1x1c0123 = vmlaq_lane_f32(vo1x1c0123, vk22c2x0123, vget_high_f32(vi4x3), 0);

469

vo0x1c4567 = vmlaq_lane_f32(vo0x1c4567, vk22c2x4567, vget_high_f32(vi2x3), 0);

470

vo1x1c4567 = vmlaq_lane_f32(vo1x1c4567, vk22c2x4567, vget_high_f32(vi4x3), 0);

vi0x0 = vi0x3;

vi1x0 = vi1x3;

vi2x0 = vi2x3;

vi3x0 = vi3x3;

vi4x0 = vi4x3;

Marat Dukhan

2020-05-18 09:35:49 -0700

[diff] [blame]

478

const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);

479

const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

480

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

481

vo0x0c0123 = vmaxq_f32(vo0x0c0123, vmin);

482

vo1x0c0123 = vmaxq_f32(vo1x0c0123, vmin);

483

vo0x0c4567 = vmaxq_f32(vo0x0c4567, vmin);

484

vo1x0c4567 = vmaxq_f32(vo1x0c4567, vmin);

485

486

vo0x1c0123 = vmaxq_f32(vo0x1c0123, vmin);

487

vo1x1c0123 = vmaxq_f32(vo1x1c0123, vmin);

488

vo0x1c4567 = vmaxq_f32(vo0x1c4567, vmin);

489

vo1x1c4567 = vmaxq_f32(vo1x1c4567, vmin);

490

491

vo0x0c0123 = vminq_f32(vo0x0c0123, vmax);

492

vo1x0c0123 = vminq_f32(vo1x0c0123, vmax);

493

vo0x0c4567 = vminq_f32(vo0x0c4567, vmax);

494

vo1x0c4567 = vminq_f32(vo1x0c4567, vmax);

495

496

vo0x1c0123 = vminq_f32(vo0x1c0123, vmax);

497

vo1x1c0123 = vminq_f32(vo1x1c0123, vmax);

498

vo0x1c4567 = vminq_f32(vo0x1c4567, vmax);

499

vo1x1c4567 = vminq_f32(vo1x1c4567, vmax);

500

501

if XNN_LIKELY(c >= 8) {

502

vst1q_f32(o1, vo1x0c0123);

503

vst1q_f32(o1 + 4, vo1x0c4567);

504

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

505

vst1q_f32(o0, vo0x0c0123);

506

vst1q_f32(o0 + 4, vo0x0c4567);

507

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

508

509

vst1q_f32(o1, vo1x1c0123);

510

vst1q_f32(o1 + 4, vo1x1c4567);

511

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

512

vst1q_f32(o0, vo0x1c0123);

513

vst1q_f32(o0 + 4, vo0x1c4567);

514

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

} else {

float* o0_tmp = o0;

float* o1_tmp = o1;

if (c & 4) {

vst1q_f32((float*) ((uintptr_t) o1_tmp + output_width_stride), vo1x1c0123);

520

vo1x1c0123 = vo1x1c4567;

521

vst1q_f32((float*) ((uintptr_t) o0_tmp + output_width_stride), vo0x1c0123);

522

vo0x1c0123 = vo0x1c4567;

523

524

vst1q_f32(o1_tmp, vo1x0c0123); o1_tmp += 4;

525

vo1x0c0123 = vo1x0c4567;

526

vst1q_f32(o0_tmp, vo0x0c0123); o0_tmp += 4;

527

vo0x0c0123 = vo0x0c4567;

528

}

529

float32x2_t vo0x0c01 = vget_low_f32(vo0x0c0123);

530

float32x2_t vo1x0c01 = vget_low_f32(vo1x0c0123);

531

float32x2_t vo0x1c01 = vget_low_f32(vo0x1c0123);

532

float32x2_t vo1x1c01 = vget_low_f32(vo1x1c0123);

533

if (c & 2) {

534

vst1_f32((float*) ((uintptr_t) o1_tmp + output_width_stride), vo1x1c01);

535

vo1x1c01 = vget_high_f32(vo1x1c0123);

536

vst1_f32((float*) ((uintptr_t) o0_tmp + output_width_stride), vo0x1c01);

537

vo0x1c01 = vget_high_f32(vo0x1c0123);

538

539

vst1_f32(o1_tmp, vo1x0c01); o1_tmp += 2;

540

vo1x0c01 = vget_high_f32(vo1x0c0123);

541

vst1_f32(o0_tmp, vo0x0c01); o0_tmp += 2;

542

vo0x0c01 = vget_high_f32(vo0x0c0123);

543

}

544

if (c & 1) {

545

vst1_lane_f32(o1_tmp, vo1x0c01, 0);

546

vst1_lane_f32(o0_tmp, vo0x0c01, 0);

547

548

vst1_lane_f32((float*) ((uintptr_t) o1_tmp + output_width_stride), vo1x1c01, 0);

549

vst1_lane_f32((float*) ((uintptr_t) o0_tmp + output_width_stride), vo0x1c01, 0);

550

}

551

552

o0 = (float*) ((uintptr_t) o0 + output_width_stride * 2);

553

o1 = (float*) ((uintptr_t) o1 + output_width_stride * 2);

554

}

555

}

556

assert(iw < 4);

Marat Dukhan

56b10cd

2020-05-18 09:35:49 -0700

[diff] [blame]

557

if XNN_LIKELY(iw & 2) {

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

558

float32x4_t vo0c0123 = vld1q_f32(w);

559

float32x4_t vo0c4567 = vld1q_f32(w + 4);

560

float32x4_t vo1c0123 = vo0c0123;

561

float32x4_t vo1c4567 = vo0c4567;

562

563

const float32x4_t vk00c0x0123 = vld1q_f32(w + 8);

564

const float32x4_t vk00c0x4567 = vld1q_f32(w + 12);

565

566

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c0x0123, vget_low_f32(vi0x0), 0);

567

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c0x0123, vget_low_f32(vi2x0), 0);

568

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c0x4567, vget_low_f32(vi0x0), 0);

569

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c0x4567, vget_low_f32(vi2x0), 0);

570

571

const float32x4_t vk10c0x0123 = vld1q_f32(w + 16);

572

const float32x4_t vk10c0x4567 = vld1q_f32(w + 20);

573

574

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c0x0123, vget_low_f32(vi1x0), 0);

575

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c0x0123, vget_low_f32(vi3x0), 0);

576

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c0x4567, vget_low_f32(vi1x0), 0);

577

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c0x4567, vget_low_f32(vi3x0), 0);

578

579

const float32x4_t vk20c0x0123 = vld1q_f32(w + 24);

580

const float32x4_t vk20c0x4567 = vld1q_f32(w + 28);

581

582

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c0x0123, vget_low_f32(vi2x0), 0);

583

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c0x0123, vget_low_f32(vi4x0), 0);

584

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c0x4567, vget_low_f32(vi2x0), 0);

585

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c0x4567, vget_low_f32(vi4x0), 0);

586

587

const float32x4_t vk00c1x0123 = vld1q_f32(w + 32);

588

const float32x4_t vk00c1x4567 = vld1q_f32(w + 36);

589

590

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c1x0123, vget_low_f32(vi0x0), 1);

591

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c1x0123, vget_low_f32(vi2x0), 1);

592

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c1x4567, vget_low_f32(vi0x0), 1);

593

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c1x4567, vget_low_f32(vi2x0), 1);

594

595

const float32x4_t vk10c1x0123 = vld1q_f32(w + 40);

596

const float32x4_t vk10c1x4567 = vld1q_f32(w + 44);

597

598

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c1x0123, vget_low_f32(vi1x0), 1);

599

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c1x0123, vget_low_f32(vi3x0), 1);

600

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c1x4567, vget_low_f32(vi1x0), 1);

601

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c1x4567, vget_low_f32(vi3x0), 1);

602

603

const float32x4_t vk20c1x0123 = vld1q_f32(w + 48);

604

const float32x4_t vk20c1x4567 = vld1q_f32(w + 52);

605

606

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c1x0123, vget_low_f32(vi2x0), 1);

607

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c1x0123, vget_low_f32(vi4x0), 1);

608

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c1x4567, vget_low_f32(vi2x0), 1);

609

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c1x4567, vget_low_f32(vi4x0), 1);

610

611

const float32x4_t vk00c2x0123 = vld1q_f32(w + 56);

612

const float32x4_t vk00c2x4567 = vld1q_f32(w + 60);

613

614

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c2x0123, vget_high_f32(vi0x0), 0);

615

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c2x0123, vget_high_f32(vi2x0), 0);

616

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c2x4567, vget_high_f32(vi0x0), 0);

617

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c2x4567, vget_high_f32(vi2x0), 0);

618

619

const float32x4_t vk10c2x0123 = vld1q_f32(w + 64);

620

const float32x4_t vk10c2x4567 = vld1q_f32(w + 68);

621

622

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c2x0123, vget_high_f32(vi1x0), 0);

623

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c2x0123, vget_high_f32(vi3x0), 0);

624

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c2x4567, vget_high_f32(vi1x0), 0);

625

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c2x4567, vget_high_f32(vi3x0), 0);

626

627

const float32x4_t vk20c2x0123 = vld1q_f32(w + 72);

628

const float32x4_t vk20c2x4567 = vld1q_f32(w + 76);

629

630

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c2x0123, vget_high_f32(vi2x0), 0);

631

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c2x0123, vget_high_f32(vi4x0), 0);

632

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c2x4567, vget_high_f32(vi2x0), 0);

633

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c2x4567, vget_high_f32(vi4x0), 0);

634

635

const float32x4_t vk01c0x0123 = vld1q_f32(w + 80);

636

const float32x4_t vk01c0x4567 = vld1q_f32(w + 84);

637

638

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c0x0123, vget_high_f32(vi0x0), 1);

639

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c0x0123, vget_high_f32(vi2x0), 1);

640

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c0x4567, vget_high_f32(vi0x0), 1);

641

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c0x4567, vget_high_f32(vi2x0), 1);

642

643

const float32x4_t vk11c0x0123 = vld1q_f32(w + 88);

644

const float32x4_t vk11c0x4567 = vld1q_f32(w + 92);

645

646

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c0x0123, vget_high_f32(vi1x0), 1);

647

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c0x0123, vget_high_f32(vi3x0), 1);

648

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c0x4567, vget_high_f32(vi1x0), 1);

649

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c0x4567, vget_high_f32(vi3x0), 1);

650

651

const float32x4_t vk21c0x0123 = vld1q_f32(w + 96);

652

const float32x4_t vk21c0x4567 = vld1q_f32(w + 100);

653

654

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c0x0123, vget_high_f32(vi2x0), 1);

655

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c0x0123, vget_high_f32(vi4x0), 1);

656

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c0x4567, vget_high_f32(vi2x0), 1);

657

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c0x4567, vget_high_f32(vi4x0), 1);

658

659

const float32x4_t vk01c1x0123 = vld1q_f32(w + 104);

660

const float32x4_t vk01c1x4567 = vld1q_f32(w + 108);

661

662

// viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )

663

const float32x4_t vi0x1 = vld1q_f32(i0); i0 += 4;

664

const float32x4_t vi1x1 = vld1q_f32(i1); i1 += 4;

665

const float32x4_t vi2x1 = vld1q_f32(i2); i2 += 4;

666

const float32x4_t vi3x1 = vld1q_f32(i3); i3 += 4;

667

const float32x4_t vi4x1 = vld1q_f32(i4); i4 += 4;

668

669

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c1x0123, vget_low_f32(vi0x1), 0);

670

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c1x0123, vget_low_f32(vi2x1), 0);

671

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c1x4567, vget_low_f32(vi0x1), 0);

672

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c1x4567, vget_low_f32(vi2x1), 0);

673

674

const float32x4_t vk11c1x0123 = vld1q_f32(w + 112);

675

const float32x4_t vk11c1x4567 = vld1q_f32(w + 116);

676

677

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c1x0123, vget_low_f32(vi1x1), 0);

678

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c1x0123, vget_low_f32(vi3x1), 0);

679

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c1x4567, vget_low_f32(vi1x1), 0);

680

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c1x4567, vget_low_f32(vi3x1), 0);

681

682

const float32x4_t vk21c1x0123 = vld1q_f32(w + 120);

683

const float32x4_t vk21c1x4567 = vld1q_f32(w + 124);

684

685

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c1x0123, vget_low_f32(vi2x1), 0);

686

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c1x0123, vget_low_f32(vi4x1), 0);

687

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c1x4567, vget_low_f32(vi2x1), 0);

688

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c1x4567, vget_low_f32(vi4x1), 0);

689

690

const float32x4_t vk01c2x0123 = vld1q_f32(w + 128);

691

const float32x4_t vk01c2x4567 = vld1q_f32(w + 132);

692

693

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c2x0123, vget_low_f32(vi0x1), 1);

694

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c2x0123, vget_low_f32(vi2x1), 1);

695

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c2x4567, vget_low_f32(vi0x1), 1);

696

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c2x4567, vget_low_f32(vi2x1), 1);

697

698

const float32x4_t vk11c2x0123 = vld1q_f32(w + 136);

699

const float32x4_t vk11c2x4567 = vld1q_f32(w + 140);

700

701

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c2x0123, vget_low_f32(vi1x1), 1);

702

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c2x0123, vget_low_f32(vi3x1), 1);

703

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c2x4567, vget_low_f32(vi1x1), 1);

704

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c2x4567, vget_low_f32(vi3x1), 1);

705

706

const float32x4_t vk21c2x0123 = vld1q_f32(w + 144);

707

const float32x4_t vk21c2x4567 = vld1q_f32(w + 148);

708

709

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c2x0123, vget_low_f32(vi2x1), 1);

710

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c2x0123, vget_low_f32(vi4x1), 1);

711

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c2x4567, vget_low_f32(vi2x1), 1);

712

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c2x4567, vget_low_f32(vi4x1), 1);

713

714

const float32x4_t vk02c0x0123 = vld1q_f32(w + 152);

715

const float32x4_t vk02c0x4567 = vld1q_f32(w + 156);

716

717

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk02c0x0123, vget_high_f32(vi0x1), 0);

718

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk02c0x0123, vget_high_f32(vi2x1), 0);

719

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk02c0x4567, vget_high_f32(vi0x1), 0);

720

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk02c0x4567, vget_high_f32(vi2x1), 0);

721

722

const float32x4_t vk12c0x0123 = vld1q_f32(w + 160);

723

const float32x4_t vk12c0x4567 = vld1q_f32(w + 164);

724

725

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk12c0x0123, vget_high_f32(vi1x1), 0);

726

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk12c0x0123, vget_high_f32(vi3x1), 0);

727

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk12c0x4567, vget_high_f32(vi1x1), 0);

728

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk12c0x4567, vget_high_f32(vi3x1), 0);

729

730

const float32x4_t vk22c0x0123 = vld1q_f32(w + 168);

731

const float32x4_t vk22c0x4567 = vld1q_f32(w + 172);

732

733

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk22c0x0123, vget_high_f32(vi2x1), 0);

734

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk22c0x0123, vget_high_f32(vi4x1), 0);

735

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk22c0x4567, vget_high_f32(vi2x1), 0);

736

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk22c0x4567, vget_high_f32(vi4x1), 0);

737

738

const float32x4_t vk02c1x0123 = vld1q_f32(w + 176);

739

const float32x4_t vk02c1x4567 = vld1q_f32(w + 180);

740

741

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk02c1x0123, vget_high_f32(vi0x1), 1);

742

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk02c1x0123, vget_high_f32(vi2x1), 1);

743

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk02c1x4567, vget_high_f32(vi0x1), 1);

744

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk02c1x4567, vget_high_f32(vi2x1), 1);

745

746

const float32x4_t vk12c1x0123 = vld1q_f32(w + 184);

747

const float32x4_t vk12c1x4567 = vld1q_f32(w + 188);

748

749

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk12c1x0123, vget_high_f32(vi1x1), 1);

750

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk12c1x0123, vget_high_f32(vi3x1), 1);

751

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk12c1x4567, vget_high_f32(vi1x1), 1);

752

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk12c1x4567, vget_high_f32(vi3x1), 1);

753

754

const float32x4_t vk22c1x0123 = vld1q_f32(w + 192);

755

const float32x4_t vk22c1x4567 = vld1q_f32(w + 196);

756

757

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk22c1x0123, vget_high_f32(vi2x1), 1);

758

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk22c1x0123, vget_high_f32(vi4x1), 1);

759

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk22c1x4567, vget_high_f32(vi2x1), 1);

760

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk22c1x4567, vget_high_f32(vi4x1), 1);

761

762

const float32x4_t vk02c2x0123 = vld1q_f32(w + 200);

763

const float32x4_t vk02c2x4567 = vld1q_f32(w + 204);

764

765

// viMx2 = ( iM2c2, iM2c1 )

766

const float32x2_t vi0x2 = vld1_f32(i0); i0 += 2;

767

const float32x2_t vi1x2 = vld1_f32(i1); i1 += 2;

768

const float32x2_t vi2x2 = vld1_f32(i2); i2 += 2;

769

const float32x2_t vi3x2 = vld1_f32(i3); i3 += 2;

770

const float32x2_t vi4x2 = vld1_f32(i4); i4 += 2;

771

772

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk02c2x0123, vi0x2, 0);

773

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk02c2x0123, vi2x2, 0);

774

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk02c2x4567, vi0x2, 0);

775

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk02c2x4567, vi2x2, 0);

776

777

const float32x4_t vk12c2x0123 = vld1q_f32(w + 208);

778

const float32x4_t vk12c2x4567 = vld1q_f32(w + 212);

779

780

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk12c2x0123, vi1x2, 0);

781

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk12c2x0123, vi3x2, 0);

782

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk12c2x4567, vi1x2, 0);

783

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk12c2x4567, vi3x2, 0);

784

785

const float32x4_t vk22c2x0123 = vld1q_f32(w + 216);

786

const float32x4_t vk22c2x4567 = vld1q_f32(w + 220);

787

788

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk22c2x0123, vi2x2, 0);

789

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk22c2x0123, vi4x2, 0);

790

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk22c2x4567, vi2x2, 0);

791

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk22c2x4567, vi4x2, 0);

792

793

vi0x0 = vcombine_f32(vget_high_f32(vi0x1), vi0x2);

794

vi1x0 = vcombine_f32(vget_high_f32(vi1x1), vi1x2);

795

vi2x0 = vcombine_f32(vget_high_f32(vi2x1), vi2x2);

796

vi3x0 = vcombine_f32(vget_high_f32(vi3x1), vi3x2);

797

vi4x0 = vcombine_f32(vget_high_f32(vi4x1), vi4x2);

798

Marat Dukhan

56b10cd

2020-05-18 09:35:49 -0700

[diff] [blame]

799

const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);

800

const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

801

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

802

vo0c0123 = vmaxq_f32(vo0c0123, vmin);

803

vo1c0123 = vmaxq_f32(vo1c0123, vmin);

804

vo0c4567 = vmaxq_f32(vo0c4567, vmin);

805

vo1c4567 = vmaxq_f32(vo1c4567, vmin);

806

807

vo0c0123 = vminq_f32(vo0c0123, vmax);

808

vo1c0123 = vminq_f32(vo1c0123, vmax);

809

vo0c4567 = vminq_f32(vo0c4567, vmax);

810

vo1c4567 = vminq_f32(vo1c4567, vmax);

811

812

if XNN_LIKELY(c >= 8) {

813

vst1q_f32(o1, vo1c0123);

814

vst1q_f32(o1 + 4, vo1c4567);

815

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

816

vst1q_f32(o0, vo0c0123);

817

vst1q_f32(o0 + 4, vo0c4567);

818

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

} else {

float* o0_tmp = o0;

float* o1_tmp = o1;

if (c & 4) {

vst1q_f32(o1_tmp, vo1c0123); o1_tmp += 4;

824

vo1c0123 = vo1c4567;

825

vst1q_f32(o0_tmp, vo0c0123); o0_tmp += 4;

826

vo0c0123 = vo0c4567;

827

}

828

float32x2_t vo0c01 = vget_low_f32(vo0c0123);

829

float32x2_t vo1c01 = vget_low_f32(vo1c0123);

830

if (c & 2) {

831

vst1_f32(o1_tmp, vo1c01); o1_tmp += 2;

832

vo1c01 = vget_high_f32(vo1c0123);

833

vst1_f32(o0_tmp, vo0c01); o0_tmp += 2;

834

vo0c01 = vget_high_f32(vo0c0123);

835

}

836

if (c & 1) {

837

vst1_lane_f32(o1_tmp, vo1c01, 0);

838

vst1_lane_f32(o0_tmp, vo0c01, 0);

839

}

840

841

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

842

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

843

}

844

}

Marat Dukhan

56b10cd

2020-05-18 09:35:49 -0700

[diff] [blame]

845

if XNN_LIKELY(iw & 1) {

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

846

float32x4_t vo0c0123 = vld1q_f32(w);

847

float32x4_t vo0c4567 = vld1q_f32(w + 4);

848

float32x4_t vo1c0123 = vo0c0123;

849

float32x4_t vo1c4567 = vo0c4567;

850

851

const float32x4_t vk00c0x0123 = vld1q_f32(w + 8);

852

const float32x4_t vk00c0x4567 = vld1q_f32(w + 12);

853

854

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c0x0123, vget_low_f32(vi0x0), 0);

855

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c0x0123, vget_low_f32(vi2x0), 0);

856

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c0x4567, vget_low_f32(vi0x0), 0);

857

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c0x4567, vget_low_f32(vi2x0), 0);

858

859

const float32x4_t vk10c0x0123 = vld1q_f32(w + 16);

860

const float32x4_t vk10c0x4567 = vld1q_f32(w + 20);

861

862

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c0x0123, vget_low_f32(vi1x0), 0);

863

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c0x0123, vget_low_f32(vi3x0), 0);

864

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c0x4567, vget_low_f32(vi1x0), 0);

865

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c0x4567, vget_low_f32(vi3x0), 0);

866

867

const float32x4_t vk20c0x0123 = vld1q_f32(w + 24);

868

const float32x4_t vk20c0x4567 = vld1q_f32(w + 28);

869

870

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c0x0123, vget_low_f32(vi2x0), 0);

871

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c0x0123, vget_low_f32(vi4x0), 0);

872

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c0x4567, vget_low_f32(vi2x0), 0);

873

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c0x4567, vget_low_f32(vi4x0), 0);

874

875

const float32x4_t vk00c1x0123 = vld1q_f32(w + 32);

876

const float32x4_t vk00c1x4567 = vld1q_f32(w + 36);

877

878

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c1x0123, vget_low_f32(vi0x0), 1);

879

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c1x0123, vget_low_f32(vi2x0), 1);

880

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c1x4567, vget_low_f32(vi0x0), 1);

881

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c1x4567, vget_low_f32(vi2x0), 1);

882

883

const float32x4_t vk10c1x0123 = vld1q_f32(w + 40);

884

const float32x4_t vk10c1x4567 = vld1q_f32(w + 44);

885

886

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c1x0123, vget_low_f32(vi1x0), 1);

887

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c1x0123, vget_low_f32(vi3x0), 1);

888

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c1x4567, vget_low_f32(vi1x0), 1);

889

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c1x4567, vget_low_f32(vi3x0), 1);

890

891

const float32x4_t vk20c1x0123 = vld1q_f32(w + 48);

892

const float32x4_t vk20c1x4567 = vld1q_f32(w + 52);

893

894

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c1x0123, vget_low_f32(vi2x0), 1);

895

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c1x0123, vget_low_f32(vi4x0), 1);

896

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c1x4567, vget_low_f32(vi2x0), 1);

897

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c1x4567, vget_low_f32(vi4x0), 1);

898

899

const float32x4_t vk00c2x0123 = vld1q_f32(w + 56);

900

const float32x4_t vk00c2x4567 = vld1q_f32(w + 60);

901

902

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk00c2x0123, vget_high_f32(vi0x0), 0);

903

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk00c2x0123, vget_high_f32(vi2x0), 0);

904

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk00c2x4567, vget_high_f32(vi0x0), 0);

905

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk00c2x4567, vget_high_f32(vi2x0), 0);

906

907

const float32x4_t vk10c2x0123 = vld1q_f32(w + 64);

908

const float32x4_t vk10c2x4567 = vld1q_f32(w + 68);

909

910

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk10c2x0123, vget_high_f32(vi1x0), 0);

911

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk10c2x0123, vget_high_f32(vi3x0), 0);

912

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk10c2x4567, vget_high_f32(vi1x0), 0);

913

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk10c2x4567, vget_high_f32(vi3x0), 0);

914

915

const float32x4_t vk20c2x0123 = vld1q_f32(w + 72);

916

const float32x4_t vk20c2x4567 = vld1q_f32(w + 76);

917

918

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk20c2x0123, vget_high_f32(vi2x0), 0);

919

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk20c2x0123, vget_high_f32(vi4x0), 0);

920

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk20c2x4567, vget_high_f32(vi2x0), 0);

921

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk20c2x4567, vget_high_f32(vi4x0), 0);

922

923

const float32x4_t vk01c0x0123 = vld1q_f32(w + 80);

924

const float32x4_t vk01c0x4567 = vld1q_f32(w + 84);

925

926

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c0x0123, vget_high_f32(vi0x0), 1);

927

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c0x0123, vget_high_f32(vi2x0), 1);

928

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c0x4567, vget_high_f32(vi0x0), 1);

929

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c0x4567, vget_high_f32(vi2x0), 1);

930

931

const float32x4_t vk11c0x0123 = vld1q_f32(w + 88);

932

const float32x4_t vk11c0x4567 = vld1q_f32(w + 92);

933

934

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c0x0123, vget_high_f32(vi1x0), 1);

935

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c0x0123, vget_high_f32(vi3x0), 1);

936

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c0x4567, vget_high_f32(vi1x0), 1);

937

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c0x4567, vget_high_f32(vi3x0), 1);

938

939

const float32x4_t vk21c0x0123 = vld1q_f32(w + 96);

940

const float32x4_t vk21c0x4567 = vld1q_f32(w + 100);

941

942

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c0x0123, vget_high_f32(vi2x0), 1);

943

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c0x0123, vget_high_f32(vi4x0), 1);

944

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c0x4567, vget_high_f32(vi2x0), 1);

945

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c0x4567, vget_high_f32(vi4x0), 1);

946

947

const float32x4_t vk01c1x0123 = vld1q_f32(w + 104);

948

const float32x4_t vk01c1x4567 = vld1q_f32(w + 108);

949

950

// viMx1 = ( iM1c2, iM1c1 )

951

const float32x2_t vi0x1 = vld1_f32(i0); i0 += 2;

952

const float32x2_t vi1x1 = vld1_f32(i1); i1 += 2;

953

const float32x2_t vi2x1 = vld1_f32(i2); i2 += 2;

954

const float32x2_t vi3x1 = vld1_f32(i3); i3 += 2;

955

const float32x2_t vi4x1 = vld1_f32(i4); i4 += 2;

956

957

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c1x0123, vi0x1, 0);

958

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c1x0123, vi2x1, 0);

959

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c1x4567, vi0x1, 0);

960

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c1x4567, vi2x1, 0);

961

962

const float32x4_t vk11c1x0123 = vld1q_f32(w + 112);

963

const float32x4_t vk11c1x4567 = vld1q_f32(w + 116);

964

965

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c1x0123, vi1x1, 0);

966

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c1x0123, vi3x1, 0);

967

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c1x4567, vi1x1, 0);

968

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c1x4567, vi3x1, 0);

969

970

const float32x4_t vk21c1x0123 = vld1q_f32(w + 120);

971

const float32x4_t vk21c1x4567 = vld1q_f32(w + 124);

972

973

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c1x0123, vi2x1, 0);

974

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c1x0123, vi4x1, 0);

975

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c1x4567, vi2x1, 0);

976

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c1x4567, vi4x1, 0);

977

978

const float32x4_t vk01c2x0123 = vld1q_f32(w + 128);

979

const float32x4_t vk01c2x4567 = vld1q_f32(w + 132);

980

981

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk01c2x0123, vi0x1, 1);

982

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk01c2x0123, vi2x1, 1);

983

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk01c2x4567, vi0x1, 1);

984

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk01c2x4567, vi2x1, 1);

985

986

const float32x4_t vk11c2x0123 = vld1q_f32(w + 136);

987

const float32x4_t vk11c2x4567 = vld1q_f32(w + 140);

988

989

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk11c2x0123, vi1x1, 1);

990

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk11c2x0123, vi3x1, 1);

991

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk11c2x4567, vi1x1, 1);

992

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk11c2x4567, vi3x1, 1);

993

994

const float32x4_t vk21c2x0123 = vld1q_f32(w + 144);

995

const float32x4_t vk21c2x4567 = vld1q_f32(w + 148);

996

997

vo0c0123 = vmlaq_lane_f32(vo0c0123, vk21c2x0123, vi2x1, 1);

998

vo1c0123 = vmlaq_lane_f32(vo1c0123, vk21c2x0123, vi4x1, 1);

999

vo0c4567 = vmlaq_lane_f32(vo0c4567, vk21c2x4567, vi2x1, 1);

1000

vo1c4567 = vmlaq_lane_f32(vo1c4567, vk21c2x4567, vi4x1, 1);

1001

Marat Dukhan

56b10cd

2020-05-18 09:35:49 -0700

[diff] [blame]

1002

const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);

1003

const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

1004

Marat Dukhan

ce7a3f8

2020-05-17 21:46:44 -0700

[diff] [blame]

1005

vo0c0123 = vmaxq_f32(vo0c0123, vmin);

1006

vo1c0123 = vmaxq_f32(vo1c0123, vmin);

1007

vo0c4567 = vmaxq_f32(vo0c4567, vmin);

1008

vo1c4567 = vmaxq_f32(vo1c4567, vmin);

1009

1010

vo0c0123 = vminq_f32(vo0c0123, vmax);

1011

vo1c0123 = vminq_f32(vo1c0123, vmax);

1012

vo0c4567 = vminq_f32(vo0c4567, vmax);

1013

vo1c4567 = vminq_f32(vo1c4567, vmax);

1014

1015

if XNN_LIKELY(c >= 8) {

1016

vst1q_f32(o1, vo1c0123);

1017

vst1q_f32(o1 + 4, vo1c4567);

1018

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

1019

vst1q_f32(o0, vo0c0123);

1020

vst1q_f32(o0 + 4, vo0c4567);

1021

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

} else {

float* o0_tmp = o0;

float* o1_tmp = o1;

if (c & 4) {

vst1q_f32(o1_tmp, vo1c0123); o1_tmp += 4;

1027

vo1c0123 = vo1c4567;

1028

vst1q_f32(o0_tmp, vo0c0123); o0_tmp += 4;

1029

vo0c0123 = vo0c4567;

1030

}

1031

float32x2_t vo0c01 = vget_low_f32(vo0c0123);

1032

float32x2_t vo1c01 = vget_low_f32(vo1c0123);

1033

if (c & 2) {

1034

vst1_f32(o1_tmp, vo1c01); o1_tmp += 2;

1035

vo1c01 = vget_high_f32(vo1c0123);

1036

vst1_f32(o0_tmp, vo0c01); o0_tmp += 2;

1037

vo0c01 = vget_high_f32(vo0c0123);

1038

}

1039

if (c & 1) {

1040

vst1_lane_f32(o1_tmp, vo1c01, 0);

1041

vst1_lane_f32(o0_tmp, vo0c01, 0);

1042

}

1043

o0 = (float*) ((uintptr_t) o0 + output_width_stride);

1044

o1 = (float*) ((uintptr_t) o1 + output_width_stride);

1045

}

1046

}

1047

// Move output pointers back to the position of the first pixel in a row,

1048

// and forward to the next block of output channels

1049

o0 = (float*) ((uintptr_t) o0 - output_channel_decrement);

1050

o1 = (float*) ((uintptr_t) o1 - output_channel_decrement);

1051

// Revert input pointers to the position of the first pixel in a row

1052

i0 = (const float*) ((uintptr_t) i0 - input_width_decrement);

1053

i1 = (const float*) ((uintptr_t) i1 - input_width_decrement);

1054

i2 = (const float*) ((uintptr_t) i2 - input_width_decrement);

1055

i3 = (const float*) ((uintptr_t) i3 - input_width_decrement);

1056

i4 = (const float*) ((uintptr_t) i4 - input_width_decrement);

1057

// Move to the block of weights for the next 8 output channels

w += 224;

c = doz(c, 8);

} while (c != 0);

// Move output pointers back to the position of the first channel, and forward to the next block of rows

1062

o0 = (float*) ((uintptr_t) o0 + output_height_increment);

1063

o1 = (float*) ((uintptr_t) o1 + output_height_increment);

1064

// Move input pointers forward to the next four rows

1065

i0 = i4;

1066

i1 = (const float*) ((uintptr_t) i0 + input_height_stride);

1067

i2 = (const float*) ((uintptr_t) i1 + input_height_stride);

1068

i3 = (const float*) ((uintptr_t) i2 + input_height_stride);

1069

i4 = (const float*) ((uintptr_t) i3 + input_height_stride);

1070

}

1071

}