blob: b3af11e74ae6b752182d36b791113ee5a5cbc7b7 [file] [log] [blame]
Anthony Barbier871448e2017-03-24 14:54:29 +00001/*
Anthony Barbierf45d5a92018-01-24 16:23:15 +00002 * Copyright (c) 2017-2018 ARM Limited.
Anthony Barbier871448e2017-03-24 14:54:29 +00003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/runtime/CL/functions/CLConvolutionLayer.h"
25
26#include "arm_compute/core/PixelValue.h"
Kaizen8938bd32017-09-28 14:38:23 +010027#include "arm_compute/core/Size2D.h"
Anthony Barbier871448e2017-03-24 14:54:29 +000028#include "arm_compute/core/Utils.h"
29#include "arm_compute/core/Validate.h"
Anthony Barbier8140e1e2017-12-14 23:48:46 +000030#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
Anthony Barbier871448e2017-03-24 14:54:29 +000031#include "arm_compute/runtime/CL/CLScheduler.h"
32
33#include <cmath>
Kaizen8938bd32017-09-28 14:38:23 +010034#include <memory>
Anthony Barbier871448e2017-03-24 14:54:29 +000035#include <tuple>
36
37using namespace arm_compute;
38
Kaizen8938bd32017-09-28 14:38:23 +010039CLConvolutionLayerReshapeWeights::CLConvolutionLayerReshapeWeights(std::shared_ptr<IMemoryManager> memory_manager)
40 : _memory_group(std::move(memory_manager)), _weights_reshape_kernel(), _weights_transposed_kernel(), _weights_reshaped(), _transpose1xW(false)
Anthony Barbier871448e2017-03-24 14:54:29 +000041{
42}
43
Anthony Barbierdbdab852017-06-23 15:42:00 +010044void CLConvolutionLayerReshapeWeights::configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, bool transpose1xW)
45{
Anthony Barbierdbdab852017-06-23 15:42:00 +010046 ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
47
48 if(biases != nullptr)
49 {
Anthony Barbier8140e1e2017-12-14 23:48:46 +000050 ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(weights->info()->data_type()));
Anthony Barbierdbdab852017-06-23 15:42:00 +010051 ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
52 ARM_COMPUTE_ERROR_ON(biases->info()->dimension(0) != weights->info()->dimension(3));
53 ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
54 }
55
Anthony Barbier8140e1e2017-12-14 23:48:46 +000056 const bool append_biases = (biases != nullptr) && !is_data_type_quantized_asymmetric(weights->info()->data_type());
57 const unsigned bias_element = (append_biases) ? 1 : 0;
58 const ICLTensor *biases_to_use = (append_biases) ? biases : nullptr;
Anthony Barbierdbdab852017-06-23 15:42:00 +010059
60 _transpose1xW = transpose1xW;
61
62 if(transpose1xW)
63 {
64 // Create tensor to store the reshaped weights
65 const unsigned int mat_weights_cols = weights->info()->dimension(3);
Anthony Barbier8140e1e2017-12-14 23:48:46 +000066 const unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element;
Anthony Barbierdbdab852017-06-23 15:42:00 +010067 TensorShape shape_wr(mat_weights_cols, mat_weights_rows);
Kaizen8938bd32017-09-28 14:38:23 +010068 const DataType dt = weights->info()->data_type();
69 const int fixed_point_position = weights->info()->fixed_point_position();
70 TensorInfo info_wr(shape_wr, 1, dt, fixed_point_position);
Anthony Barbierdbdab852017-06-23 15:42:00 +010071
72 _weights_reshaped.allocator()->init(info_wr);
Kaizen8938bd32017-09-28 14:38:23 +010073 _memory_group.manage(&_weights_reshaped);
Anthony Barbier8140e1e2017-12-14 23:48:46 +000074 _weights_reshape_kernel.configure(weights, biases_to_use, &_weights_reshaped);
Anthony Barbierdbdab852017-06-23 15:42:00 +010075 _weights_transposed_kernel.configure(&_weights_reshaped, output);
76 _weights_reshaped.allocator()->allocate();
77 }
78 else
79 {
Anthony Barbier8140e1e2017-12-14 23:48:46 +000080 _weights_reshape_kernel.configure(weights, biases_to_use, output);
Anthony Barbierdbdab852017-06-23 15:42:00 +010081 }
Anthony Barbierf45d5a92018-01-24 16:23:15 +000082
83 output->info()->set_quantization_info(weights->info()->quantization_info());
Anthony Barbierdbdab852017-06-23 15:42:00 +010084}
85
86void CLConvolutionLayerReshapeWeights::run()
87{
Kaizen8938bd32017-09-28 14:38:23 +010088 _memory_group.acquire();
89
Anthony Barbierdbdab852017-06-23 15:42:00 +010090 CLScheduler::get().enqueue(_weights_reshape_kernel);
91 if(_transpose1xW)
92 {
93 CLScheduler::get().enqueue(_weights_transposed_kernel);
94 }
Kaizen8938bd32017-09-28 14:38:23 +010095
96 _memory_group.release();
Anthony Barbierdbdab852017-06-23 15:42:00 +010097}
98
Kaizen8938bd32017-09-28 14:38:23 +010099CLConvolutionLayer::CLConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000100 : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _interleave_kernel(), _mm_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(),
101 _col2im_kernel(), _im2col_output(), _interleave_output(), _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _are_weights_reshaped(false), _is_quantized(false),
102 _is_interleaved_transposed(false)
Anthony Barbierdbdab852017-06-23 15:42:00 +0100103{
104}
105
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000106void CLConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, bool is_interleaved_transposed, bool are_weights_reshaped)
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000107{
108 if(_is_quantized)
109 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000110 if(are_weights_reshaped)
111 {
112 ARM_COMPUTE_ERROR("Weights already reshaped are not suppported with gemmlowp");
113 }
114 else
115 {
116 // Since we need negative offsets for computing convolution, we need to change QuantizationInfo()
117 // Extract and negate input and weights offset
118 const QuantizationInfo input_quantization_info = input->info()->quantization_info();
119 const QuantizationInfo weights_quantization_info = weights->info()->quantization_info();
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000120
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000121 input->info()->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset));
122 weights->info()->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset));
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000123
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000124 _mm_gemmlowp.configure(input, weights, output, GEMMInfo(false, false, true /* Reshape weights only for the first run*/));
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000125
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000126 // Revert back QuantizatioInfo as input and weights could be used in other convolution layers
127 input->info()->set_quantization_info(input_quantization_info);
128 weights->info()->set_quantization_info(weights_quantization_info);
129 }
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000130 }
131 else
132 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000133 if(are_weights_reshaped)
134 {
135 // Configure matrix multiply kernel
136 _mm_kernel.configure(input, weights, output, 1.f, is_interleaved_transposed);
137 }
138 else
139 {
140 // Configure matrix multiply function
141 _mm_gemm.configure(input, weights, nullptr, output, 1.0f, 0.0f, GEMMInfo(false, false, true /* Reshape weights only for the first run*/));
142 }
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000143 }
144}
145
Anthony Barbierdbdab852017-06-23 15:42:00 +0100146void CLConvolutionLayer::configure(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info)
Anthony Barbier871448e2017-03-24 14:54:29 +0000147{
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000148 ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32);
Kaizen8938bd32017-09-28 14:38:23 +0100149 ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
150 ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights);
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000151 ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && CLScheduler::get().target() == GPUTarget::BIFROST);
Anthony Barbierdbdab852017-06-23 15:42:00 +0100152 ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && weights->info()->dimension(2) != input->info()->dimension(2));
Anthony Barbier871448e2017-03-24 14:54:29 +0000153 ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000154 ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && is_data_type_quantized_asymmetric(input->info()->data_type()));
155
156 _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
Anthony Barbier871448e2017-03-24 14:54:29 +0000157
158 if(biases != nullptr)
159 {
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000160 if(_is_quantized)
161 {
162 ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(biases, 1, DataType::S32);
163 }
164 else
165 {
166 ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
167 }
Kaizen8938bd32017-09-28 14:38:23 +0100168 ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases);
Anthony Barbierdbdab852017-06-23 15:42:00 +0100169 ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && biases->info()->dimension(0) != weights->info()->dimension(3));
Anthony Barbier871448e2017-03-24 14:54:29 +0000170 ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
171 }
172
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000173 const DataType dt = input->info()->data_type();
Kaizen8938bd32017-09-28 14:38:23 +0100174
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000175 // Set the GPU target for matrix multiply and im2col and col2im
Kaizen8938bd32017-09-28 14:38:23 +0100176 _mm_kernel.set_target(CLScheduler::get().target());
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000177 _im2col_kernel.set_target(CLScheduler::get().target());
178 _col2im_kernel.set_target(CLScheduler::get().target());
Kaizen8938bd32017-09-28 14:38:23 +0100179
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000180 const bool append_bias = (biases != nullptr) && (!_is_quantized);
181 _are_weights_reshaped = weights_info.are_reshaped();
Anthony Barbier871448e2017-03-24 14:54:29 +0000182
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000183 const unsigned bias_element = (append_bias) ? 1 : 0;
184 const ICLTensor *biases_to_use = (append_bias) ? biases : nullptr;
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000185
Kaizen8938bd32017-09-28 14:38:23 +0100186 // Get parameters from conv_info
Anthony Barbierdbdab852017-06-23 15:42:00 +0100187 unsigned int stride_x = 0;
188 unsigned int stride_y = 0;
Anthony Barbier871448e2017-03-24 14:54:29 +0000189 std::tie(stride_x, stride_y) = conv_info.stride();
Anthony Barbier871448e2017-03-24 14:54:29 +0000190
Anthony Barbier871448e2017-03-24 14:54:29 +0000191 // Get convolved dimensions
192 unsigned int conv_w = 0;
193 unsigned int conv_h = 0;
Anthony Barbierdbdab852017-06-23 15:42:00 +0100194
Kaizen8938bd32017-09-28 14:38:23 +0100195 const unsigned int kernel_width = (_are_weights_reshaped) ? weights_info.kernel_size().first : weights->info()->dimension(0);
196 const unsigned int kernel_height = (_are_weights_reshaped) ? weights_info.kernel_size().second : weights->info()->dimension(1);
197 std::tie(conv_w, conv_h) = scaled_dimensions(input->info()->dimension(0), input->info()->dimension(1), kernel_width, kernel_height,
198 conv_info);
Anthony Barbier871448e2017-03-24 14:54:29 +0000199
Anthony Barbierdbdab852017-06-23 15:42:00 +0100200 // Check if its a "fully connected" convolution
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000201 const bool is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
202 _is_interleaved_transposed = (!is_fully_connected_convolution) && (!_is_quantized) && (_are_weights_reshaped);
Anthony Barbierdbdab852017-06-23 15:42:00 +0100203
Kaizen8938bd32017-09-28 14:38:23 +0100204 unsigned int mat_weights_cols = weights->info()->dimension(3);
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000205 unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element;
Kaizen8938bd32017-09-28 14:38:23 +0100206
207 // Reshape weights if needed
Anthony Barbierdbdab852017-06-23 15:42:00 +0100208 if(_are_weights_reshaped)
209 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000210 if(is_fully_connected_convolution || _is_quantized)
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000211 {
212 mat_weights_cols = weights->info()->dimension(0);
213 mat_weights_rows = weights->info()->dimension(1);
214 }
215 else
216 {
217 mat_weights_cols = weights_info.num_kernels();
218 const unsigned int quarter_reshaped_cols = weights->info()->dimension(0) / 4;
219 mat_weights_rows = quarter_reshaped_cols + bias_element;
220 }
Anthony Barbierdbdab852017-06-23 15:42:00 +0100221 }
222 else
223 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000224 // _weights_reshaped will be auto configured in the kernel.
225 // Just append biases and do not transpose 1xW as it will be reshaped in CLGEMM
226 _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, false);
227
Kaizen8938bd32017-09-28 14:38:23 +0100228 weights = &_weights_reshaped;
Anthony Barbierdbdab852017-06-23 15:42:00 +0100229 }
Kaizen8938bd32017-09-28 14:38:23 +0100230
Anthony Barbier871448e2017-03-24 14:54:29 +0000231 // Create tensor to store im2col reshaped inputs
Kaizen8938bd32017-09-28 14:38:23 +0100232 const unsigned int mat_input_cols = mat_weights_rows;
233 const unsigned int mat_input_rows = conv_w * conv_h;
234 TensorShape shape_im2col = input->info()->tensor_shape();
Anthony Barbier871448e2017-03-24 14:54:29 +0000235 shape_im2col.set(0, mat_input_cols);
236 shape_im2col.set(1, mat_input_rows);
237 shape_im2col.set(2, 1);
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000238 //input->clone() doesn't work with subtensors for grouped convolutions.
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000239 TensorInfo im2col_reshaped_info(shape_im2col, 1, dt, input->info()->fixed_point_position());
240 im2col_reshaped_info.set_quantization_info(input->info()->quantization_info());
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000241 _im2col_output.allocator()->init(im2col_reshaped_info);
242 _memory_group.manage(&_im2col_output);
Anthony Barbier871448e2017-03-24 14:54:29 +0000243
244 // Create GEMM output tensor
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000245 TensorShape shape_gemm = _im2col_output.info()->tensor_shape();
Anthony Barbier871448e2017-03-24 14:54:29 +0000246 shape_gemm.set(0, mat_weights_cols);
247 shape_gemm.set(1, mat_input_rows);
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000248 const DataType gemm_data_type = _is_quantized ? DataType::S32 : dt;
249 // GEMM output should be S32 for acquiring raw integer accumulator without quantized postprocessing for quantized asymmetric input.
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000250 //input->clone() doesn't work with subtensors for grouped convolutions.
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000251 TensorInfo info_gemm(shape_gemm, 1, gemm_data_type, input->info()->fixed_point_position());
252 info_gemm.set_quantization_info(output->info()->quantization_info());
253 _gemm_output.allocator()->init(info_gemm);
Kaizen8938bd32017-09-28 14:38:23 +0100254 _memory_group.manage(&_gemm_output);
Anthony Barbier871448e2017-03-24 14:54:29 +0000255
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000256 // Configure im2col
257 _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, append_bias);
Anthony Barbierdbdab852017-06-23 15:42:00 +0100258
Kaizen8938bd32017-09-28 14:38:23 +0100259 // Configure matrix multiply
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000260 if(_is_interleaved_transposed)
Anthony Barbier871448e2017-03-24 14:54:29 +0000261 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000262 // Configure GEMMInterleave4x4. _input_interleaved_reshaped will be auto configured in the kernel
263 _interleave_kernel.configure(&_im2col_output, &_interleave_output);
264 _memory_group.manage(&_interleave_output);
265
266 // Configure GEMM
267 configure_mm(&_interleave_output, weights, &_gemm_output, true, _are_weights_reshaped);
268 _interleave_output.allocator()->allocate();
Anthony Barbier871448e2017-03-24 14:54:29 +0000269 }
270 else
271 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000272 configure_mm(&_im2col_output, weights, &_gemm_output, false, _are_weights_reshaped);
Anthony Barbierdbdab852017-06-23 15:42:00 +0100273 }
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000274 _im2col_output.allocator()->allocate();
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000275
276 // Configure output stage for quantized case
277 if(_is_quantized)
278 {
279 float multiplier = input->info()->quantization_info().scale * weights->info()->quantization_info().scale / output->info()->quantization_info().scale;
280 int output_multiplier, output_shift;
281 quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift);
282 _gemmlowp_output_stage.configure(&_gemm_output, biases, &_tmp_output, output_multiplier, output_shift, output->info()->quantization_info().offset);
283 _gemm_output.allocator()->allocate();
284 }
285
286 // Configure Col2Im
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000287 _col2im_kernel.configure(_is_quantized ? &_tmp_output : &_gemm_output, output, std::make_pair(conv_w, conv_h));
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000288 if(_is_quantized)
289 {
290 _tmp_output.allocator()->allocate();
291 }
292 else
293 {
294 _gemm_output.allocator()->allocate();
295 }
Kaizen8938bd32017-09-28 14:38:23 +0100296
297 ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(0) != conv_w) || (output->info()->dimension(1) != conv_h), "Output shape does not match the expected one");
298
299 // Allocate intermediate tensor
300 if(!_are_weights_reshaped)
301 {
302 _weights_reshaped.allocator()->allocate();
303 }
Anthony Barbier871448e2017-03-24 14:54:29 +0000304}
305
306void CLConvolutionLayer::run()
307{
308 // Run weights reshaping (Runs once for every configure)
Anthony Barbierdbdab852017-06-23 15:42:00 +0100309 if(!_are_weights_reshaped)
Anthony Barbier871448e2017-03-24 14:54:29 +0000310 {
Anthony Barbierdbdab852017-06-23 15:42:00 +0100311 _are_weights_reshaped = true;
312 _reshape_weights.run();
Anthony Barbier871448e2017-03-24 14:54:29 +0000313 }
314
Kaizen8938bd32017-09-28 14:38:23 +0100315 _memory_group.acquire();
316
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000317 // Run im2col
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000318 CLScheduler::get().enqueue(_im2col_kernel);
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000319
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000320 // Note: _is_interleaved_transposed is true only if the weights passed to the function have been passed already reshaped
321 // and if we do not have QASYMM8 data type. If this flag is true, we need to run the
322 // gemm kernel instead of gemm function
323 if(_is_interleaved_transposed)
Anthony Barbierdbdab852017-06-23 15:42:00 +0100324 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000325 // Run interleave4x4 kernel
326 CLScheduler::get().enqueue(_interleave_kernel);
Anthony Barbier871448e2017-03-24 14:54:29 +0000327
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000328 // Run matrix multiply kernel
329 CLScheduler::get().enqueue(_mm_kernel);
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000330 }
331 else
332 {
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000333 // Runs CLGEMM or CLGEMMLowpMatrixMultiplyCore functions
334 if(_is_quantized)
335 {
336 // Run gemmlowp
337 _mm_gemmlowp.run();
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000338
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000339 // Run output stage
340 _gemmlowp_output_stage.run();
341 }
342 else
343 {
344 // Run gemm
345 _mm_gemm.run();
346 }
Anthony Barbier8140e1e2017-12-14 23:48:46 +0000347 }
Anthony Barbier871448e2017-03-24 14:54:29 +0000348
349 // Reshape output matrix
Anthony Barbierf45d5a92018-01-24 16:23:15 +0000350 CLScheduler::get().enqueue(_col2im_kernel, false);
Kaizen8938bd32017-09-28 14:38:23 +0100351
352 _memory_group.release();
Anthony Barbier871448e2017-03-24 14:54:29 +0000353}