arm_compute v18.01
Change-Id: I9bfa178c2e38bfd5fc812e62aab6760d87748e05
diff --git a/src/runtime/CL/CLPyramid.cpp b/src/runtime/CL/CLPyramid.cpp
index 7cd5518..865f389 100644
--- a/src/runtime/CL/CLPyramid.cpp
+++ b/src/runtime/CL/CLPyramid.cpp
@@ -61,6 +61,7 @@
const bool is_orb_scale = (SCALE_PYRAMID_ORB == _info.scale());
TensorShape tensor_shape = _info.tensor_shape();
+ // Note: Look-up table used by the OpenVX sample implementation
const std::array<float, 4> c_orbscale =
{
{
diff --git a/src/runtime/CL/CLScheduler.cpp b/src/runtime/CL/CLScheduler.cpp
index 71a749f..65292fe 100644
--- a/src/runtime/CL/CLScheduler.cpp
+++ b/src/runtime/CL/CLScheduler.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2017 ARM Limited.
+ * Copyright (c) 2016-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -28,6 +28,8 @@
using namespace arm_compute;
+std::once_flag CLScheduler::_initialize_symbols;
+
CLScheduler::CLScheduler()
: _context(), _queue(), _target(GPUTarget::MIDGARD), _is_initialised(false), _cl_tuner()
{
@@ -35,6 +37,7 @@
CLScheduler &CLScheduler::get()
{
+ std::call_once(_initialize_symbols, opencl_is_available);
static CLScheduler scheduler;
return scheduler;
}
@@ -59,4 +62,4 @@
{
_queue.flush();
}
-}
\ No newline at end of file
+}
diff --git a/src/runtime/CL/CLSubTensor.cpp b/src/runtime/CL/CLSubTensor.cpp
index b228c0a..5f58024 100644
--- a/src/runtime/CL/CLSubTensor.cpp
+++ b/src/runtime/CL/CLSubTensor.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -29,11 +29,11 @@
using namespace arm_compute;
-CLSubTensor::CLSubTensor(ICLTensor *parent, const TensorShape &tensor_shape, const Coordinates &coords)
+CLSubTensor::CLSubTensor(ICLTensor *parent, const TensorShape &tensor_shape, const Coordinates &coords, bool extend_parent)
: _parent(nullptr), _info()
{
ARM_COMPUTE_ERROR_ON(parent == nullptr);
- _info = SubTensorInfo(parent->info(), tensor_shape, coords);
+ _info = SubTensorInfo(parent->info(), tensor_shape, coords, extend_parent);
_parent = parent;
}
diff --git a/src/runtime/CL/functions/CLConvolution.cpp b/src/runtime/CL/functions/CLConvolution.cpp
index a9b0867..0131801 100644
--- a/src/runtime/CL/functions/CLConvolution.cpp
+++ b/src/runtime/CL/functions/CLConvolution.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2017 ARM Limited.
+ * Copyright (c) 2016, 2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -53,7 +53,8 @@
}
template <unsigned int matrix_size>
-void CLConvolutionSquare<matrix_size>::configure(ICLTensor *input, ICLTensor *output, const int16_t *conv, uint32_t scale, BorderMode border_mode, uint8_t constant_border_value)
+void CLConvolutionSquare<matrix_size>::configure(ICLTensor *input, ICLTensor *output, const int16_t *conv, uint32_t scale, BorderMode border_mode,
+ uint8_t constant_border_value)
{
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
ARM_COMPUTE_ERROR_ON(conv == nullptr);
diff --git a/src/runtime/CL/functions/CLConvolutionLayer.cpp b/src/runtime/CL/functions/CLConvolutionLayer.cpp
index 0ed3351..b3af11e 100644
--- a/src/runtime/CL/functions/CLConvolutionLayer.cpp
+++ b/src/runtime/CL/functions/CLConvolutionLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,9 +43,6 @@
void CLConvolutionLayerReshapeWeights::configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, bool transpose1xW)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, output);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(weights, output);
ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
if(biases != nullptr)
@@ -82,13 +79,14 @@
{
_weights_reshape_kernel.configure(weights, biases_to_use, output);
}
+
+ output->info()->set_quantization_info(weights->info()->quantization_info());
}
void CLConvolutionLayerReshapeWeights::run()
{
_memory_group.acquire();
- cl::CommandQueue q = CLScheduler::get().queue();
CLScheduler::get().enqueue(_weights_reshape_kernel);
if(_transpose1xW)
{
@@ -99,33 +97,49 @@
}
CLConvolutionLayer::CLConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
- : _memory_group(memory_manager), _reshape_weights(), _input_im2col_kernel(), _input_interleave_kernel(), _mm_kernel(), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(), _output_col2im_kernel(),
- _input_im2col_reshaped(), _input_interleaved_reshaped(), _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _append_bias(false), _is_fully_connected_convolution(false),
- _are_weights_reshaped(false), _is_quantized(false)
+ : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _interleave_kernel(), _mm_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(),
+ _col2im_kernel(), _im2col_output(), _interleave_output(), _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _are_weights_reshaped(false), _is_quantized(false),
+ _is_interleaved_transposed(false)
{
}
-void CLConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, bool is_interleaved_transposed)
+void CLConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, bool is_interleaved_transposed, bool are_weights_reshaped)
{
if(_is_quantized)
{
- // Since we need negative offsets for computing convolution, we need to change QuantizationInfo()
- // Extract and negate input and weights offset
- const QuantizationInfo input_quantization_info = input->info()->quantization_info();
- const QuantizationInfo weights_quantization_info = weights->info()->quantization_info();
+ if(are_weights_reshaped)
+ {
+ ARM_COMPUTE_ERROR("Weights already reshaped are not suppported with gemmlowp");
+ }
+ else
+ {
+ // Since we need negative offsets for computing convolution, we need to change QuantizationInfo()
+ // Extract and negate input and weights offset
+ const QuantizationInfo input_quantization_info = input->info()->quantization_info();
+ const QuantizationInfo weights_quantization_info = weights->info()->quantization_info();
- input->info()->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset));
- weights->info()->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset));
+ input->info()->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset));
+ weights->info()->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset));
- _mm_gemmlowp.configure(input, weights, output, GEMMInfo(false, false, true /* Reshape weights only for the first run*/));
+ _mm_gemmlowp.configure(input, weights, output, GEMMInfo(false, false, true /* Reshape weights only for the first run*/));
- // Revert back QuantizatioInfo as input and weights could be used in other convolution layers
- input->info()->set_quantization_info(input_quantization_info);
- weights->info()->set_quantization_info(weights_quantization_info);
+ // Revert back QuantizatioInfo as input and weights could be used in other convolution layers
+ input->info()->set_quantization_info(input_quantization_info);
+ weights->info()->set_quantization_info(weights_quantization_info);
+ }
}
else
{
- _mm_kernel.configure(input, weights, output, 1.f, is_interleaved_transposed);
+ if(are_weights_reshaped)
+ {
+ // Configure matrix multiply kernel
+ _mm_kernel.configure(input, weights, output, 1.f, is_interleaved_transposed);
+ }
+ else
+ {
+ // Configure matrix multiply function
+ _mm_gemm.configure(input, weights, nullptr, output, 1.0f, 0.0f, GEMMInfo(false, false, true /* Reshape weights only for the first run*/));
+ }
}
}
@@ -134,6 +148,7 @@
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32);
ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights);
+ ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && CLScheduler::get().target() == GPUTarget::BIFROST);
ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && weights->info()->dimension(2) != input->info()->dimension(2));
ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && is_data_type_quantized_asymmetric(input->info()->data_type()));
@@ -157,14 +172,16 @@
const DataType dt = input->info()->data_type();
- // Set the GPU target for matrix multiply
+ // Set the GPU target for matrix multiply and im2col and col2im
_mm_kernel.set_target(CLScheduler::get().target());
+ _im2col_kernel.set_target(CLScheduler::get().target());
+ _col2im_kernel.set_target(CLScheduler::get().target());
- _append_bias = (biases != nullptr) && (!_is_quantized);
- _are_weights_reshaped = weights_info.are_reshaped();
+ const bool append_bias = (biases != nullptr) && (!_is_quantized);
+ _are_weights_reshaped = weights_info.are_reshaped();
- const unsigned bias_element = (_append_bias) ? 1 : 0;
- const ICLTensor *biases_to_use = (_append_bias) ? biases : nullptr;
+ const unsigned bias_element = (append_bias) ? 1 : 0;
+ const ICLTensor *biases_to_use = (append_bias) ? biases : nullptr;
// Get parameters from conv_info
unsigned int stride_x = 0;
@@ -181,8 +198,8 @@
conv_info);
// Check if its a "fully connected" convolution
- _is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
- const bool run_interleaved = (!_is_fully_connected_convolution && !_is_quantized);
+ const bool is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
+ _is_interleaved_transposed = (!is_fully_connected_convolution) && (!_is_quantized) && (_are_weights_reshaped);
unsigned int mat_weights_cols = weights->info()->dimension(3);
unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element;
@@ -190,7 +207,7 @@
// Reshape weights if needed
if(_are_weights_reshaped)
{
- if(_is_fully_connected_convolution || _is_quantized)
+ if(is_fully_connected_convolution || _is_quantized)
{
mat_weights_cols = weights->info()->dimension(0);
mat_weights_rows = weights->info()->dimension(1);
@@ -204,22 +221,10 @@
}
else
{
- if(_is_fully_connected_convolution || _is_quantized)
- {
- // Create tensor to store the reshaped weights
- TensorShape shape_wr(mat_weights_cols, mat_weights_rows);
- _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_wr));
- _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, false /* 1xW transpose */);
- }
- else
- {
- // Create tensor to store transposed weights
- const float transpose_width = 16.0f / input->info()->element_size();
- TensorShape shape_wt(mat_weights_rows * static_cast<unsigned int>(transpose_width), static_cast<unsigned int>(std::ceil(mat_weights_cols / transpose_width)));
- _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_wt));
- _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, true /* 1xW transpose */);
- }
- _weights_reshaped.info()->set_quantization_info(weights->info()->quantization_info());
+ // _weights_reshaped will be auto configured in the kernel.
+ // Just append biases and do not transpose 1xW as it will be reshaped in CLGEMM
+ _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, false);
+
weights = &_weights_reshaped;
}
@@ -230,50 +235,43 @@
shape_im2col.set(0, mat_input_cols);
shape_im2col.set(1, mat_input_rows);
shape_im2col.set(2, 1);
+ //input->clone() doesn't work with subtensors for grouped convolutions.
TensorInfo im2col_reshaped_info(shape_im2col, 1, dt, input->info()->fixed_point_position());
im2col_reshaped_info.set_quantization_info(input->info()->quantization_info());
- _input_im2col_reshaped.allocator()->init(im2col_reshaped_info);
- _memory_group.manage(&_input_im2col_reshaped);
-
- // Create tensor (interleave) to prepare input tensor for GEMM
- if(run_interleaved)
- {
- TensorShape shape_interleaved = shape_im2col;
- shape_interleaved.set(0, shape_interleaved.x() * 4);
- shape_interleaved.set(1, std::ceil(shape_interleaved.y() / 4.f));
- TensorInfo interleaved_info(shape_interleaved, 1, dt, input->info()->fixed_point_position());
- interleaved_info.set_quantization_info(input->info()->quantization_info());
- _input_interleaved_reshaped.allocator()->init(interleaved_info);
- _memory_group.manage(&_input_interleaved_reshaped);
- }
+ _im2col_output.allocator()->init(im2col_reshaped_info);
+ _memory_group.manage(&_im2col_output);
// Create GEMM output tensor
- TensorShape shape_gemm = _input_im2col_reshaped.info()->tensor_shape();
+ TensorShape shape_gemm = _im2col_output.info()->tensor_shape();
shape_gemm.set(0, mat_weights_cols);
shape_gemm.set(1, mat_input_rows);
const DataType gemm_data_type = _is_quantized ? DataType::S32 : dt;
// GEMM output should be S32 for acquiring raw integer accumulator without quantized postprocessing for quantized asymmetric input.
+ //input->clone() doesn't work with subtensors for grouped convolutions.
TensorInfo info_gemm(shape_gemm, 1, gemm_data_type, input->info()->fixed_point_position());
info_gemm.set_quantization_info(output->info()->quantization_info());
_gemm_output.allocator()->init(info_gemm);
_memory_group.manage(&_gemm_output);
- // Configure kernels
- _input_im2col_kernel.set_target(CLScheduler::get().target());
- _input_im2col_kernel.configure(input, &_input_im2col_reshaped, Size2D(kernel_width, kernel_height), conv_info, _append_bias);
+ // Configure im2col
+ _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, append_bias);
// Configure matrix multiply
- if(run_interleaved)
+ if(_is_interleaved_transposed)
{
- _input_interleave_kernel.configure(&_input_im2col_reshaped, &_input_interleaved_reshaped);
- configure_mm(&_input_interleaved_reshaped, weights, &_gemm_output);
- _input_interleaved_reshaped.allocator()->allocate();
+ // Configure GEMMInterleave4x4. _input_interleaved_reshaped will be auto configured in the kernel
+ _interleave_kernel.configure(&_im2col_output, &_interleave_output);
+ _memory_group.manage(&_interleave_output);
+
+ // Configure GEMM
+ configure_mm(&_interleave_output, weights, &_gemm_output, true, _are_weights_reshaped);
+ _interleave_output.allocator()->allocate();
}
else
{
- configure_mm(&_input_im2col_reshaped, weights, &_gemm_output, false);
+ configure_mm(&_im2col_output, weights, &_gemm_output, false, _are_weights_reshaped);
}
- _input_im2col_reshaped.allocator()->allocate();
+ _im2col_output.allocator()->allocate();
// Configure output stage for quantized case
if(_is_quantized)
@@ -286,8 +284,7 @@
}
// Configure Col2Im
- _output_col2im_kernel.set_target(CLScheduler::get().target());
- _output_col2im_kernel.configure(_is_quantized ? &_tmp_output : &_gemm_output, output, std::make_pair(conv_w, conv_h));
+ _col2im_kernel.configure(_is_quantized ? &_tmp_output : &_gemm_output, output, std::make_pair(conv_w, conv_h));
if(_is_quantized)
{
_tmp_output.allocator()->allocate();
@@ -318,32 +315,39 @@
_memory_group.acquire();
// Run im2col
- CLScheduler::get().enqueue(_input_im2col_kernel);
+ CLScheduler::get().enqueue(_im2col_kernel);
- if(!_is_fully_connected_convolution && !_is_quantized)
+ // Note: _is_interleaved_transposed is true only if the weights passed to the function have been passed already reshaped
+ // and if we do not have QASYMM8 data type. If this flag is true, we need to run the
+ // gemm kernel instead of gemm function
+ if(_is_interleaved_transposed)
{
- // Run interleave4x4
- CLScheduler::get().enqueue(_input_interleave_kernel);
- }
+ // Run interleave4x4 kernel
+ CLScheduler::get().enqueue(_interleave_kernel);
- // Runs matrix multiply on reshaped matrices
- if(_is_quantized)
- {
- _mm_gemmlowp.run();
+ // Run matrix multiply kernel
+ CLScheduler::get().enqueue(_mm_kernel);
}
else
{
- CLScheduler::get().enqueue(_mm_kernel);
- }
+ // Runs CLGEMM or CLGEMMLowpMatrixMultiplyCore functions
+ if(_is_quantized)
+ {
+ // Run gemmlowp
+ _mm_gemmlowp.run();
- // Run output stage for quantized case
- if(_is_quantized)
- {
- _gemmlowp_output_stage.run();
+ // Run output stage
+ _gemmlowp_output_stage.run();
+ }
+ else
+ {
+ // Run gemm
+ _mm_gemm.run();
+ }
}
// Reshape output matrix
- CLScheduler::get().enqueue(_output_col2im_kernel, false);
+ CLScheduler::get().enqueue(_col2im_kernel, false);
_memory_group.release();
}
diff --git a/src/runtime/CL/functions/CLDeconvolutionLayer.cpp b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp
new file mode 100644
index 0000000..1c55722
--- /dev/null
+++ b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2017, 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/CL/functions/CLDeconvolutionLayer.h"
+
+#include "arm_compute/core/Helpers.h"
+#include "arm_compute/core/Utils.h"
+#include "arm_compute/core/Validate.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
+
+#include <memory>
+#include <tuple>
+
+using namespace arm_compute;
+using namespace arm_compute::misc::shape_calculator;
+
+CLDeconvolutionLayer::CLDeconvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
+ : _memory_group(std::move(memory_manager)),
+ _scale_f(),
+ _conv_f(),
+ _scaled_output()
+{
+}
+
+Status CLDeconvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *bias, ITensorInfo *output, const PadStrideInfo &info,
+ unsigned int inner_border_right, unsigned int inner_border_top)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(0) != weights->dimension(1));
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(0) < 1);
+
+ const unsigned int stride_x = info.stride().first;
+ const unsigned int stride_y = info.stride().second;
+
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(inner_border_right > stride_x - 1, "inner_border_right must be smaller than stride_x");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(inner_border_top > stride_y - 1, "inner_border_top must be smaller than stride_y");
+
+ auto out_dims = deconvolution_output_dimensions(input->dimension(0), input->dimension(1), weights->dimension(0), weights->dimension(1),
+ info.pad().first, info.pad().second, inner_border_right, inner_border_top, stride_x, stride_y);
+
+ const TensorShape output_shape = deconvolution_output_shape(out_dims, input->tensor_shape(), weights->tensor_shape());
+
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output, weights, bias);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, output, weights, bias);
+
+ if(bias != nullptr)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, bias);
+ }
+
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimX) != output_shape.x(), "Output's width is invalid.");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimY) != output_shape.y(), "Output's height is invalid.");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimZ) != output_shape.z(), "Output's depth is invalid.");
+
+ TensorInfo scale_out_info(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(compute_deconvolution_shape(*input, stride_x, stride_y, inner_border_right, inner_border_top,
+ info)));
+ const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
+
+ ARM_COMPUTE_RETURN_ON_ERROR(CLDeconvolutionLayerUpsample::validate(input, &scale_out_info, BorderSize(inner_border_right, inner_border_top), info));
+ ARM_COMPUTE_RETURN_ON_ERROR(CLDirectConvolutionLayer::validate(&scale_out_info, weights, bias, output, conv_info));
+
+ return Status{};
+}
+
+void CLDeconvolutionLayer::configure(ICLTensor *input, const ICLTensor *weights, const ICLTensor *bias, ICLTensor *output, const PadStrideInfo &info,
+ unsigned int inner_border_right, unsigned int inner_border_top)
+{
+ ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
+
+ const unsigned int stride_x = info.stride().first;
+ const unsigned int stride_y = info.stride().second;
+
+ auto out_dims = deconvolution_output_dimensions(input->info()->dimension(0), input->info()->dimension(1), weights->info()->dimension(0), weights->info()->dimension(1),
+ info.pad().first, info.pad().second, inner_border_top, inner_border_right, stride_x, stride_y);
+
+ const TensorShape output_shape = deconvolution_output_shape(out_dims, input->info()->tensor_shape(), weights->info()->tensor_shape());
+
+ // Output auto initialization if not yet initialized
+ auto_init_if_empty(*output->info(), output_shape, 1, input->info()->data_type(), input->info()->fixed_point_position());
+
+ // Perform validation step
+ ARM_COMPUTE_ERROR_THROW_ON(CLDeconvolutionLayer::validate(input->info(), weights->info(), bias == nullptr ? nullptr : bias->info(), output->info(), info, inner_border_right, inner_border_top));
+
+ _memory_group.manage(&_scaled_output);
+
+ // configure scale function
+ // Init and allocate intermmidiate tensor for output, same size as input but the first two axis are the same as the output tensor
+ TensorShape scale_out_shape(input->info()->tensor_shape());
+ const unsigned int out_x = input->info()->dimension(0) + (input->info()->dimension(0) - 1) * (stride_x - 1) + inner_border_right + 2 * info.pad().first;
+ const unsigned int out_y = input->info()->dimension(1) + (input->info()->dimension(1) - 1) * (stride_y - 1) + inner_border_top + 2 * info.pad().second;
+ scale_out_shape.set(0, out_x);
+ scale_out_shape.set(1, out_y);
+ TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(), input->info()->fixed_point_position());
+ _scaled_output.allocator()->init(scale_out_info);
+
+ _scale_f.configure(input, &_scaled_output, BorderSize(inner_border_top, inner_border_right), info);
+
+ // setup the function to convolve the upscaled output
+ const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
+ _conv_f.configure(&_scaled_output, weights, bias, output, conv_info);
+ _scaled_output.allocator()->allocate();
+}
+
+void CLDeconvolutionLayer::run()
+{
+ _memory_group.acquire();
+ _scale_f.run();
+ _conv_f.run();
+ _memory_group.release();
+}
diff --git a/src/runtime/CL/functions/CLDeconvolutionLayerUpsample.cpp b/src/runtime/CL/functions/CLDeconvolutionLayerUpsample.cpp
new file mode 100644
index 0000000..13a24f8
--- /dev/null
+++ b/src/runtime/CL/functions/CLDeconvolutionLayerUpsample.cpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2017, 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/CL/functions/CLDeconvolutionLayerUpsample.h"
+
+#include "arm_compute/core/CL/OpenCL.h"
+#include "arm_compute/core/Utils.h"
+#include "arm_compute/runtime/CL/CLScheduler.h"
+
+#include <cmath>
+#include <memory>
+#include <tuple>
+
+using namespace arm_compute;
+
+CLDeconvolutionLayerUpsample::CLDeconvolutionLayerUpsample() // NOLINT
+ : _upsample(),
+ _output(nullptr)
+{
+}
+
+Status CLDeconvolutionLayerUpsample::validate(const ITensorInfo *input, const ITensorInfo *output, const BorderSize &inner_border,
+ const PadStrideInfo &info)
+{
+ return CLDeconvolutionLayerUpsampleKernel::validate(input, output, inner_border, info);
+}
+
+void CLDeconvolutionLayerUpsample::configure(ICLTensor *input, ICLTensor *output, const BorderSize &inner_border,
+ const PadStrideInfo &info)
+{
+ ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
+
+ _output = output;
+ _upsample.configure(input, _output, inner_border, info);
+}
+
+void CLDeconvolutionLayerUpsample::run()
+{
+ _output->map(CLScheduler::get().queue(), true);
+ memset(_output->buffer(), 0, _output->info()->total_size());
+ _output->unmap(CLScheduler::get().queue());
+
+ CLScheduler::get().enqueue(_upsample, false);
+}
diff --git a/src/runtime/CL/functions/CLFullyConnectedLayer.cpp b/src/runtime/CL/functions/CLFullyConnectedLayer.cpp
index 7fd81cd..68c6576 100644
--- a/src/runtime/CL/functions/CLFullyConnectedLayer.cpp
+++ b/src/runtime/CL/functions/CLFullyConnectedLayer.cpp
@@ -25,6 +25,7 @@
#include "arm_compute/core/Size2D.h"
#include "arm_compute/core/Validate.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
#include "arm_compute/runtime/CL/CLScheduler.h"
#include "support/ToolchainSupport.h"
@@ -32,6 +33,34 @@
#include <algorithm>
using namespace arm_compute;
+using namespace arm_compute::misc::shape_calculator;
+
+namespace
+{
+Status validate_mm(const ITensorInfo &input, const ITensorInfo &weights, const ITensorInfo &output, bool is_interleaved_transposed)
+{
+ const GPUTarget gpu_target = CLScheduler::get().target();
+
+ if(is_data_type_quantized_asymmetric(input.data_type()))
+ {
+ // Since we need negative offsets for computing convolution, we need to change QuantizationInfo()
+ // Extract and negate input and weights offset
+ const QuantizationInfo input_quantization_info(input.quantization_info().scale, -input.quantization_info().offset);
+ const QuantizationInfo weights_quantization_info(weights.quantization_info().scale, -weights.quantization_info().offset);
+
+ // Validate gemmlowp function
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpMatrixMultiplyCore::validate(&input.clone()->set_quantization_info(input_quantization_info),
+ &weights.clone()->set_quantization_info(weights_quantization_info),
+ &output));
+ }
+ else
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixMultiplyKernel::validate(&input, &weights, &output, 1.f, is_interleaved_transposed, gpu_target));
+ }
+
+ return Status{};
+}
+} // namespace
void CLFullyConnectedLayerReshapeWeights::configure(const ICLTensor *input, ICLTensor *output)
{
@@ -40,6 +69,11 @@
_kernel = std::move(k);
}
+Status CLFullyConnectedLayerReshapeWeights::validate(const ITensorInfo *input, const ITensorInfo *output)
+{
+ return CLTransposeKernel::validate(input, output);
+}
+
CLFullyConnectedLayer::CLFullyConnectedLayer(std::shared_ptr<IMemoryManager> memory_manager)
: _memory_group(memory_manager), _im2col_kernel(), _reshape_weights_kernel(), _mm_kernel(), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(), _accumulate_biases_kernel(), _im2col_output(),
_gemmlowp_output(), _reshape_weights_output(), _are_weights_reshaped(true), _is_fc_after_conv(true), _accumulate_biases(false), _is_quantized(false)
@@ -80,8 +114,7 @@
// If the fully connected layer is called after a convolution layer, the input tensor must be linearized
// Initialize output tensor for im2col
- TensorShape shape_im2col = input->info()->tensor_shape();
- shape_im2col.collapse(3);
+ TensorShape shape_im2col = compute_im2col_shape(*input->info());
_im2col_output.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col));
// Configure im2col kernel
@@ -105,9 +138,15 @@
void CLFullyConnectedLayer::configure(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, bool transpose_weights, bool are_weights_reshaped)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights, output);
- ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 2);
+ ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
+
+ // Perform validate step
+ ARM_COMPUTE_ERROR_THROW_ON(CLFullyConnectedLayer::validate(input->info(),
+ weights->info(),
+ biases != nullptr ? biases->info() : nullptr,
+ output->info(),
+ transpose_weights,
+ are_weights_reshaped));
_are_weights_reshaped = transpose_weights ? are_weights_reshaped : true;
_is_fc_after_conv = true;
@@ -192,6 +231,86 @@
}
}
+Status CLFullyConnectedLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, bool transpose_weights, bool are_weights_reshaped)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 2);
+
+ bool weights_reshaped = transpose_weights ? are_weights_reshaped : true;
+ bool is_fc_after_conv = true;
+ bool is_quantized = is_data_type_quantized_asymmetric(input->data_type());
+ const GPUTarget gpu_target = CLScheduler::get().target();
+
+ const ITensorInfo &im2col_input = TensorInfo(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(compute_im2col_shape(*input)));
+ const ITensorInfo &reshaped_weights = TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(compute_transposed_shape(*weights)));
+ const ITensorInfo &gemmlowp_output = TensorInfo(output->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32));
+
+ // Configure accumulate biases kernel for non quantized asymmetric types
+ if(biases != nullptr && !is_quantized)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixAccumulateBiasesKernel::validate(output, biases, gpu_target));
+ }
+
+ // With the Fully Connected layer we can have 4 different cases:
+ // 1) Convolution layer -> Fully Connected layer without batches
+ // 2) Fully Connected layer -> Fully Connected layer without batches
+ // 3) Convolution layer -> Fully Connected layer with batches
+ // 4) Fully Connected layer -> Fully Connected layer with batches
+
+ const ITensorInfo *input_to_use = input;
+ const ITensorInfo *weights_to_use = weights;
+ const ITensorInfo *tmp_output = (is_quantized) ? &gemmlowp_output : output;
+
+ if(!weights_reshaped)
+ {
+ // Validate reshape weights kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayerReshapeWeights::validate(weights, &reshaped_weights));
+ weights_to_use = &reshaped_weights;
+ }
+
+ // Check if we have a fully connected layer with batches
+ const bool is_batched_fc_layer = output->dimension(1) > 1;
+
+ if(is_batched_fc_layer)
+ {
+ is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) && (std::equal(input->tensor_shape().cbegin() + 3,
+ input->tensor_shape().cend(),
+ output->tensor_shape().cbegin() + 1));
+ }
+ else
+ {
+ is_fc_after_conv = input->num_dimensions() > 1;
+ }
+
+ if(is_fc_after_conv)
+ {
+ // Fully Connected layer after a Convolution Layer without batches
+ ARM_COMPUTE_RETURN_ERROR_ON((weights_to_use->dimension(1) != (input->dimension(0) * input->dimension(1) * input->dimension(2))));
+
+ // Validate im2col kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(CLIm2ColKernel::validate(input, &im2col_input, Size2D(1, 1), PadStrideInfo(1, 1, 0, 0), false));
+ input_to_use = &im2col_input;
+ }
+ else
+ {
+ // Fully Connected layer after a Fully Connected Layer without batches
+ ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != weights_to_use->dimension(1));
+ }
+ // Validate matrix multiply kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(*input_to_use, *weights_to_use, *tmp_output, false));
+
+ // Validate output stage for asymmetric quantized types
+ if(is_quantized)
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint::validate(&gemmlowp_output, biases, output));
+ }
+
+ return Status{};
+}
+
void CLFullyConnectedLayer::run()
{
// Reshape of the weights (happens only once)
diff --git a/src/runtime/CL/functions/CLGEMM.cpp b/src/runtime/CL/functions/CLGEMM.cpp
index ca0228f..c676a10 100644
--- a/src/runtime/CL/functions/CLGEMM.cpp
+++ b/src/runtime/CL/functions/CLGEMM.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -39,20 +39,23 @@
using namespace arm_compute;
CLGEMM::CLGEMM(std::shared_ptr<IMemoryManager> memory_manager)
- : _memory_group(std::move(memory_manager)), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _ma_kernel(), _tmp_a(), _tmp_b(), _is_interleaved_transposed(false), _run_addition(false)
+ : _memory_group(std::move(memory_manager)), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _ma_kernel(), _tmp_a(), _tmp_b(), _is_interleaved_transposed(false), _run_addition(false),
+ _is_first_run(true), _reshape_b_only_on_first_run(false)
{
}
-void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta)
+void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info)
{
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, b, output);
+ ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported");
+ ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported");
if(c != nullptr)
{
ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, c);
ARM_COMPUTE_ERROR_ON_MSG(a->info()->dimension(1) != c->info()->dimension(1), "The C matrix must have the same number of rows as the matrix A");
- ARM_COMPUTE_ERROR_ON_MSG(b->info()->dimension(0) != c->info()->dimension(0), "The C matrix must have the same number of columns as the matrix C");
+ ARM_COMPUTE_ERROR_ON_MSG(b->info()->dimension(0) != c->info()->dimension(0), "The C matrix must have the same number of columns as the matrix B");
ARM_COMPUTE_ERROR_ON_MSG(c->info()->dimension(0) != output->info()->dimension(0), "The C matrix must have the same number of rows as the output matrix");
ARM_COMPUTE_ERROR_ON_MSG(c->info()->dimension(1) != output->info()->dimension(1), "The C matrix must have the same number of columns as the output matrix");
}
@@ -60,7 +63,11 @@
ARM_COMPUTE_ERROR_ON_MSG(a->info()->dimension(0) != b->info()->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
// If the input tensor has less than 16 rows, we run a special version of GEMM without reshaping the input tensors
- _is_interleaved_transposed = a->info()->dimension(1) > 16;
+ // For Bifrost architectures we do not reshape the input matrices
+ _is_interleaved_transposed = (a->info()->dimension(1) > 16 && CLScheduler::get().target() != GPUTarget::BIFROST);
+
+ // Check if we need to reshape the matrix B only on the first run
+ _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run();
const ICLTensor *matrix_a = a;
const ICLTensor *matrix_b = b;
@@ -73,31 +80,17 @@
matrix_a = &_tmp_a;
matrix_b = &_tmp_b;
- TensorShape shape_tmp_a = a->info()->tensor_shape();
- TensorShape shape_tmp_b = b->info()->tensor_shape();
-
- shape_tmp_a.set(0, a->info()->dimension(0) * 4);
- shape_tmp_a.set(1, std::ceil(a->info()->dimension(1) / 4.0f));
-
- const unsigned int transpose_w = max_cl_vector_width / data_size_from_type(b->info()->data_type());
- shape_tmp_b.set(0, b->info()->dimension(1) * transpose_w);
- shape_tmp_b.set(1, std::ceil(b->info()->dimension(0) / static_cast<float>(transpose_w)));
-
- TensorInfo info_a(shape_tmp_a, 1, a->info()->data_type(), a->info()->fixed_point_position());
- _tmp_a.allocator()->init(info_a);
-
- TensorInfo info_b(shape_tmp_b, 1, b->info()->data_type(), b->info()->fixed_point_position());
- _tmp_b.allocator()->init(info_b);
-
- // Manage intermediate buffers
- _memory_group.manage(&_tmp_a);
- _memory_group.manage(&_tmp_b);
+ // _tmp_a and _tmp_b will be auto configured in _interleave_kernel and in _transpose_kernel
// Configure interleave kernel
_interleave_kernel.configure(a, &_tmp_a);
// Configure transpose kernel
_transpose_kernel.configure(b, &_tmp_b);
+
+ // Manage intermediate buffers
+ _memory_group.manage(&_tmp_a);
+ _memory_group.manage(&_tmp_b);
}
_mm_kernel.configure(matrix_a, matrix_b, output, alpha, _is_interleaved_transposed);
@@ -126,8 +119,18 @@
// Run interleave kernel
CLScheduler::get().enqueue(_interleave_kernel, false);
- // Run transpose kernel
- CLScheduler::get().enqueue(_transpose_kernel, false);
+ if(_is_first_run)
+ {
+ // Run transpose kernel
+ CLScheduler::get().enqueue(_transpose_kernel, false);
+
+ _is_first_run = false;
+ }
+ else if(!_reshape_b_only_on_first_run)
+ {
+ // Run transpose kernel
+ CLScheduler::get().enqueue(_transpose_kernel, false);
+ }
}
// Run matrix multiply kernel
diff --git a/src/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.cpp b/src/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.cpp
index 5c6f5b4..ddcab6a 100644
--- a/src/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.cpp
+++ b/src/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.cpp
@@ -29,9 +29,11 @@
#include "arm_compute/core/TensorInfo.h"
#include "arm_compute/core/Types.h"
#include "arm_compute/core/Validate.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
#include "arm_compute/runtime/CL/CLScheduler.h"
using namespace arm_compute;
+using namespace arm_compute::misc::shape_calculator;
CLGEMMLowpMatrixMultiplyCore::CLGEMMLowpMatrixMultiplyCore(std::shared_ptr<IMemoryManager> memory_manager)
: _memory_group(std::move(memory_manager)), _mm_kernel(), _mtx_a_reshape_kernel(), _mtx_b_reshape_kernel(), _mtx_a_reduction_kernel(), _mtx_b_reduction_kernel(), _offset_contribution_kernel(),
@@ -41,14 +43,9 @@
void CLGEMMLowpMatrixMultiplyCore::configure(const ICLTensor *a, const ICLTensor *b, ICLTensor *output, const GEMMInfo &gemm_info)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QASYMM8);
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::S32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, b);
- ARM_COMPUTE_ERROR_ON_MSG((a)->info()->dimension(0) != (b)->info()->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
- ARM_COMPUTE_ERROR_ON_MSG((a)->info()->dimension(1) != (output)->info()->dimension(1), "The output matrix must have the same number of rows as the matrix A");
- ARM_COMPUTE_ERROR_ON_MSG((b)->info()->dimension(0) != (output)->info()->dimension(0), "The output matrix must have the same number of columns as the matrix B");
- ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported");
- ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported");
+ ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output);
+ ARM_COMPUTE_UNUSED(gemm_info);
+ ARM_COMPUTE_ERROR_THROW_ON(CLGEMMLowpMatrixMultiplyCore::validate(a->info(), b->info(), output->info(), gemm_info));
_reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run();
_a_offset = a->info()->quantization_info().offset;
@@ -65,18 +62,8 @@
matrix_a = &_tmp_a;
matrix_b = &_tmp_b;
- // The interleaved output matrix will have the following shape: [ a_height * 4, ceil(a_width / 4.0f) ]
- TensorShape shape_tmp_a = a->info()->tensor_shape();
- shape_tmp_a.set(0, a->info()->dimension(0) * 4);
- shape_tmp_a.set(1, std::ceil(a->info()->dimension(1) / 4.f));
-
- // The transpose1xW output matrix will have the following shape: [ b_height * 16, ceil(b_width / 16.0f) ]
- TensorShape shape_tmp_b = b->info()->tensor_shape();
- shape_tmp_b.set(0, b->info()->dimension(1) * 16);
- shape_tmp_b.set(1, std::ceil(b->info()->dimension(0) / 16.f));
-
- TensorInfo info_a(shape_tmp_a, 1, a->info()->data_type());
- TensorInfo info_b(shape_tmp_b, 1, b->info()->data_type());
+ TensorInfo info_a(compute_interleaved_shape(*a->info()), 1, a->info()->data_type());
+ TensorInfo info_b(compute_transpose1xW_shape(*b->info()), 1, b->info()->data_type());
_tmp_a.allocator()->init(info_a);
_tmp_b.allocator()->init(info_b);
_memory_group.manage(&_tmp_a);
@@ -95,13 +82,7 @@
// Initialize matrix B reduction kernel only if _a_offset is not equal to 0
if(_a_offset != 0)
{
- TensorShape shape_vector_sum_col = b->info()->tensor_shape();
-
- if(shape_vector_sum_col.num_dimensions() > 1)
- {
- shape_vector_sum_col.remove_dimension(1);
- }
- TensorInfo info_vector_sum_col(shape_vector_sum_col, 1, DataType::S32);
+ TensorInfo info_vector_sum_col(compute_reductionA_shape(*b->info()), 1, DataType::S32);
_vector_sum_col.allocator()->init(info_vector_sum_col);
_memory_group.manage(&_vector_sum_col);
@@ -112,13 +93,7 @@
// Initialize Matrix A reduction kernel only if _b_offset is not equal to 0
if(_b_offset != 0)
{
- TensorShape shape_vector_sum_row = a->info()->tensor_shape();
- shape_vector_sum_row.set(Window::DimX, a->info()->dimension(1));
- if(a->info()->num_dimensions() > 1)
- {
- shape_vector_sum_row.remove_dimension(1);
- }
- TensorInfo info_vector_sum_row(shape_vector_sum_row, 1, DataType::S32);
+ TensorInfo info_vector_sum_row(compute_reductionB_shape(*a->info()), 1, DataType::S32);
_vector_sum_row.allocator()->init(info_vector_sum_row);
_memory_group.manage(&_vector_sum_row);
@@ -147,6 +122,67 @@
}
}
+Status CLGEMMLowpMatrixMultiplyCore::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *output, const GEMMInfo &gemm_info)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QASYMM8);
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::S32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, b);
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((a)->dimension(0) != (b)->dimension(1),
+ "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((a)->dimension(1) != (output)->dimension(1),
+ "The output matrix must have the same number of rows as the matrix A");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((b)->dimension(0) != (output)->dimension(0),
+ "The output matrix must have the same number of columns as the matrix B");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported");
+
+ int32_t a_offset = a->quantization_info().offset;
+ int32_t b_offset = b->quantization_info().offset;
+ bool is_interleaved_transposed = a->dimension(1) > 16;
+
+ if(is_interleaved_transposed)
+ {
+ TensorInfo info_a(compute_interleaved_shape(*a), 1, a->data_type());
+ TensorInfo info_b(compute_transpose1xW_shape(*b), 1, b->data_type());
+
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMInterleave4x4Kernel::validate(a, &info_a));
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMTranspose1xWKernel::validate(b, &info_b));
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpMatrixMultiplyKernel::validate(&info_a, &info_b, output));
+ }
+ else
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpMatrixMultiplyKernel::validate(a, b, output));
+ }
+
+ TensorInfo info_vector_sum_col, info_vector_sum_row;
+
+ // Validate matrix B reduction kernel only if _a_offset is not equal to 0
+ if(a_offset != 0)
+ {
+ info_vector_sum_col = TensorInfo(compute_reductionA_shape(*b), 1, DataType::S32);
+
+ // Configure Matrix B reduction kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpMatrixBReductionKernel::validate(b, &info_vector_sum_col));
+ }
+
+ // Validate Matrix A reduction kernel only if _b_offset is not equal to 0
+ if(b_offset != 0)
+ {
+ info_vector_sum_row = TensorInfo(compute_reductionB_shape(*a), 1, DataType::S32);
+
+ // Configure matrix A reduction kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpMatrixAReductionKernel::validate(a, &info_vector_sum_row));
+ }
+
+ // Validate offset contribution kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMLowpOffsetContributionKernel::validate(output,
+ a_offset == 0 ? nullptr : &info_vector_sum_col,
+ b_offset == 0 ? nullptr : &info_vector_sum_row,
+ a_offset, b_offset));
+
+ return Status{};
+}
+
void CLGEMMLowpMatrixMultiplyCore::run()
{
_memory_group.acquire();
diff --git a/src/runtime/CL/functions/CLHarrisCorners.cpp b/src/runtime/CL/functions/CLHarrisCorners.cpp
index 059528f..423faea 100644
--- a/src/runtime/CL/functions/CLHarrisCorners.cpp
+++ b/src/runtime/CL/functions/CLHarrisCorners.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2017 ARM Limited.
+ * Copyright (c) 2016-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -63,8 +63,9 @@
void CLHarrisCorners::configure(ICLImage *input, float threshold, float min_dist,
float sensitivity, int32_t gradient_size, int32_t block_size, ICLKeyPointArray *corners,
- BorderMode border_mode, uint8_t constant_border_value)
+ BorderMode border_mode, uint8_t constant_border_value, bool use_fp16)
{
+ ARM_COMPUTE_UNUSED(use_fp16);
ARM_COMPUTE_ERROR_ON_TENSOR_NOT_2D(input);
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
ARM_COMPUTE_ERROR_ON(!(block_size == 3 || block_size == 5 || block_size == 7));
diff --git a/src/runtime/CL/functions/CLMagnitude.cpp b/src/runtime/CL/functions/CLMagnitude.cpp
index 9d6ac7a..b1284db 100644
--- a/src/runtime/CL/functions/CLMagnitude.cpp
+++ b/src/runtime/CL/functions/CLMagnitude.cpp
@@ -32,7 +32,7 @@
void CLMagnitude::configure(const ICLTensor *input1, const ICLTensor *input2, ICLTensor *output, MagnitudeType mag_type, bool use_fp16)
{
- ARM_COMPUTE_UNUSED(use_fp16); //TODO(COMPMID-644): Add half float support
+ ARM_COMPUTE_UNUSED(use_fp16);
auto k = arm_compute::support::cpp14::make_unique<CLMagnitudePhaseKernel>();
k->configure(input1, input2, output, nullptr, mag_type);
diff --git a/src/runtime/CL/functions/CLPermute.cpp b/src/runtime/CL/functions/CLPermute.cpp
new file mode 100644
index 0000000..f23e231
--- /dev/null
+++ b/src/runtime/CL/functions/CLPermute.cpp
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/CL/functions/CLPermute.h"
+
+#include "arm_compute/core/CL/ICLTensor.h"
+#include "arm_compute/core/CL/kernels/CLPermuteKernel.h"
+#include "support/ToolchainSupport.h"
+
+using namespace arm_compute;
+
+void CLPermute::configure(const ICLTensor *input, ICLTensor *output, const PermutationVector &perm)
+{
+ auto k = arm_compute::support::cpp14::make_unique<CLPermuteKernel>();
+ k->configure(input, output, perm);
+ _kernel = std::move(k);
+}
\ No newline at end of file
diff --git a/src/runtime/CL/functions/CLPoolingLayer.cpp b/src/runtime/CL/functions/CLPoolingLayer.cpp
index 2341633..201bf87 100644
--- a/src/runtime/CL/functions/CLPoolingLayer.cpp
+++ b/src/runtime/CL/functions/CLPoolingLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -40,16 +40,14 @@
k->configure(input, output, pool_info);
_kernel = std::move(k);
- // Configure border depending on operation required
+ // Configure border depending on operation required (quantize border in case of asymmetric data_type)
BorderMode border_mode = (PoolingType::MAX == pool_info.pool_type()) ? BorderMode::REPLICATE : BorderMode::CONSTANT;
- // Quantize border in case data type is quantized asymmetric data type
- uint32_t border_value = 0;
+ PixelValue zero_value(0.f);
if(is_data_type_quantized_asymmetric(input->info()->data_type()) && !pool_info.exclude_padding())
{
- border_value = static_cast<uint32_t>(input->info()->quantization_info().quantize(0.f, RoundingPolicy::TO_NEAREST_UP));
+ zero_value = PixelValue(static_cast<uint32_t>(input->info()->quantization_info().offset));
}
-
- _border_handler.configure(input, _kernel->border_size(), border_mode, PixelValue(border_value));
+ _border_handler.configure(input, _kernel->border_size(), border_mode, zero_value);
}
Status CLPoolingLayer::validate(const ITensorInfo *input, const ITensorInfo *output, const PoolingLayerInfo &pool_info)
diff --git a/src/runtime/GLES_COMPUTE/GCScheduler.cpp b/src/runtime/GLES_COMPUTE/GCScheduler.cpp
index b2235ea..fcc8559 100644
--- a/src/runtime/GLES_COMPUTE/GCScheduler.cpp
+++ b/src/runtime/GLES_COMPUTE/GCScheduler.cpp
@@ -28,11 +28,27 @@
using namespace arm_compute;
-GCScheduler::GCScheduler() = default;
+std::once_flag GCScheduler::_initialize_symbols;
+
+GCScheduler::GCScheduler()
+ : _display(EGL_NO_DISPLAY), _context(EGL_NO_CONTEXT)
+{
+}
+
+GCScheduler::~GCScheduler()
+{
+ eglDestroyContext(_display, _context);
+ eglTerminate(_display);
+
+ _context = EGL_NO_CONTEXT;
+ _display = EGL_NO_DISPLAY;
+}
void GCScheduler::default_init()
{
- GCKernelLibrary::get().init("./cs_shaders/");
+ setup_context();
+
+ GCKernelLibrary::get().init("./cs_shaders/", _display, _context);
}
void GCScheduler::init(EGLDisplay dpy, EGLContext ctx)
@@ -42,11 +58,12 @@
GCScheduler &GCScheduler::get()
{
+ std::call_once(_initialize_symbols, opengles31_is_available);
static GCScheduler scheduler;
return scheduler;
}
-void GCScheduler::enqueue(IGCKernel &kernel, bool flush)
+void GCScheduler::dispatch(IGCKernel &kernel, bool flush)
{
kernel.run(kernel.window());
if(flush)
@@ -55,7 +72,60 @@
}
}
-void GCScheduler::sync()
+void GCScheduler::memory_barrier()
{
ARM_COMPUTE_GL_CHECK(glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT));
}
+
+void GCScheduler::setup_context()
+{
+ EGLBoolean res;
+ _display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+
+ ARM_COMPUTE_ERROR_ON_MSG(_display == EGL_NO_DISPLAY, "Failed to get display: 0x%x.", eglGetError());
+
+ res = eglInitialize(_display, nullptr, nullptr);
+
+ ARM_COMPUTE_ERROR_ON_MSG(res == EGL_FALSE, "Failed to initialize egl: 0x%x.", eglGetError());
+ ARM_COMPUTE_UNUSED(res);
+
+ const char *egl_extension_st = eglQueryString(_display, EGL_EXTENSIONS);
+ ARM_COMPUTE_ERROR_ON_MSG((strstr(egl_extension_st, "EGL_KHR_create_context") == nullptr), "Failed to query EGL_KHR_create_context");
+ ARM_COMPUTE_ERROR_ON_MSG((strstr(egl_extension_st, "EGL_KHR_surfaceless_context") == nullptr), "Failed to query EGL_KHR_surfaceless_context");
+ ARM_COMPUTE_UNUSED(egl_extension_st);
+
+ const EGLint config_attribs[] =
+ {
+ EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT_KHR,
+ EGL_NONE
+ };
+ EGLConfig cfg;
+ EGLint count;
+
+ res = eglChooseConfig(_display, config_attribs, &cfg, 1, &count);
+
+ ARM_COMPUTE_ERROR_ON_MSG(res == EGL_FALSE, "Failed to choose config: 0x%x.", eglGetError());
+ ARM_COMPUTE_UNUSED(res);
+
+ res = eglBindAPI(EGL_OPENGL_ES_API);
+
+ ARM_COMPUTE_ERROR_ON_MSG(res == EGL_FALSE, "Failed to bind api: 0x%x.", eglGetError());
+
+ const EGLint attribs[] =
+ {
+ EGL_CONTEXT_CLIENT_VERSION, 3,
+ EGL_NONE
+ };
+ _context = eglCreateContext(_display,
+ cfg,
+ EGL_NO_CONTEXT,
+ attribs);
+
+ ARM_COMPUTE_ERROR_ON_MSG(_context == EGL_NO_CONTEXT, "Failed to create context: 0x%x.", eglGetError());
+ ARM_COMPUTE_UNUSED(res);
+
+ res = eglMakeCurrent(_display, EGL_NO_SURFACE, EGL_NO_SURFACE, _context);
+
+ ARM_COMPUTE_ERROR_ON_MSG(res == EGL_FALSE, "Failed to make current: 0x%x.", eglGetError());
+ ARM_COMPUTE_UNUSED(res);
+}
diff --git a/src/runtime/GLES_COMPUTE/IGCSimpleFunction.cpp b/src/runtime/GLES_COMPUTE/IGCSimpleFunction.cpp
index 199ee46..f2926b0 100644
--- a/src/runtime/GLES_COMPUTE/IGCSimpleFunction.cpp
+++ b/src/runtime/GLES_COMPUTE/IGCSimpleFunction.cpp
@@ -38,7 +38,7 @@
{
ARM_COMPUTE_ERROR_ON_MSG(!_kernel, "The child class didn't set the GLES kernel or function isn't configured");
- GCScheduler::get().enqueue(_border_handler, false);
- GCScheduler::get().sync();
- GCScheduler::get().enqueue(*_kernel);
+ GCScheduler::get().dispatch(_border_handler, false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(*_kernel);
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCArithmeticAddition.cpp b/src/runtime/GLES_COMPUTE/functions/GCArithmeticAddition.cpp
new file mode 100755
index 0000000..b99716b
--- /dev/null
+++ b/src/runtime/GLES_COMPUTE/functions/GCArithmeticAddition.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2016, 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/GLES_COMPUTE/functions/GCArithmeticAddition.h"
+
+#include "arm_compute/core/GLES_COMPUTE/kernels/GCArithmeticAdditionKernel.h"
+#include "support/ToolchainSupport.h"
+
+#include <utility>
+
+using namespace arm_compute;
+
+void GCArithmeticAddition::configure(const IGCTensor *input1, const IGCTensor *input2, IGCTensor *output, ConvertPolicy policy)
+{
+ auto k = arm_compute::support::cpp14::make_unique<GCArithmeticAdditionKernel>();
+ k->configure(input1, input2, output, policy);
+ _kernel = std::move(k);
+}
+
+Status GCArithmeticAddition::validate(const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, ConvertPolicy policy)
+{
+ return GCArithmeticAdditionKernel::validate(input1, input2, output, policy);
+}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCBatchNormalizationLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCBatchNormalizationLayer.cpp
index 2e546a6..99bdf43 100755
--- a/src/runtime/GLES_COMPUTE/functions/GCBatchNormalizationLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCBatchNormalizationLayer.cpp
@@ -44,5 +44,5 @@
void GCBatchNormalizationLayer::run()
{
- GCScheduler::get().enqueue(_norm_kernel, true);
+ GCScheduler::get().dispatch(_norm_kernel, true);
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCConvolutionLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCConvolutionLayer.cpp
new file mode 100644
index 0000000..5689722
--- /dev/null
+++ b/src/runtime/GLES_COMPUTE/functions/GCConvolutionLayer.cpp
@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2017-2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "arm_compute/runtime/GLES_COMPUTE/functions/GCConvolutionLayer.h"
+
+#include "arm_compute/core/PixelValue.h"
+#include "arm_compute/core/Size2D.h"
+#include "arm_compute/core/Utils.h"
+#include "arm_compute/core/Validate.h"
+#include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
+
+#include <cmath>
+#include <memory>
+#include <tuple>
+
+using namespace arm_compute;
+
+GCConvolutionLayerReshapeWeights::GCConvolutionLayerReshapeWeights()
+ : _weights_reshape_kernel(), _weights_transposed_kernel(), _weights_reshaped(), _transpose1xW(false)
+{
+}
+
+void GCConvolutionLayerReshapeWeights::configure(const IGCTensor *weights, const IGCTensor *biases, IGCTensor *output, bool transpose1xW)
+{
+ ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::F16, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, output);
+ ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(weights->info()->data_type()));
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
+ ARM_COMPUTE_ERROR_ON(biases->info()->dimension(0) != weights->info()->dimension(3));
+ ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
+ }
+
+ const bool append_biases = (biases != nullptr) && !is_data_type_quantized_asymmetric(weights->info()->data_type());
+ const unsigned bias_element = (append_biases) ? 1 : 0;
+ const IGCTensor *biases_to_use = (append_biases) ? biases : nullptr;
+
+ _transpose1xW = transpose1xW;
+
+ if(transpose1xW)
+ {
+ // Create tensor to store the reshaped weights
+ const unsigned int mat_weights_cols = weights->info()->dimension(3);
+ const unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element;
+ TensorShape shape_wr(mat_weights_cols, mat_weights_rows);
+ const DataType dt = weights->info()->data_type();
+ const int fixed_point_position = weights->info()->fixed_point_position();
+ TensorInfo info_wr(shape_wr, 1, dt, fixed_point_position);
+
+ _weights_reshaped.allocator()->init(info_wr);
+ _weights_reshape_kernel.configure(weights, biases_to_use, &_weights_reshaped);
+ _weights_transposed_kernel.configure(&_weights_reshaped, output);
+ _weights_reshaped.allocator()->allocate();
+ }
+ else
+ {
+ _weights_reshape_kernel.configure(weights, biases_to_use, output);
+ }
+}
+
+void GCConvolutionLayerReshapeWeights::run()
+{
+ GCScheduler::get().dispatch(_weights_reshape_kernel);
+ if(_transpose1xW)
+ {
+ GCScheduler::get().dispatch(_weights_transposed_kernel);
+ }
+}
+
+GCConvolutionLayer::GCConvolutionLayer()
+ : _reshape_weights(), _input_im2col_kernel(), _input_interleave_kernel(), _mm_kernel(), _output_col2im_kernel(), _fill_border(), _input_im2col_reshaped(), _input_interleaved_reshaped(),
+ _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _append_bias(false), _is_fully_connected_convolution(false), _are_weights_reshaped(false)
+{
+}
+
+void GCConvolutionLayer::configure_mm(const IGCTensor *input, const IGCTensor *weights, IGCTensor *output, bool is_interleaved_transposed)
+{
+ _mm_kernel.configure(input, weights, output, 1.f, is_interleaved_transposed);
+}
+
+void GCConvolutionLayer::configure(const IGCTensor *input, const IGCTensor *weights, const IGCTensor *biases, IGCTensor *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info)
+{
+ ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
+ ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && weights->info()->dimension(2) != input->info()->dimension(2));
+ ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
+ ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && biases->info()->dimension(0) != weights->info()->dimension(3));
+ ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
+ }
+
+ const DataType dt = input->info()->data_type();
+
+ _append_bias = (biases != nullptr);
+ _are_weights_reshaped = weights_info.are_reshaped();
+
+ const unsigned bias_element = (_append_bias) ? 1 : 0;
+ const IGCTensor *biases_to_use = (_append_bias) ? biases : nullptr;
+
+ // Get parameters from conv_info
+ unsigned int stride_x = 0;
+ unsigned int stride_y = 0;
+ std::tie(stride_x, stride_y) = conv_info.stride();
+
+ // Get convolved dimensions
+ unsigned int conv_w = 0;
+ unsigned int conv_h = 0;
+
+ const unsigned int kernel_width = (_are_weights_reshaped) ? weights_info.kernel_size().first : weights->info()->dimension(0);
+ const unsigned int kernel_height = (_are_weights_reshaped) ? weights_info.kernel_size().second : weights->info()->dimension(1);
+ std::tie(conv_w, conv_h) = scaled_dimensions(input->info()->dimension(0), input->info()->dimension(1), kernel_width, kernel_height,
+ conv_info);
+
+ // Check if its a "fully connected" convolution
+ _is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
+ const bool run_interleaved = (!_is_fully_connected_convolution);
+
+ unsigned int mat_weights_cols = weights->info()->dimension(3);
+ unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element;
+
+ // Reshape weights if needed
+ if(_are_weights_reshaped)
+ {
+ if(_is_fully_connected_convolution)
+ {
+ mat_weights_cols = weights->info()->dimension(0);
+ mat_weights_rows = weights->info()->dimension(1);
+ }
+ else
+ {
+ mat_weights_cols = weights_info.num_kernels();
+ const unsigned int quarter_reshaped_cols = weights->info()->dimension(0) / 4;
+ mat_weights_rows = quarter_reshaped_cols + bias_element;
+ }
+ }
+ else
+ {
+ if(_is_fully_connected_convolution)
+ {
+ // Create tensor to store the reshaped weights
+ int num_elems_read_per_iteration_x = 1;
+ if(dt == DataType::F16)
+ {
+ num_elems_read_per_iteration_x = 2;
+ }
+ TensorShape shape_wr((ceil_to_multiple(mat_weights_cols, num_elems_read_per_iteration_x)), mat_weights_rows);
+ _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_wr));
+ _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, false /* 1xW transpose */);
+ }
+ else
+ {
+ // Create tensor to store transposed weights
+ const float transpose_width = 16.0f / input->info()->element_size();
+ TensorShape shape_wt(mat_weights_rows * static_cast<unsigned int>(transpose_width), static_cast<unsigned int>(std::ceil(mat_weights_cols / transpose_width)));
+ _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_wt));
+ _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, true /* 1xW transpose */);
+ }
+ weights = &_weights_reshaped;
+ }
+
+ // Create tensor to store im2col reshaped inputs
+ const unsigned int mat_input_cols = mat_weights_rows;
+ const unsigned int mat_input_rows = conv_w * conv_h;
+ TensorShape shape_im2col = input->info()->tensor_shape();
+ shape_im2col.set(0, mat_input_cols);
+ shape_im2col.set(1, mat_input_rows);
+ shape_im2col.set(2, 1);
+
+ // FIXME: input->clone() doesn't work with subtensors for grouped convolutions.
+ TensorInfo im2col_reshaped_info(shape_im2col, 1, dt, input->info()->fixed_point_position());
+ _input_im2col_reshaped.allocator()->init(im2col_reshaped_info);
+
+ // Create tensor (interleave) to prepare input tensor for GEMM
+ if(run_interleaved)
+ {
+ TensorShape shape_interleaved = shape_im2col;
+ shape_interleaved.set(0, shape_interleaved.x() * 4);
+ shape_interleaved.set(1, std::ceil(shape_interleaved.y() / 4.f));
+
+ // FIXME: input->clone() doesn't work with subtensors for grouped convolutions.
+ TensorInfo interleaved_info(shape_interleaved, 1, dt, input->info()->fixed_point_position());
+ _input_interleaved_reshaped.allocator()->init(interleaved_info);
+ }
+
+ // Create GEMM output tensor
+ TensorShape shape_gemm = _input_im2col_reshaped.info()->tensor_shape();
+ shape_gemm.set(0, mat_weights_cols);
+ shape_gemm.set(1, mat_input_rows);
+ const DataType gemm_data_type = dt;
+
+ // FIXME: input->clone() doesn't work with subtensors for grouped convolutions.
+ TensorInfo info_gemm(shape_gemm, 1, gemm_data_type, input->info()->fixed_point_position());
+ _gemm_output.allocator()->init(info_gemm);
+
+ // Configure kernels
+ if(dt == DataType::F16)
+ {
+ BorderSize border_size = BorderSize(conv_info.pad_top(), conv_info.pad_right(), conv_info.pad_bottom(), conv_info.pad_left());
+ input->info()->extend_padding(border_size);
+ _fill_border.configure(input, border_size, BorderMode::CONSTANT, PixelValue(0)); // for PAD of im2col fp16: consider it as border
+ }
+ _input_im2col_kernel.configure(input, &_input_im2col_reshaped, Size2D(kernel_width, kernel_height), conv_info, _append_bias);
+
+ // Configure matrix multiply
+ if(run_interleaved)
+ {
+ _input_interleave_kernel.configure(&_input_im2col_reshaped, &_input_interleaved_reshaped);
+ configure_mm(&_input_interleaved_reshaped, weights, &_gemm_output);
+ _input_interleaved_reshaped.allocator()->allocate();
+ }
+ else
+ {
+ configure_mm(&_input_im2col_reshaped, weights, &_gemm_output, false);
+ }
+ _input_im2col_reshaped.allocator()->allocate();
+
+ // Configure Col2Im
+ _output_col2im_kernel.configure(&_gemm_output, output, std::make_pair(conv_w, conv_h));
+ _gemm_output.allocator()->allocate();
+
+ ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(0) != conv_w) || (output->info()->dimension(1) != conv_h), "Output shape does not match the expected one");
+
+ // Allocate intermediate tensor
+ if(!_are_weights_reshaped)
+ {
+ _weights_reshaped.allocator()->allocate();
+ }
+}
+
+void GCConvolutionLayer::run()
+{
+ // Run weights reshaping (Runs once for every configure)
+ if(!_are_weights_reshaped)
+ {
+ _are_weights_reshaped = true;
+ _reshape_weights.run();
+ }
+
+ // Run im2col
+ GCScheduler::get().dispatch(_fill_border);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_input_im2col_kernel);
+
+ if(!_is_fully_connected_convolution)
+ {
+ GCScheduler::get().memory_barrier();
+ // Run interleave4x4
+ GCScheduler::get().dispatch(_input_interleave_kernel);
+ }
+
+ GCScheduler::get().memory_barrier();
+ // Runs matrix multiply on reshaped matrices
+ GCScheduler::get().dispatch(_mm_kernel);
+
+ GCScheduler::get().memory_barrier();
+ // Reshape output matrix
+ GCScheduler::get().dispatch(_output_col2im_kernel, false);
+}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCDepthConcatenateLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCDepthConcatenateLayer.cpp
index ee0b121..689d8be 100755
--- a/src/runtime/GLES_COMPUTE/functions/GCDepthConcatenateLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCDepthConcatenateLayer.cpp
@@ -63,7 +63,8 @@
{
for(unsigned i = 0; i < _num_inputs; i++)
{
- GCScheduler::get().enqueue(_border_handlers_vector[i], false);
- GCScheduler::get().enqueue(_concat_kernels_vector[i], true);
+ GCScheduler::get().dispatch(_border_handlers_vector[i], false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_concat_kernels_vector[i], true);
}
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCDepthwiseConvolutionLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCDepthwiseConvolutionLayer.cpp
new file mode 100644
index 0000000..ef65989
--- /dev/null
+++ b/src/runtime/GLES_COMPUTE/functions/GCDepthwiseConvolutionLayer.cpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/GLES_COMPUTE/functions/GCDepthwiseConvolutionLayer.h"
+
+#include "arm_compute/core/GLES_COMPUTE/IGCTensor.h"
+#include "arm_compute/core/PixelValue.h"
+#include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
+#include "support/ToolchainSupport.h"
+
+using namespace arm_compute;
+
+void GCDepthwiseConvolutionLayer3x3::configure(IGCTensor *input, const IGCTensor *weights, const IGCTensor *biases, IGCTensor *output, const PadStrideInfo &conv_info)
+{
+ auto k = arm_compute::support::cpp14::make_unique<GCDepthwiseConvolutionLayer3x3Kernel>();
+ k->configure(input, weights, biases, output, conv_info);
+ _kernel = std::move(k);
+
+ // Configure border handler
+ _border_handler.configure(input, _kernel->border_size(), BorderMode::CONSTANT, PixelValue(0));
+}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCDropoutLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCDropoutLayer.cpp
index 032c2fd..6407464 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCDropoutLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCDropoutLayer.cpp
@@ -46,5 +46,5 @@
void GCDropoutLayer::run()
{
- GCScheduler::get().enqueue(_dropout_kernel);
+ GCScheduler::get().dispatch(_dropout_kernel);
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCFullyConnectedLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCFullyConnectedLayer.cpp
index 63cb40e..9e4f0f6 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCFullyConnectedLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCFullyConnectedLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -61,7 +61,7 @@
_im2col_output.allocator()->init(TensorInfo(shape_im2col, 1, dt));
// Configure im2col kernel
- _im2col_kernel.configure(input, &_im2col_output, std::make_pair(1, 1), PadStrideInfo(1, 1, 0, 0), false);
+ _im2col_kernel.configure(input, &_im2col_output, Size2D(1, 1), PadStrideInfo(1, 1, 0, 0), false);
// Configure matrix multiply kernel
_mm_kernel.configure(&_im2col_output, weights, output, 1.0f, false);
@@ -159,19 +159,22 @@
// Linearize input if it comes from a convolutional layer
if(_is_fc_after_conv)
{
- GCScheduler::get().enqueue(_im2col_kernel, false);
+ GCScheduler::get().dispatch(_im2col_kernel, false);
}
- GCScheduler::get().sync();
+ if(!_are_weights_reshaped || _is_fc_after_conv)
+ {
+ GCScheduler::get().memory_barrier();
+ }
// Run matrix multiply
- GCScheduler::get().enqueue(_mm_kernel, !_accumulate_biases);
+ GCScheduler::get().dispatch(_mm_kernel, !_accumulate_biases);
// Accumulate biases if provided
if(_accumulate_biases)
{
- GCScheduler::get().sync();
+ GCScheduler::get().memory_barrier();
- GCScheduler::get().enqueue(_accumulate_biases_kernel);
+ GCScheduler::get().dispatch(_accumulate_biases_kernel);
}
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCGEMM.cpp b/src/runtime/GLES_COMPUTE/functions/GCGEMM.cpp
index c47a0e7..7aa2d42 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCGEMM.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCGEMM.cpp
@@ -38,6 +38,7 @@
#include "arm_compute/runtime/ITensorAllocator.h"
using namespace arm_compute;
+using namespace arm_compute::gles_compute;
GCGEMM::GCGEMM()
: _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _ma_kernel(), _tmp_a(), _tmp_b(), _is_interleaved_transposed(false), _run_addition(false)
@@ -116,18 +117,20 @@
if(_is_interleaved_transposed)
{
// Run interleave kernel
- GCScheduler::get().enqueue(_interleave_kernel, false);
+ GCScheduler::get().dispatch(_interleave_kernel, false);
// Run transpose kernel
- GCScheduler::get().enqueue(_transpose_kernel, false);
+ GCScheduler::get().dispatch(_transpose_kernel, false);
+ GCScheduler::get().memory_barrier();
}
// Run matrix multiply kernel
- GCScheduler::get().enqueue(_mm_kernel, !_run_addition);
+ GCScheduler::get().dispatch(_mm_kernel, !_run_addition);
// Run matrix addition kernel
if(_run_addition)
{
- GCScheduler::get().enqueue(_ma_kernel);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_ma_kernel);
}
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCNormalizationLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCNormalizationLayer.cpp
index d30ed52..fc3882d 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCNormalizationLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCNormalizationLayer.cpp
@@ -55,7 +55,9 @@
void GCNormalizationLayer::run()
{
- GCScheduler::get().enqueue(_multiply_kernel, false);
- GCScheduler::get().enqueue(_border_handler, false);
- GCScheduler::get().enqueue(_norm_kernel, false);
+ GCScheduler::get().dispatch(_multiply_kernel, false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_border_handler, false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_norm_kernel, true);
}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCNormalizePlanarYUVLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCNormalizePlanarYUVLayer.cpp
new file mode 100755
index 0000000..5fb971c
--- /dev/null
+++ b/src/runtime/GLES_COMPUTE/functions/GCNormalizePlanarYUVLayer.cpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "arm_compute/runtime/GLES_COMPUTE/functions/GCNormalizePlanarYUVLayer.h"
+
+#include "arm_compute/core/Error.h"
+#include "arm_compute/core/TensorInfo.h"
+#include "arm_compute/core/Types.h"
+#include "arm_compute/core/Validate.h"
+#include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
+
+using namespace arm_compute;
+
+GCNormalizePlanarYUVLayer::GCNormalizePlanarYUVLayer()
+ : _norm_kernel()
+{
+}
+
+void GCNormalizePlanarYUVLayer::configure(const IGCTensor *input, IGCTensor *output, const IGCTensor *mean, const IGCTensor *sd)
+{
+ _norm_kernel.configure(input, output, mean, sd);
+}
+
+void GCNormalizePlanarYUVLayer::run()
+{
+ GCScheduler::get().dispatch(_norm_kernel, true);
+}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCPoolingLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCPoolingLayer.cpp
index 46a60cd..ff03eff 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCPoolingLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCPoolingLayer.cpp
@@ -23,8 +23,8 @@
*/
#include "arm_compute/runtime/GLES_COMPUTE/functions/GCPoolingLayer.h"
+#include "arm_compute/core/GLES_COMPUTE/IGCTensor.h"
#include "arm_compute/core/GLES_COMPUTE/kernels/GCPoolingLayerKernel.h"
-#include "arm_compute/core/PixelValue.h"
#include "support/ToolchainSupport.h"
using namespace arm_compute;
@@ -40,3 +40,8 @@
BorderMode border_mode = (PoolingType::MAX == pool_info.pool_type()) ? BorderMode::REPLICATE : BorderMode::CONSTANT;
_border_handler.configure(input, _kernel->border_size(), border_mode, PixelValue(0.0f));
}
+
+Status GCPoolingLayer::validate(const ITensorInfo *input, const ITensorInfo *output, const PoolingLayerInfo &pool_info)
+{
+ return GCPoolingLayerKernel::validate(input, output, pool_info);
+}
\ No newline at end of file
diff --git a/src/runtime/GLES_COMPUTE/functions/GCScale.cpp b/src/runtime/GLES_COMPUTE/functions/GCScale.cpp
new file mode 100644
index 0000000..cfe65a3
--- /dev/null
+++ b/src/runtime/GLES_COMPUTE/functions/GCScale.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016, 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/runtime/GLES_COMPUTE/functions/GCScale.h"
+
+#include "arm_compute/core/Error.h"
+#include "arm_compute/core/GLES_COMPUTE/IGCTensor.h"
+#include "arm_compute/core/GLES_COMPUTE/kernels/GCScaleKernel.h"
+#include "arm_compute/core/Validate.h"
+#include "support/ToolchainSupport.h"
+
+using namespace arm_compute;
+
+void GCScale::configure(IGCTensor *input, IGCTensor *output, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, SamplingPolicy sampling_policy)
+{
+ auto k = arm_compute::support::cpp14::make_unique<GCScaleKernel>();
+ k->configure(input, output, policy, border_mode == BorderMode::UNDEFINED, sampling_policy);
+ _kernel = std::move(k);
+ _border_handler.configure(input, _kernel->border_size(), border_mode, constant_border_value);
+}
diff --git a/src/runtime/GLES_COMPUTE/functions/GCSoftmaxLayer.cpp b/src/runtime/GLES_COMPUTE/functions/GCSoftmaxLayer.cpp
index 34464ff..5221c5c 100644
--- a/src/runtime/GLES_COMPUTE/functions/GCSoftmaxLayer.cpp
+++ b/src/runtime/GLES_COMPUTE/functions/GCSoftmaxLayer.cpp
@@ -63,9 +63,9 @@
void GCSoftmaxLayer::run()
{
- GCScheduler::get().enqueue(_max_kernel, false);
- GCScheduler::get().sync();
- GCScheduler::get().enqueue(_shift_exp_sum_kernel, false);
- GCScheduler::get().sync();
- GCScheduler::get().enqueue(_norm_kernel);
+ GCScheduler::get().dispatch(_max_kernel, false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_shift_exp_sum_kernel, false);
+ GCScheduler::get().memory_barrier();
+ GCScheduler::get().dispatch(_norm_kernel);
}
diff --git a/src/runtime/NEON/functions/NEConvolution.cpp b/src/runtime/NEON/functions/NEConvolution.cpp
index f10ffa6..b84dfd3 100644
--- a/src/runtime/NEON/functions/NEConvolution.cpp
+++ b/src/runtime/NEON/functions/NEConvolution.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2017 ARM Limited.
+ * Copyright (c) 2016, 2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -54,7 +54,8 @@
}
template <unsigned int matrix_size>
-void NEConvolutionSquare<matrix_size>::configure(ITensor *input, ITensor *output, const int16_t *conv, uint32_t scale, BorderMode border_mode, uint8_t constant_border_value)
+void NEConvolutionSquare<matrix_size>::configure(ITensor *input, ITensor *output, const int16_t *conv, uint32_t scale, BorderMode border_mode,
+ uint8_t constant_border_value)
{
ARM_COMPUTE_ERROR_ON(conv == nullptr);
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
diff --git a/src/runtime/NEON/functions/NEConvolutionLayer.cpp b/src/runtime/NEON/functions/NEConvolutionLayer.cpp
index 25c639f..8f7d940 100644
--- a/src/runtime/NEON/functions/NEConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEConvolutionLayer.cpp
@@ -44,6 +44,16 @@
namespace arm_compute
{
+namespace
+{
+TensorShape get_reshaped_weights_shape(const ITensorInfo *weights, bool has_bias)
+{
+ const unsigned int mat_weights_cols = weights->dimension(3);
+ const unsigned int mat_weights_rows = weights->dimension(0) * weights->dimension(1) * weights->dimension(2) + (has_bias ? 1 : 0);
+ return TensorShape(mat_weights_cols, mat_weights_rows);
+}
+} // namespace
+
NEConvolutionLayerReshapeWeights::NEConvolutionLayerReshapeWeights(std::shared_ptr<IMemoryManager> memory_manager)
: _memory_group(std::move(memory_manager)), _weights_reshape_kernel(), _weights_transposed_kernel(), _weights_reshaped(), _transpose1xW(false)
{
@@ -51,18 +61,12 @@
void NEConvolutionLayerReshapeWeights::configure(const ITensor *weights, const ITensor *biases, ITensor *output, bool transpose1xW)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, output);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(weights, output);
- ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
-
- if(biases != nullptr)
- {
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(weights, biases);
- ARM_COMPUTE_ERROR_ON(biases->info()->dimension(0) != weights->info()->dimension(3));
- ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
- }
+ // Perform validation step
+ ARM_COMPUTE_ERROR_ON_NULLPTR(weights, output);
+ ARM_COMPUTE_ERROR_THROW_ON(NEConvolutionLayerReshapeWeights::validate(weights->info(),
+ (biases != nullptr) ? biases->info() : nullptr,
+ output->info(),
+ transpose1xW));
// Check if bias are present, if yes they will be embedded to the weights matrix
const bool _has_bias = (biases != nullptr);
@@ -72,10 +76,7 @@
if(transpose1xW)
{
// Create tensor to store the reshaped weights
- const unsigned int mat_weights_cols = weights->info()->dimension(3);
- const unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + (_has_bias ? 1 : 0);
- TensorShape shape_wr(mat_weights_cols, mat_weights_rows);
- TensorInfo info_wr(shape_wr, 1, weights->info()->data_type(), weights->info()->fixed_point_position());
+ TensorInfo info_wr = weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(get_reshaped_weights_shape(weights->info(), _has_bias));
_weights_reshaped.allocator()->init(info_wr);
_memory_group.manage(&_weights_reshaped);
@@ -91,6 +92,46 @@
}
}
+Status NEConvolutionLayerReshapeWeights::validate(const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, bool transpose1xW)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4);
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(weights, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(3));
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1);
+ }
+
+ // Check if bias are present, if yes they will be embedded to the weights matrix
+ const bool has_bias = (biases != nullptr);
+
+ // Checks performed when biases are present
+ if(has_bias)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(3));
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1);
+ }
+
+ if(transpose1xW)
+ {
+ TensorInfo weights_reshaped = weights->clone()->set_tensor_shape(get_reshaped_weights_shape(weights, has_bias));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEWeightsReshapeKernel::validate(weights, biases, &weights_reshaped));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMTranspose1xWKernel::validate(&weights_reshaped, output));
+ }
+ else
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(NEWeightsReshapeKernel::validate(weights, biases, output));
+ }
+
+ return Status{};
+}
+
void NEConvolutionLayerReshapeWeights::run()
{
_memory_group.acquire();
@@ -105,6 +146,62 @@
_memory_group.release();
}
+namespace
+{
+TensorShape get_reshaped_weights_shape_conv(const ITensorInfo *weights, bool has_bias, bool is_fully_connected_convolution)
+{
+ unsigned int mat_weights_cols = weights->dimension(3);
+ unsigned int mat_weights_rows = weights->dimension(0) * weights->dimension(1) * weights->dimension(2) + (has_bias ? 1 : 0);
+
+ if(is_fully_connected_convolution)
+ {
+ // Create tensor to store the reshaped weights
+ return TensorShape(mat_weights_cols, mat_weights_rows);
+ }
+ else
+ {
+ // Create tensor to store transposed weights
+ const float transpose_width = 16.0f / weights->element_size();
+ return TensorShape(mat_weights_rows * static_cast<unsigned int>(transpose_width), static_cast<unsigned int>(std::ceil(mat_weights_cols / transpose_width)));
+ }
+}
+
+Status validate_and_initialize_values(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const PadStrideInfo &conv_info, const WeightsInfo &weights_info, DataType &dt,
+ bool &has_bias,
+ bool &are_weights_reshaped, unsigned int &kernel_width, unsigned int &kernel_height, bool &is_fully_connected_convolution, unsigned int &mat_weights_cols, unsigned int &mat_weights_rows,
+ unsigned int &conv_w, unsigned int &conv_h)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights);
+ ARM_COMPUTE_RETURN_ERROR_ON(!weights_info.are_reshaped() && weights->dimension(2) != input->dimension(2));
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4);
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON(!weights_info.are_reshaped() && biases->dimension(0) != weights->dimension(3));
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1);
+ }
+
+ dt = input->data_type();
+ has_bias = (biases != nullptr);
+ are_weights_reshaped = weights_info.are_reshaped();
+ kernel_width = (are_weights_reshaped) ? weights_info.kernel_size().first : weights->dimension(0);
+ kernel_height = (are_weights_reshaped) ? weights_info.kernel_size().second : weights->dimension(1);
+ mat_weights_cols = weights->dimension(3);
+ mat_weights_rows = weights->dimension(0) * weights->dimension(1) * weights->dimension(2) + (has_bias ? 1 : 0);
+
+ std::tie(conv_w, conv_h) = scaled_dimensions(input->dimension(0), input->dimension(1), kernel_width, kernel_height,
+ conv_info);
+
+ is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
+
+ return Status{};
+}
+} // namespace
+
NEConvolutionLayer::NEConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
: _memory_group(std::move(memory_manager)), _input_im2col_kernel(), _input_interleave_kernel(), _reshape_weights(), _mm_kernel(), _mm_optimised_kernel(nullptr), _output_col2im_kernel(),
_input_im2col_reshaped(), _input_interleaved_reshaped(), _weights_reshaped(), _gemm_output(), _workspace(), _has_bias(false), _is_fully_connected_convolution(false), _are_weights_reshaped(false)
@@ -113,42 +210,25 @@
void NEConvolutionLayer::configure(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights);
- ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && weights->info()->dimension(2) != input->info()->dimension(2));
- ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4);
+ // Perform validate step
+ ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
- if(biases != nullptr)
- {
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases);
- ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && biases->info()->dimension(0) != weights->info()->dimension(3));
- ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1);
- }
+ DataType dt{};
+ unsigned int kernel_width = 0;
+ unsigned int kernel_height = 0;
+ unsigned int mat_weights_cols = 0;
+ unsigned int mat_weights_rows = 0;
+ unsigned int conv_w = 0;
+ unsigned int conv_h = 0;
- const DataType dt = input->info()->data_type();
- const int fixed_point_position = input->info()->fixed_point_position();
+ Status status = validate_and_initialize_values(input->info(), weights->info(), (biases == nullptr) ? nullptr : biases->info(), conv_info, weights_info, dt, _has_bias, _are_weights_reshaped,
+ kernel_width, kernel_height,
+ _is_fully_connected_convolution,
+ mat_weights_cols, mat_weights_rows, conv_w, conv_h);
- _has_bias = (biases != nullptr);
- _are_weights_reshaped = weights_info.are_reshaped();
+ ARM_COMPUTE_ERROR_THROW_ON(status);
- // Get parameters from conv_info
- unsigned int stride_x = 0;
- unsigned int stride_y = 0;
- std::tie(stride_x, stride_y) = conv_info.stride();
-
- // Get convolved dimensions
- unsigned int conv_w = 0;
- unsigned int conv_h = 0;
-
- const unsigned int kernel_width = (_are_weights_reshaped) ? weights_info.kernel_size().first : weights->info()->dimension(0);
- const unsigned int kernel_height = (_are_weights_reshaped) ? weights_info.kernel_size().second : weights->info()->dimension(1);
- std::tie(conv_w, conv_h) = scaled_dimensions(input->info()->dimension(0), input->info()->dimension(1), kernel_width, kernel_height,
- conv_info);
-
- // Check if its a "fully connected" convolution, i.e. the output size is 1x1xnum_kernels
- _is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1));
+ const unsigned int fixed_point_position = input->info()->fixed_point_position();
#if defined(__arm__)
if(NEScheduler::get().cpu_info().CPU == CPUTarget::ARMV7 && dt == DataType::F32)
@@ -162,9 +242,6 @@
}
#endif /* defined(__arm__) || defined(__aarch64__) */
- unsigned int mat_weights_cols = weights->info()->dimension(3);
- unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + (_has_bias ? 1 : 0);
-
// Reshape weights if needed
if(_mm_optimised_kernel != nullptr)
{
@@ -230,7 +307,7 @@
shape_im2col.set(0, mat_input_cols);
shape_im2col.set(1, mat_input_rows);
shape_im2col.set(2, 1);
- _input_im2col_reshaped.allocator()->init(TensorInfo(shape_im2col, 1, dt, fixed_point_position));
+ _input_im2col_reshaped.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col));
_memory_group.manage(&_input_im2col_reshaped);
// Create tensor (interleave) to prepare input tensor for GEMM
@@ -239,7 +316,7 @@
TensorShape shape_interleaved(shape_im2col);
shape_interleaved.set(0, shape_interleaved.x() * 4);
shape_interleaved.set(1, std::ceil(shape_interleaved.y() / 4.f));
- _input_interleaved_reshaped.allocator()->init(TensorInfo(shape_interleaved, 1, dt, fixed_point_position));
+ _input_interleaved_reshaped.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_interleaved));
_memory_group.manage(&_input_interleaved_reshaped);
}
@@ -247,7 +324,7 @@
TensorShape shape_gemm(_input_im2col_reshaped.info()->tensor_shape());
shape_gemm.set(0, mat_weights_cols);
shape_gemm.set(1, mat_input_rows);
- _gemm_output.allocator()->init(TensorInfo(shape_gemm, 1, dt, fixed_point_position));
+ _gemm_output.allocator()->init(_input_im2col_reshaped.info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_gemm));
_memory_group.manage(&_gemm_output);
// Configure kernels
@@ -273,14 +350,7 @@
_memory_group.manage(&_workspace);
// Configure matrix multiplication kernel
- if(_is_fully_connected_convolution)
- {
- _mm_optimised_kernel->configure(&_input_im2col_reshaped, weights, &_gemm_output, &_workspace, 1.f, 0.f);
- }
- else
- {
- _mm_optimised_kernel->configure(&_input_im2col_reshaped, weights, &_gemm_output, &_workspace);
- }
+ _mm_optimised_kernel->configure(&_input_im2col_reshaped, weights, &_gemm_output, &_workspace);
_workspace.allocator()->allocate();
}
@@ -303,8 +373,6 @@
_output_col2im_kernel.configure(&_gemm_output, output, Size2D(conv_w, conv_h));
_gemm_output.allocator()->allocate();
- ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(0) != conv_w) || (output->info()->dimension(1) != conv_h), "Output shape does not match the expected one");
-
// Allocate intermediate tensor
if(!_are_weights_reshaped)
{
@@ -312,6 +380,128 @@
}
}
+Status NEConvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info,
+ const WeightsInfo &weights_info)
+{
+ DataType dt{};
+ bool has_bias{};
+ bool are_weights_reshaped{};
+ bool is_fully_connected_convolution{};
+ unsigned int kernel_width = 0;
+ unsigned int kernel_height = 0;
+ unsigned int mat_weights_cols = 0;
+ unsigned int mat_weights_rows = 0;
+ unsigned int conv_w = 0;
+ unsigned int conv_h = 0;
+
+ Status status = validate_and_initialize_values(input, weights, biases, conv_info, weights_info, dt, has_bias, are_weights_reshaped, kernel_width, kernel_height,
+ is_fully_connected_convolution, mat_weights_cols, mat_weights_rows,
+ conv_w, conv_h);
+
+ ARM_COMPUTE_RETURN_ON_ERROR(status);
+
+ std::unique_ptr<ITensorInfo> reshaped_weights = weights->clone();
+ bool optimised_kernel = false;
+
+#if defined(__arm__)
+ if(NEScheduler::get().cpu_info().CPU == CPUTarget::ARMV7 && dt == DataType::F32)
+ {
+ optimised_kernel = true;
+ }
+#elif defined(__aarch64__)
+ if(NEScheduler::get().cpu_info().CPU >= CPUTarget::ARMV8 && dt == DataType::F32)
+ {
+ optimised_kernel = true;
+ }
+#endif /* defined(__arm__) || defined(__aarch64__) */
+
+ // Reshape weights if needed
+ if(optimised_kernel)
+ {
+ if(are_weights_reshaped)
+ {
+ mat_weights_cols = weights_info.num_kernels();
+ mat_weights_rows = weights->dimension(1);
+ }
+ else
+ {
+ TensorShape reshaped_weights_shape{ mat_weights_cols, mat_weights_rows };
+
+ // Create tensor to store the reshaped weights
+ reshaped_weights->set_tensor_shape(get_reshaped_weights_shape_conv(weights, has_bias, is_fully_connected_convolution));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEConvolutionLayerReshapeWeights::validate(weights, biases, reshaped_weights.get(), !is_fully_connected_convolution /* 1xW transpose */));
+ weights = reshaped_weights.get();
+ }
+ }
+ else
+ {
+ if(are_weights_reshaped)
+ {
+ const unsigned int transpose_width = 16 / input->element_size();
+ mat_weights_cols = weights_info.num_kernels();
+ mat_weights_rows = weights->dimension(0) / transpose_width + (has_bias ? 1 : 0);
+ }
+ else
+ {
+ TensorShape reshaped_weights_shape;
+
+ if(is_fully_connected_convolution)
+ {
+ reshaped_weights_shape = TensorShape{ mat_weights_cols, mat_weights_rows };
+ }
+ else
+ {
+ // Create tensor to store transposed weights
+ const float transpose_width = 16.0f / input->element_size();
+ reshaped_weights_shape = TensorShape{ mat_weights_rows *static_cast<unsigned int>(transpose_width),
+ static_cast<unsigned int>(std::ceil(mat_weights_cols / transpose_width)) };
+ }
+
+ // Create tensor to store the reshaped weights
+ reshaped_weights->set_tensor_shape(get_reshaped_weights_shape_conv(weights, has_bias, is_fully_connected_convolution));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEConvolutionLayerReshapeWeights::validate(weights, biases, reshaped_weights.get(), !is_fully_connected_convolution /* 1xW transpose */));
+ weights = reshaped_weights.get();
+ }
+ }
+
+ // Validate im2col
+ const unsigned int mat_input_cols = mat_weights_rows;
+ const unsigned int mat_input_rows = conv_w * conv_h;
+ TensorShape shape_im2col = input->tensor_shape();
+ shape_im2col.set(0, mat_input_cols);
+ shape_im2col.set(1, mat_input_rows);
+ shape_im2col.set(2, 1);
+ TensorInfo im2_col_info = input->clone()->set_tensor_shape(shape_im2col);
+ ARM_COMPUTE_RETURN_ON_ERROR(NEIm2ColKernel::validate(input, &im2_col_info, Size2D(weights->dimension(0), weights->dimension(1)), conv_info, has_bias));
+
+ // Create GEMM output tensor
+ TensorShape shape_gemm(im2_col_info.tensor_shape());
+ shape_gemm.set(0, mat_weights_cols);
+ shape_gemm.set(1, mat_input_rows);
+ TensorInfo gemm_output_info = input->clone()->set_tensor_shape(shape_gemm);
+
+ // Validate GEMM interleave and multiply
+ if(!is_fully_connected_convolution)
+ {
+ TensorShape shape_interleaved = shape_im2col;
+ shape_interleaved.set(0, shape_interleaved.x() * 4);
+ shape_interleaved.set(1, std::ceil(shape_interleaved.y() / 4.f));
+ TensorInfo input_interleaved_info = input->clone()->set_tensor_shape(shape_interleaved);
+ ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMInterleave4x4Kernel::validate(&im2_col_info, &input_interleaved_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixMultiplyKernel::validate(&input_interleaved_info, weights, &gemm_output_info));
+ }
+ else
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixMultiplyKernel::validate(&im2_col_info, weights, &gemm_output_info));
+ }
+
+ ARM_COMPUTE_RETURN_ON_ERROR(NECol2ImKernel::validate(&gemm_output_info, output, Size2D(conv_w, conv_h)));
+
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((output->dimension(0) != conv_w) || (output->dimension(1) != conv_h), "Output shape does not match the expected one");
+
+ return Status{};
+}
+
void NEConvolutionLayer::run()
{
// Run weights reshaping (Runs once for every configure)
diff --git a/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp b/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp
index 7b4e77b..7bce8a6 100644
--- a/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017, 2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -24,39 +24,43 @@
#include "arm_compute/runtime/NEON/functions/NEDeconvolutionLayer.h"
#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/PixelValue.h"
#include "arm_compute/core/Utils.h"
#include "arm_compute/core/Validate.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
using namespace arm_compute;
+using namespace arm_compute::misc::shape_calculator;
NEDeconvolutionLayer::NEDeconvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
: _memory_group(std::move(memory_manager)),
- _scale_f(),
_conv_f(),
- _scaled_output()
+ _scaled_output(),
+ _input(nullptr),
+ _info(),
+ _inner_border()
{
}
void NEDeconvolutionLayer::configure(ITensor *input, const ITensor *weights, const ITensor *bias, ITensor *output, const PadStrideInfo &info,
- unsigned int ax, unsigned int ay, float upscalex, float upscaley)
+ unsigned int inner_border_right, unsigned int inner_border_top)
{
ARM_COMPUTE_ERROR_ON_NULLPTR(output);
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
ARM_COMPUTE_ERROR_ON(weights->info()->dimension(0) != weights->info()->dimension(1));
- ARM_COMPUTE_ERROR_ON(weights->info()->dimension(0) < 1);
+ ARM_COMPUTE_ERROR_ON(weights->info()->dimension(0) != 1 && weights->info()->dimension(0) != 3 && weights->info()->dimension(0) != 5);
- auto out_dims = deconvolution_output_dimensions(input->info()->dimension(0), input->info()->dimension(1), weights->info()->dimension(0), weights->info()->dimension(1),
- info.pad().first, info.pad().second, ax, ay, upscalex, upscaley, info.round());
+ _input = input;
+ _info = info;
+ _inner_border = std::make_pair(inner_border_right, inner_border_top);
+
+ const unsigned int stride_x = info.stride().first;
+ const unsigned int stride_y = info.stride().second;
+ auto out_dims = deconvolution_output_dimensions(input->info()->dimension(0), input->info()->dimension(1), weights->info()->dimension(0), weights->info()->dimension(1),
+ info.pad().first, info.pad().second, inner_border_right, inner_border_top, stride_x, stride_y);
const TensorShape output_shape = deconvolution_output_shape(out_dims, input->info()->tensor_shape(), weights->info()->tensor_shape());
- // Output auto initialization if not yet initialized
- auto_init_if_empty(*output->info(), output_shape, 1, input->info()->data_type(), input->info()->fixed_point_position());
-
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output, weights, bias);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, output, weights, bias);
-
+ ARM_COMPUTE_UNUSED(output_shape);
ARM_COMPUTE_ERROR_ON_MSG(output->info()->dimension(Window::DimX) != output_shape.x(), "Output's width is invalid.");
ARM_COMPUTE_ERROR_ON_MSG(output->info()->dimension(Window::DimY) != output_shape.y(), "Output's height is invalid.");
ARM_COMPUTE_ERROR_ON_MSG(output->info()->dimension(Window::DimZ) != output_shape.z(), "Output's depth is invalid.");
@@ -64,51 +68,51 @@
_memory_group.manage(&_scaled_output);
// configure scale function
- //Init and allocate intermmidiate tensor for output, same size as input but the first two axis are the same as the output tensor
- TensorShape scale_out_shape(input->info()->tensor_shape());
- scale_out_shape.set(0, output->info()->dimension(0));
- scale_out_shape.set(1, output->info()->dimension(1));
- TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(), input->info()->fixed_point_position());
+ // Init and allocate intermmidiate tensor for output, same size as input but the first two axis are the same as the output tensor
+ const TensorInfo scale_out_info(compute_deconvolution_shape(*input->info(), stride_x, stride_y, inner_border_right, inner_border_top, info), 1, input->info()->data_type(),
+ input->info()->fixed_point_position());
_scaled_output.allocator()->init(scale_out_info);
- const unsigned int kernel_size = weights->info()->dimension(0);
- // Padding for the upsampled image is calculated with the equiation: p' = k - p - 1, where k is kernel size and p is the input padding
- ARM_COMPUTE_ERROR_ON(info.pad().first > (kernel_size - 1));
- const unsigned int tr_px = kernel_size - info.pad().first - 1;
- const unsigned int tr_py = kernel_size - info.pad().second - 1;
- const unsigned int tr_stride = 1;
- const PadStrideInfo transposed_info(tr_stride, tr_stride, tr_px, tr_py);
- _scale_f.configure(input, &_scaled_output, std::make_pair(ax, ay), std::make_pair(info.stride().first - 1u, info.stride().second - 1u), transposed_info);
+
// setup the function to convolve the upscaled output
- switch(kernel_size)
- {
- case 1:
- {
- _conv_f.configure(&_scaled_output, weights, bias, output, PadStrideInfo(1, 1, 0, 0, DimensionRoundingType::CEIL));
- break;
- }
- case 3:
- {
- _conv_f.configure(&_scaled_output, weights, bias, output, PadStrideInfo(1, 1, 1, 1, DimensionRoundingType::CEIL));
- break;
- }
- case 5:
- {
- _conv_f.configure(&_scaled_output, weights, bias, output, PadStrideInfo(1, 1, 2, 2, DimensionRoundingType::CEIL));
- break;
- }
- default:
- {
- ARM_COMPUTE_ERROR("Not supported");
- break;
- }
- }
+ const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
+ _conv_f.configure(&_scaled_output, weights, bias, output, conv_info);
_scaled_output.allocator()->allocate();
}
void NEDeconvolutionLayer::run()
{
_memory_group.acquire();
- _scale_f.run();
+
+ // Initialize _scaled_output buffer
+ const int width_in = _input->info()->dimension(0);
+ const int height_in = _input->info()->dimension(1);
+ const int width_scaled = _scaled_output.info()->dimension(0);
+ const int height_scaled = _scaled_output.info()->dimension(1);
+ const int num_2d_slices = _input->info()->tensor_shape().total_size() / (width_in * height_in);
+ const int stride_x = _info.stride().first;
+ const int stride_y = _info.stride().second;
+
+ std::fill_n(reinterpret_cast<float *>(_scaled_output.buffer()), _scaled_output.info()->tensor_shape().total_size(), 0.f);
+
+ // scaled_output is the input for the forward convolution. We copy the input elements to scaled_output
+ // and insert rows and columns with zeroes depending on the stride values.
+ for(int slice = 0; slice < num_2d_slices; ++slice)
+ {
+ const int start_x = _info.pad().first;
+ const int start_y = _inner_border.second + _info.pad().second;
+ const int end_y = height_scaled - _info.pad().second;
+ const int end_x = width_scaled - _inner_border.first - _info.pad().first;
+
+ for(int yi = start_y, in_y = 0; yi < end_y; yi += stride_y, in_y++)
+ {
+ for(int xi = start_x, in_x = 0; xi < end_x; xi += stride_x, in_x++)
+ {
+ const auto in = *(reinterpret_cast<float *>(_input->buffer() + _input->info()->offset_element_in_bytes(Coordinates(in_x, in_y, slice))));
+ *(reinterpret_cast<float *>(_scaled_output.buffer() + _scaled_output.info()->offset_element_in_bytes(Coordinates(xi, yi, slice)))) = in;
+ }
+ }
+ }
+
_conv_f.run();
_memory_group.release();
}
diff --git a/src/runtime/NEON/functions/NEDeconvolutionLayerUpsample.cpp b/src/runtime/NEON/functions/NEDeconvolutionLayerUpsample.cpp
deleted file mode 100644
index 79b9b2d..0000000
--- a/src/runtime/NEON/functions/NEDeconvolutionLayerUpsample.cpp
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2016, 2017 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include "arm_compute/runtime/NEON/functions/NEDeconvolutionLayerUpsample.h"
-
-#include "arm_compute/core/Coordinates.h"
-#include "arm_compute/core/Error.h"
-#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/NEON/kernels/NEDeconvolutionLayerUpsampleKernel.h"
-#include "arm_compute/core/PixelValue.h"
-#include "arm_compute/core/TensorInfo.h"
-#include "arm_compute/core/Window.h"
-#include "arm_compute/runtime/NEON/NEScheduler.h"
-#include "arm_compute/runtime/TensorAllocator.h"
-#include "support/ToolchainSupport.h"
-
-#include <cmath>
-#include <cstddef>
-#include <utility>
-
-using namespace arm_compute;
-
-namespace
-{
-inline void precompute_offsets(ITensor *offsets, float wr, size_t input_element_size, const std::pair<unsigned int, unsigned int> &a,
- const std::pair<unsigned int, unsigned int> &iz, const PadStrideInfo &info)
-{
- ARM_COMPUTE_ERROR_ON(nullptr == offsets);
- Window win;
- const int padx = info.pad().first;
- const int pady = info.pad().second;
- const int ax = a.first;
- const int ay = a.second;
- const int offset_width = offsets->info()->dimension(0);
- const int offset_height = offsets->info()->dimension(1);
- // The values of ax and ay denote the number of ZEROS to be added on the top and right inner border of the image.
- // Step value along the XY axis will depend on the number of zeros to be inserted between samples (number of zeros + 1).
- // Pre-compute the X offset, Y's stride is unknown at this point so we can't precompute Y's offsets
- for(int yi = ay; yi < (offset_height - pady); yi += (1 + iz.second))
- {
- for(int xi = padx; xi < (offset_width - ax); xi += (1 + iz.first))
- {
- int *ptr = reinterpret_cast<int *>(offsets->ptr_to_element(Coordinates(xi, yi)));
- const size_t in_xi = (xi + 0.5f) * wr;
- *reinterpret_cast<int32_t *>(ptr) = in_xi * input_element_size;
- }
- }
-}
-} // namespace
-
-NEDeconvolutionLayerUpsample::NEDeconvolutionLayerUpsample(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
- : _memory_group(std::move(memory_manager)),
- _offsets(),
- _border_handler(),
- _upsample()
-{
-}
-
-void NEDeconvolutionLayerUpsample::configure(ITensor *input, ITensor *output, const std::pair<unsigned int, unsigned int> &a,
- const std::pair<unsigned int, unsigned int> &iz, const PadStrideInfo &info)
-{
- ARM_COMPUTE_ERROR_ON(nullptr == input);
- ARM_COMPUTE_ERROR_ON(nullptr == output);
-
- for(size_t i = 2; i < Coordinates::num_max_dimensions; ++i)
- {
- ARM_COMPUTE_ERROR_ON(input->info()->dimension(i) != output->info()->dimension(i));
- }
-
- // Get the tensor shape
- const TensorShape shape(output->info()->dimension(0), output->info()->dimension(1));
-
- // Compute the ratio between source width/height and destination width/height
- const auto wr = static_cast<float>(input->info()->dimension(0)) / static_cast<float>(output->info()->dimension(0));
- const auto hr = static_cast<float>(input->info()->dimension(1)) / static_cast<float>(output->info()->dimension(1));
- ARM_COMPUTE_UNUSED(hr);
- // Get the element size of the input image
- const size_t input_element_size = input->info()->element_size();
-
- TensorInfo tensor_info_offsets(shape, Format::S32);
- _offsets.allocator()->init(tensor_info_offsets);
-
- _upsample.configure(input, &_offsets, output);
-
- // Allocate once the configure methods have been called
- _offsets.allocator()->allocate();
- // Pre-compute offsets for nearest interpolation
- std::fill_n(reinterpret_cast<int32_t *>(_offsets.buffer()), _offsets.info()->total_size() / sizeof(int32_t), -1 * input_element_size);
- precompute_offsets(&_offsets, wr, input_element_size, a, iz, info);
-
- _border_handler.configure(input, _upsample.border_size(), BorderMode::CONSTANT, PixelValue(0.f));
-}
-
-void NEDeconvolutionLayerUpsample::run()
-{
- NEScheduler::get().schedule(&_border_handler, Window::DimZ);
- _memory_group.acquire();
- NEScheduler::get().schedule(&_upsample, Window::DimY);
- _memory_group.release();
-}
diff --git a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
index b890c6f..2d08b45 100644
--- a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -26,28 +26,56 @@
#include "arm_compute/core/Helpers.h"
#include "arm_compute/core/ITensor.h"
#include "arm_compute/core/PixelValue.h"
+#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
#include "arm_compute/runtime/NEON/NEScheduler.h"
#include "support/ToolchainSupport.h"
using namespace arm_compute;
NEDepthwiseConvolutionLayer3x3::NEDepthwiseConvolutionLayer3x3()
- : _kernel(), _bias_kernel(), _border_handler(), _has_bias(false)
+ : _kernel(), _output_stage_kernel(), _border_handler(), _accumulator(), _has_bias(false), _is_quantized(false)
{
}
void NEDepthwiseConvolutionLayer3x3::configure(ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const PadStrideInfo &conv_info)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output, weights);
+ ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
- // Call convolution kernel
- _kernel.configure(input, weights, output, conv_info);
- _border_handler.configure(input, _kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<float>(0.f)));
- if(biases != nullptr)
+ PixelValue zero_value(0.f);
+
+ _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
+ _has_bias = biases != nullptr;
+
+ // Allocate the intermediate accumulator tensor in case of fixed point input
+ if(_is_quantized)
{
- _bias_kernel.configure(output, biases);
- _has_bias = true;
+ _accumulator.allocator()->init(TensorInfo(output->info()->tensor_shape(), 1, DataType::S32));
+ _accumulator.info()->set_quantization_info(input->info()->quantization_info());
+ zero_value = PixelValue(static_cast<uint32_t>(input->info()->quantization_info().offset));
+ }
+
+ // Configure depthwise convolution kernel
+ _kernel.configure(input, weights, (_is_quantized) ? &_accumulator : output, conv_info);
+
+ // Configure border handler
+ _border_handler.configure(input, _kernel.border_size(), BorderMode::CONSTANT, zero_value);
+
+ // Configure biases accumulation
+ if(_has_bias || _is_quantized)
+ {
+ if(_is_quantized)
+ {
+ float multiplier = input->info()->quantization_info().scale * weights->info()->quantization_info().scale / output->info()->quantization_info().scale;
+ int output_multiplier, output_shift;
+ quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift);
+ _output_stage_kernel.configure(&_accumulator, biases, output, output_multiplier, output_shift, output->info()->quantization_info().offset);
+ _accumulator.allocator()->allocate();
+ }
+ else
+ {
+ _output_stage_kernel.configure(output, biases);
+ }
}
}
@@ -55,9 +83,9 @@
{
NEScheduler::get().schedule(&_border_handler, Window::DimX);
NEScheduler::get().schedule(&_kernel, Window::DimX);
- if(_has_bias)
+ if(_has_bias || _is_quantized)
{
- NEScheduler::get().schedule(&_bias_kernel, Window::DimX);
+ NEScheduler::get().schedule(&_output_stage_kernel, Window::DimX);
}
}
diff --git a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
index afa5d97..c26c99a 100644
--- a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
@@ -34,7 +34,7 @@
using namespace arm_compute;
NEDirectConvolutionLayer::NEDirectConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
- : _memory_group(std::move(memory_manager)), _accumulate_bias_kernel(), _conv_kernel(), _input_border_handler(), _accumulator(), _has_bias(false)
+ : _memory_group(std::move(memory_manager)), _output_stage_kernel(), _conv_kernel(), _input_border_handler(), _accumulator(), _has_bias(false), _is_fixed_point(false)
{
}
@@ -50,16 +50,16 @@
_has_bias = (bias != nullptr);
// Allocate the intermediate accumulator tensor in case of fixed point input
- if(is_data_type_fixed_point(input->info()->data_type()))
+ _is_fixed_point = is_data_type_fixed_point(input->info()->data_type());
+ if(_is_fixed_point)
{
const DataType promoted_dt = (input->info()->data_type() == DataType::QS8) ? DataType::QS16 : DataType::QS32;
_accumulator.allocator()->init(TensorInfo(output->info()->tensor_shape(), 1, promoted_dt, output->info()->fixed_point_position()));
_memory_group.manage(&_accumulator);
_conv_kernel.configure(input, weights, &_accumulator, conv_info);
- if(_has_bias)
- {
- _accumulate_bias_kernel.configure(&_accumulator, bias, output);
- }
+
+ // When no bias is provided, we need to downscale the accumulator tensor
+ _output_stage_kernel.configure(&_accumulator, bias, output);
_accumulator.allocator()->allocate();
}
else
@@ -67,7 +67,7 @@
_conv_kernel.configure(input, weights, output, conv_info);
if(_has_bias)
{
- _accumulate_bias_kernel.configure(output, bias);
+ _output_stage_kernel.configure(output, bias);
}
}
@@ -90,20 +90,17 @@
// Validate Convolution kernel
ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerKernel::validate(input, weights, &accumulator, conv_info));
- // Validate bias
- ARM_COMPUTE_RETURN_ERROR_ON_MSG((bias == nullptr) && is_data_type_fixed_point(data_type),
- "Biases should be provided for fixed point inputs");
if(bias != nullptr)
{
ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, bias);
ARM_COMPUTE_RETURN_ERROR_ON_MSG(bias->dimension(0) != weights->dimension(3),
"Biases size and number of input feature maps should match");
ARM_COMPUTE_RETURN_ERROR_ON_MSG(bias->num_dimensions() > 1, "Biases should be one dimensional");
-
- // Validate bias kernel
- ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerBiasAccumulateKernel::validate(&accumulator, bias, output));
}
+ // Validate bias kernel
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerOutputStageKernel::validate(&accumulator, bias, output));
+
return Status{};
}
@@ -114,10 +111,9 @@
_memory_group.acquire();
NEScheduler::get().schedule(&_conv_kernel, Window::DimZ);
- if(_has_bias)
+ if(_has_bias || _is_fixed_point)
{
- NEScheduler::get().schedule(&_accumulate_bias_kernel, Window::DimY);
+ NEScheduler::get().schedule(&_output_stage_kernel, Window::DimY);
}
-
_memory_group.release();
}
diff --git a/src/runtime/NEON/functions/NEGEMM.cpp b/src/runtime/NEON/functions/NEGEMM.cpp
index 950f4c9..e640b06 100644
--- a/src/runtime/NEON/functions/NEGEMM.cpp
+++ b/src/runtime/NEON/functions/NEGEMM.cpp
@@ -50,15 +50,17 @@
{
NEGEMM::NEGEMM(std::shared_ptr<IMemoryManager> memory_manager)
: _memory_group(std::move(memory_manager)), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _mm_optimised_kernel(nullptr), _ma_kernel(), _tmp_a(), _tmp_b(), _workspace(),
- _run_vector_matrix_multiplication(false), _run_addition(false)
+ _run_vector_matrix_multiplication(false), _run_addition(false), _is_first_run(true), _reshape_b_only_on_first_run(false)
{
}
-void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta)
+void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info)
{
ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::F32, DataType::F16, DataType::QS8, DataType::QS16);
ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, b, d);
ARM_COMPUTE_ERROR_ON_MSG(a->info()->dimension(0) != b->info()->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
+ ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported");
+ ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported");
if(c != nullptr)
{
@@ -70,6 +72,8 @@
ARM_COMPUTE_ERROR_ON_MSG(c->info()->dimension(1) != d->info()->dimension(1), "The C matrix must have the same number of columns as the output matrix");
}
+ // Check if we need to reshape the matrix B only on the first run
+ _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run();
_run_vector_matrix_multiplication = a->info()->dimension(1) < 2;
// Check if the first input tensor is a vector.
@@ -142,7 +146,7 @@
_memory_group.manage(&_workspace);
// Configure matrix multiplication kernel
- _mm_optimised_kernel->configure(a, b, d, &_workspace, alpha, 0.f);
+ _mm_optimised_kernel->configure(a, b, d, &_workspace, alpha, 0.f, false /* is_transposed_0 */, false /* is_transposed_1 */);
_workspace.allocator()->allocate();
}
else
@@ -207,8 +211,18 @@
// Run interleave kernel
NEScheduler::get().schedule(&_interleave_kernel, Window::DimY);
- // Run transpose kernel
- NEScheduler::get().schedule(&_transpose_kernel, Window::DimY);
+ if(_is_first_run)
+ {
+ // Run transpose kernel
+ NEScheduler::get().schedule(&_transpose_kernel, Window::DimY);
+
+ _is_first_run = false;
+ }
+ else if(!_reshape_b_only_on_first_run)
+ {
+ // Run transpose kernel
+ NEScheduler::get().schedule(&_transpose_kernel, Window::DimY);
+ }
}
NEScheduler::get().schedule(&_mm_kernel, _run_vector_matrix_multiplication ? Window::DimX : Window::DimY);
diff --git a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp
index 6e03ffa..9b36e81 100644
--- a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp
+++ b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp
@@ -74,7 +74,7 @@
#endif /* __aarch64__ */
#ifdef ARM_COMPUTE_AARCH64_V8_2
- if(ci.CPU == CPUTarget::A75_DOT)
+ if(ci.CPU == CPUTarget::A75_DOT || ci.CPU == CPUTarget::A55_DOT)
{
// Configure matrix multiply kernel
GemmInterleaved<gemm_s8_12x8, int8_t, int32_t> gemm(&ci, M, N, K, false, false);
@@ -87,10 +87,6 @@
_mm_kernel = std::move(k);
_workspace.allocator()->allocate();
}
- else if(ci.CPU == CPUTarget::A55_DOT)
- {
- ARM_COMPUTE_ERROR_ON("WIP");
- }
else
#elif defined(ARM_COMPUTE_AARCH64_V8A)
if(ci.CPU == CPUTarget::A53)
diff --git a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp
index 50aa5b6..c4028dc 100644
--- a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp
+++ b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp
@@ -84,7 +84,7 @@
// Configure matrix multiplication kernel
auto k = arm_compute::support::cpp14::make_unique<NEGEMMLowpAArch64V8P4Kernel>();
- k->configure(a, b, output, &_workspace, 1.f, 1.f);
+ k->configure(a, b, output, &_workspace, 1.f, 1.f, false, false);
_mm_kernel = std::move(k);
}
else
diff --git a/src/runtime/NEON/functions/NEPoolingLayer.cpp b/src/runtime/NEON/functions/NEPoolingLayer.cpp
index 530c7fc..8a32507 100644
--- a/src/runtime/NEON/functions/NEPoolingLayer.cpp
+++ b/src/runtime/NEON/functions/NEPoolingLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,9 +43,14 @@
// Configure pooling kernel
_pooling_layer_kernel.configure(input, output, pool_info);
- // Configure border depending on operation required
+ // Configure border depending on operation required (quantize border in case of asymmetric data_type)
BorderMode border_mode = (pool_info.pool_type() == PoolingType::MAX) ? BorderMode::REPLICATE : BorderMode::CONSTANT;
- _border_handler.configure(input, _pooling_layer_kernel.border_size(), border_mode, PixelValue(static_cast<float>(0.f)));
+ PixelValue zero_value(0.f);
+ if(is_data_type_quantized_asymmetric(input->info()->data_type()) && !pool_info.exclude_padding())
+ {
+ zero_value = PixelValue(static_cast<uint32_t>(input->info()->quantization_info().offset));
+ }
+ _border_handler.configure(input, _pooling_layer_kernel.border_size(), border_mode, zero_value);
}
Status NEPoolingLayer::validate(const ITensorInfo *input, const ITensorInfo *output, const PoolingLayerInfo &pool_info)
diff --git a/src/runtime/NEON/functions/NEWinogradLayer.cpp b/src/runtime/NEON/functions/NEWinogradLayer.cpp
index 3251de4..da46f87 100644
--- a/src/runtime/NEON/functions/NEWinogradLayer.cpp
+++ b/src/runtime/NEON/functions/NEWinogradLayer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,7 +43,8 @@
namespace arm_compute
{
NEWinogradLayer::NEWinogradLayer(std::shared_ptr<IMemoryManager> memory_manager)
- : _memory_group(std::move(memory_manager)), _winograd_kernel(), _weights_workspace(), _workspace(), _kernel_storage(), _input(), _weights(), _output(), _reshaped_kernel(false), _conv()
+ : _memory_group(std::move(memory_manager)), _winograd_kernel(), _permute_input(), _permute_weights(), _permute_output(), _input_workspace(), _output_workspace(), _kernel_storage(), _input_nhwc(),
+ _output_nhwc(), _weights_hwio(), _input(), _weights(), _output(), _reshaped_kernel(false), _conv()
{
} /* arm_compute */
@@ -71,85 +72,107 @@
ARM_COMPUTE_ERROR_ON_MSG(stride_y != 1 || stride_x != 1, "Winograd layer only supports unit strides.");
// Get convolved dimensions
- auto padding = PADDING_VALID;
- const int in_channels = input->info()->dimension(2);
+ const int in_channels = input->info()->dimension(2);
+ const int out_channels = output->info()->dimension(2);
- const int out_channels = output->info()->dimension(2);
- const int weights_width = weights->info()->dimension(0);
- const int weights_height = weights->info()->dimension(1);
-
- const KernelShape kernel_shape({ out_channels, weights_height, weights_width, in_channels });
const Tensor4DShape in_shape(internal_get_input_shape(input));
// Get the memory required to instantiate a new Winograd operator.
- constexpr size_t kstore_alignment = 64;
- const size_t kernel_storage_per_thread = NEWinogradLayerKernel::get_kernel_storage_size(kernel_shape);
- _kernel_storage.allocator()->init(TensorInfo(TensorShape{ (kernel_storage_per_thread + kstore_alignment - 1) }, 1, DataType::U8));
+ constexpr size_t storage_alignment = 64;
+ const size_t kernel_storage_size = NEWinogradLayerKernel::get_weight_storage_size(out_channels, in_channels) * sizeof(float);
+ _kernel_storage.allocator()->init(TensorInfo(TensorShape{ (kernel_storage_size + storage_alignment - 1) }, 1, DataType::U8));
_memory_group.manage(&_kernel_storage);
-
- // Get workbench size and allocate memory
- constexpr size_t wspace_alignment = 64;
- const size_t ws_size = NEWinogradLayerKernel::get_working_space_size(in_shape, kernel_shape, padding);
- _workspace.allocator()->init(TensorInfo(TensorShape{ (ws_size + wspace_alignment - 1) }, 1, DataType::U8));
- _memory_group.manage(&_workspace);
-
- // Workspace for weights transform
- const size_t weights_transform_size = NEWinogradLayerKernel::get_kernel_transform_working_size(kernel_shape);
- _weights_workspace.allocator()->init(TensorInfo(TensorShape{ (weights_transform_size + wspace_alignment - 1) }, 1, DataType::U8));
- _memory_group.manage(&_weights_workspace);
-
+ _memory_group.manage(&_input_nhwc);
_kernel_storage.allocator()->allocate();
- _workspace.allocator()->allocate();
- _weights_workspace.allocator()->allocate();
+ // Input storage
+ const size_t input_storage_size = NEWinogradLayerKernel::get_input_storage_size(in_shape.n_batches, in_shape.n_channels, in_shape.n_rows, in_shape.n_cols, false) * sizeof(float);
+ _input_workspace.allocator()->init(TensorInfo(TensorShape{ (input_storage_size + storage_alignment - 1) }, 1, DataType::U8));
+ _memory_group.manage(&_input_workspace);
+ _input_workspace.allocator()->allocate();
+
+ // Output storage
+ const size_t output_storage_size = NEWinogradLayerKernel::get_output_storage_size(in_shape.n_batches, in_shape.n_rows, in_shape.n_cols, out_channels, false) * sizeof(float);
+ _output_workspace.allocator()->init(TensorInfo(TensorShape{ (output_storage_size + storage_alignment - 1) }, 1, DataType::U8));
+ _memory_group.manage(&_output_workspace);
+ _output_workspace.allocator()->allocate();
+
+ // configure and allocate dst tensor to be used to convert from winograd domain to spatial domain when calling to reshape_output()
+ TensorInfo info(TensorShape(_output->info()->dimension(2), _output->info()->dimension(0),
+ _output->info()->dimension(1), _output->info()->dimension(3)),
+ 1, _output->info()->data_type());
+ _output_nhwc.allocator()->init(info);
+
+ _output_nhwc.allocator()->allocate();
+
+ // Re-order a weight tensor from [Output feature map x Input feature map x Height x Width] to [Height x Width x Input feature map x Output feature map]
+ switch(weights->info()->num_dimensions())
+ {
+ case 3:
+ {
+ _permute_weights.configure(weights, &_weights_hwio, PermutationVector(2U, 0U, 1U));
+ break;
+ }
+ case 4:
+ {
+ _permute_weights.configure(weights, &_weights_hwio, PermutationVector(3U, 2U, 0U, 1U));
+ break;
+ }
+ default:
+ {
+ ARM_COMPUTE_ERROR("Not supported.");
+ break;
+ }
+ }
+
+ _weights_hwio.allocator()->allocate();
+
+ // configure the kernel to transform the input tensor from NCHW -> NHWC
+ _permute_input.configure(input, &_input_nhwc, PermutationVector(2U, 0U, 1U));
+
+ _input_nhwc.allocator()->allocate();
// Create Winograd operator object
- _conv = support::cpp14::make_unique<Winograd3x3F32>(kernel_shape, in_shape, padding, _kernel_storage.buffer());
+ _conv = support::cpp14::make_unique<Winograd3x3F32>(
+ in_shape.n_batches,
+ in_shape.n_channels,
+ in_shape.n_rows,
+ in_shape.n_cols,
+ out_channels,
+ false,
+ reinterpret_cast<const float *>(_weights_hwio.buffer()),
+ reinterpret_cast<float *>(_kernel_storage.buffer()),
+ reinterpret_cast<float *>(_input_nhwc.buffer()),
+ reinterpret_cast<float *>(_input_workspace.buffer()),
+ reinterpret_cast<float *>(_output_nhwc.buffer()),
+ reinterpret_cast<float *>(_output_workspace.buffer()));
// Configure the kernel, padding not needed so it's safe to call configure after allocare
- _winograd_kernel.configure(output, _conv.get());
+ _winograd_kernel.configure(_conv.get());
+
+ // Reorder the convoluted output to ACL's ordering NCHW
+ _permute_output.configure(&_output_nhwc, _output, PermutationVector(1U, 2U, 0U));
+
}
void NEWinogradLayer::run()
{
-#if defined(__aarch64__)
_memory_group.acquire();
if(!_reshaped_kernel)
{
- _conv->transform_weights(reinterpret_cast<const float *>(_weights->buffer()), reinterpret_cast<float *>(_weights_workspace.buffer()));
_reshaped_kernel = true;
+ _permute_weights.run();
+ _conv->transform_weights();
}
- const Tensor4DShape in_shape(internal_get_input_shape(_input));
- auto padding = PADDING_VALID;
-
//Bring channels to the front as Winograd code expects the tensor to be in the format NHWC
- _conv->nchw2nhwc(in_shape, padding, _workspace.buffer(), reinterpret_cast<const float *>(_input->buffer()));
-
- //Get ptrs into the workspace
- std::pair<void *, void *> nhwc_ptrs = _conv->get_nhwc_ptrs(in_shape, padding, _workspace.buffer());
-
- //Setup matrices ptrs and transfor the input tensor to the appropriate form before running GEMM.
- _conv->reshape_input(in_shape, padding, nhwc_ptrs.second, _workspace.buffer());
-
+ _permute_input.run();
+ // Transform input tensor to the winograd domain
+ _conv->transform_input();
//Run 16 GEMMs in multiple threads, each kernel runs one or more GEMMs
- NEScheduler::get().schedule(&_winograd_kernel, Window::DimY);
-
- //Transform the output to the appropriate form
- _conv->reshape_output(in_shape, padding, nhwc_ptrs.first);
-
- //Transform back to NCHW
- _conv->nhwc2nchw(in_shape, padding, _workspace.buffer(), reinterpret_cast<float *>(_output->buffer()));
-
+ NEScheduler::get().schedule(&_winograd_kernel, Window::DimX);
+ // Transform output tensor to the spatial domain
+ _conv->transform_output();
+ // Reorder the convoluted output to ACL's ordering NCHW
+ _permute_output.run();
_memory_group.release();
-#else /* __aarch64__ */
- ARM_COMPUTE_UNUSED(_winograd_kernel);
- ARM_COMPUTE_UNUSED(_workspace);
- ARM_COMPUTE_UNUSED(_kernel_storage);
- ARM_COMPUTE_UNUSED(_input);
- ARM_COMPUTE_UNUSED(_weights);
- ARM_COMPUTE_UNUSED(_output);
- ARM_COMPUTE_UNUSED(_reshaped_kernel);
- ARM_COMPUTE_UNUSED(_conv);
- ARM_COMPUTE_ERROR("Winograd only supported for aarch64, recompile with arch=arm64-v8a.");
-#endif /* __aarch64__ */
}
} // namespace arm_compute
diff --git a/src/runtime/SubTensor.cpp b/src/runtime/SubTensor.cpp
index 32924be..c5b8f33 100644
--- a/src/runtime/SubTensor.cpp
+++ b/src/runtime/SubTensor.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -27,11 +27,11 @@
using namespace arm_compute;
-SubTensor::SubTensor(ITensor *parent, const TensorShape &tensor_shape, const Coordinates &coords)
+SubTensor::SubTensor(ITensor *parent, const TensorShape &tensor_shape, const Coordinates &coords, bool extend_parent)
: _parent(nullptr), _info()
{
ARM_COMPUTE_ERROR_ON(parent == nullptr);
- _info = SubTensorInfo(parent->info(), tensor_shape, coords);
+ _info = SubTensorInfo(parent->info(), tensor_shape, coords, extend_parent);
_parent = parent;
}