blob: a89c4e3dbf2fbcf8b649ffcf3067f9708e05a205 [file] [log] [blame]
Jenkinsb3a371b2018-05-23 11:36:53 +01001/*
2 * Copyright (c) 2018 ARM Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/runtime/CL/functions/CLLSTMLayer.h"
25
26#include "arm_compute/core/PixelValue.h"
27#include "arm_compute/core/Utils.h"
28#include "arm_compute/core/Validate.h"
29#include "arm_compute/core/utils/misc/ShapeCalculator.h"
30#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
31#include "arm_compute/runtime/CL/CLScheduler.h"
32
33#include <cmath>
34#include <memory>
35#include <tuple>
36
37using namespace arm_compute;
38using namespace arm_compute::misc::shape_calculator;
39
40CLLSTMLayer::CLLSTMLayer(std::shared_ptr<IMemoryManager> memory_manager)
Jenkins52ba29e2018-08-29 15:32:11 +000041 : _memory_group(std::move(memory_manager)), _fully_connected_input_gate(), _gemm_input_gate(), _transpose_input_gate(), _accum_input_gate1(), _accum_input_gate2(), _subtract_input_gate(),
42 _pixelwise_mul_input_gate(), _activation_input_gate(), _fully_connected_forget_gate(), _gemm_forget_gate(), _transpose_forget_gate(), _accum_forget_gate1(), _accum_forget_gate2(),
43 _pixelwise_mul_forget_gate(), _activation_forget_gate(), _fully_connected_cell_state(), _gemm_cell_state1(), _gemm_cell_state2(), _transpose_cell_state(), _accum_cell_state1(), _accum_cell_state2(),
44 _pixelwise_mul_cell_state1(), _activation_cell_state(), _cell_clip(), _pixelwise_mul_cell_state2(), _fully_connected_output(), _gemm_output(), _pixelwise_mul_output_state1(), _transpose_output(),
45 _accum_output1(), _accum_output2(), _activation_output(), _activation_output_state(), _pixelwise_mul_output_state2(), _fully_connected_output_state(), _gemm_output_state(), _accum_output_state(),
46 _projection_clip(), _copy_cell_state(), _copy_output(), _concat_scratch_buffer(), _input_gate_out1(), _input_gate_out2(), _input_gate_out3(), _input_gate_out4(), _input_gate_out5(),
47 _forget_gate_out1(), _forget_gate_out2(), _forget_gate_out3(), _forget_gate_out4(), _forget_gate_out5(), _cell_state_out1(), _cell_state_out2(), _cell_state_out3(), _cell_state_out4(),
48 _cell_state_out5(), _output1(), _output2(), _output3(), _output4(), _output5(), _cell_state_activation(), _output_state1(), _ones(), _run_peephole_opt(false), _run_cifg_opt(false),
49 _perform_cell_clipping(false), _has_projection_weights(false), _perform_projection_clipping(false)
Jenkinsb3a371b2018-05-23 11:36:53 +010050{
51}
52
Jenkins52ba29e2018-08-29 15:32:11 +000053void CLLSTMLayer::configure(const ICLTensor *input,
54 const ICLTensor *input_to_forget_weights, const ICLTensor *input_to_cell_weights, const ICLTensor *input_to_output_weights,
Jenkinsb3a371b2018-05-23 11:36:53 +010055 const ICLTensor *recurrent_to_forget_weights, const ICLTensor *recurrent_to_cell_weights, const ICLTensor *recurrent_to_output_weights,
56 const ICLTensor *forget_gate_bias, const ICLTensor *cell_bias, const ICLTensor *output_gate_bias,
Jenkins52ba29e2018-08-29 15:32:11 +000057 const ICLTensor *output_state_in, const ICLTensor *cell_state_in,
58 ICLTensor *scratch_buffer, ICLTensor *output_state_out, ICLTensor *cell_state_out, ICLTensor *output,
59 const LSTMParams<ICLTensor> &lstm_params, const ActivationLayerInfo &activation_info, float cell_threshold, float projection_threshold)
Jenkinsb3a371b2018-05-23 11:36:53 +010060{
Jenkins52ba29e2018-08-29 15:32:11 +000061 ARM_COMPUTE_ERROR_ON_NULLPTR(input,
62 input_to_forget_weights, input_to_cell_weights, input_to_output_weights,
63 recurrent_to_forget_weights, recurrent_to_cell_weights, recurrent_to_output_weights,
64 forget_gate_bias, cell_bias, output_gate_bias,
65 output_state_in, cell_state_in,
66 scratch_buffer, output_state_out, cell_state_out, output);
67
68 // Set lstm parameters
Jenkinsb3a371b2018-05-23 11:36:53 +010069 LSTMParams<ITensorInfo> lstm_params_info;
70 if(lstm_params.has_peephole_opt())
71 {
Jenkins52ba29e2018-08-29 15:32:11 +000072 lstm_params_info.set_peephole_params(lstm_params.cell_to_forget_weights()->info(), lstm_params.cell_to_output_weights()->info());
Jenkinsb3a371b2018-05-23 11:36:53 +010073 }
74 if(lstm_params.has_projection())
75 {
Jenkins52ba29e2018-08-29 15:32:11 +000076 lstm_params_info.set_projection_params(lstm_params.projection_weights()->info(),
77 lstm_params.projection_bias() != nullptr ? lstm_params.projection_bias()->info() : nullptr);
Jenkinsb3a371b2018-05-23 11:36:53 +010078 }
79 if(!lstm_params.has_cifg_opt())
80 {
Jenkins52ba29e2018-08-29 15:32:11 +000081 const ITensorInfo *cell_to_input_weights_info = (lstm_params.has_peephole_opt()) ? lstm_params.cell_to_input_weights()->info() : nullptr;
Jenkinsb3a371b2018-05-23 11:36:53 +010082 lstm_params_info.set_cifg_params(lstm_params.input_to_input_weights()->info(), lstm_params.recurrent_to_input_weights()->info(),
Jenkins52ba29e2018-08-29 15:32:11 +000083 cell_to_input_weights_info, lstm_params.input_gate_bias()->info());
Jenkinsb3a371b2018-05-23 11:36:53 +010084 }
Jenkins52ba29e2018-08-29 15:32:11 +000085
86 // Validate
Jenkinsb3a371b2018-05-23 11:36:53 +010087 ARM_COMPUTE_ERROR_THROW_ON(CLLSTMLayer::validate(input->info(), input_to_forget_weights->info(),
88 input_to_cell_weights->info(), input_to_output_weights->info(),
89 recurrent_to_forget_weights->info(), recurrent_to_cell_weights->info(), recurrent_to_output_weights->info(),
90 forget_gate_bias->info(), cell_bias->info(), output_gate_bias->info(),
Jenkins52ba29e2018-08-29 15:32:11 +000091 output_state_in->info(), cell_state_in->info(),
92 scratch_buffer->info(), output_state_out->info(), cell_state_out->info(), output->info(),
93 lstm_params_info, activation_info, cell_threshold, projection_threshold));
Jenkinsb3a371b2018-05-23 11:36:53 +010094
Jenkins52ba29e2018-08-29 15:32:11 +000095 const TensorShape cell_state_shape = cell_state_in->info()->tensor_shape();
Jenkinsb3a371b2018-05-23 11:36:53 +010096
Jenkins52ba29e2018-08-29 15:32:11 +000097 // Configure block that calculates the forget gate
98 // forget_gate = Activation(input * input_to_forget_weights + output_state_in * recurrent_to_forget_weights + PixelWiseMul(cell_state, cell_to_forget_weights) + forget_gate_bias)
Jenkinsb3a371b2018-05-23 11:36:53 +010099 TensorShape forget_gate1_shape = compute_transposed_shape(*recurrent_to_output_weights->info());
Jenkinsb3a371b2018-05-23 11:36:53 +0100100 _forget_gate_out1.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
101 _forget_gate_out2.allocator()->init(TensorInfo(forget_gate1_shape, 1, input->info()->data_type()));
102 _forget_gate_out3.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkins52ba29e2018-08-29 15:32:11 +0000103 _forget_gate_out5.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100104
Jenkinsb3a371b2018-05-23 11:36:53 +0100105 _memory_group.manage(&_forget_gate_out1);
Jenkins52ba29e2018-08-29 15:32:11 +0000106 _fully_connected_forget_gate.configure(input, input_to_forget_weights, forget_gate_bias, &_forget_gate_out1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100107 _memory_group.manage(&_forget_gate_out2);
Jenkins52ba29e2018-08-29 15:32:11 +0000108 _transpose_forget_gate.configure(recurrent_to_forget_weights, &_forget_gate_out2);
Jenkinsb3a371b2018-05-23 11:36:53 +0100109 _memory_group.manage(&_forget_gate_out3);
Jenkins52ba29e2018-08-29 15:32:11 +0000110 _gemm_forget_gate.configure(output_state_in, &_forget_gate_out2, nullptr, &_forget_gate_out3, 1.f, 0.f);
Jenkinsb3a371b2018-05-23 11:36:53 +0100111 _forget_gate_out2.allocator()->allocate();
Jenkins52ba29e2018-08-29 15:32:11 +0000112 _memory_group.manage(&_forget_gate_out5);
113 _accum_forget_gate1.configure(&_forget_gate_out1, &_forget_gate_out3, &_forget_gate_out5, ConvertPolicy::SATURATE);
114 CLTensor *forget_gate_out = &_forget_gate_out5;
Jenkinsb3a371b2018-05-23 11:36:53 +0100115
116 if(lstm_params.has_peephole_opt())
117 {
Jenkins52ba29e2018-08-29 15:32:11 +0000118 _forget_gate_out4.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100119
120 _run_peephole_opt = true;
121 _memory_group.manage(&_forget_gate_out4);
Jenkins52ba29e2018-08-29 15:32:11 +0000122 _pixelwise_mul_forget_gate.configure(cell_state_in, lstm_params.cell_to_forget_weights(), &_forget_gate_out4, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
123 _accum_forget_gate2.configure(&_forget_gate_out5, &_forget_gate_out4, &_forget_gate_out3, ConvertPolicy::SATURATE);
Jenkinsb3a371b2018-05-23 11:36:53 +0100124 _forget_gate_out4.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100125 _forget_gate_out5.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100126 forget_gate_out = &_forget_gate_out3;
127 }
128 else
129 {
130 _forget_gate_out3.allocator()->allocate();
131 }
132 _activation_forget_gate.configure(forget_gate_out, &_forget_gate_out1, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC));
Jenkinsb3a371b2018-05-23 11:36:53 +0100133
Jenkinsb3a371b2018-05-23 11:36:53 +0100134 // Configure block that calculates the input gate
Jenkins52ba29e2018-08-29 15:32:11 +0000135 // input_gate = Activation(input * input_to_input_weights + output_state * recurrent_to_input_weights + PixelWiseMul(cell_state, cell_to_input_weights) + input_gate_bias), without CIFG
Jenkinsb3a371b2018-05-23 11:36:53 +0100136 // input_gate = 1 - forget_gate, with CIFG
Jenkins52ba29e2018-08-29 15:32:11 +0000137 _input_gate_out1.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100138 if(lstm_params.has_cifg_opt())
139 {
140 _memory_group.manage(&_input_gate_out1);
141 _ones.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
142 _subtract_input_gate.configure(&_ones, &_forget_gate_out1, &_input_gate_out1, ConvertPolicy::SATURATE);
143 _ones.allocator()->allocate();
144 _run_cifg_opt = true;
145 }
146 else
147 {
Jenkins52ba29e2018-08-29 15:32:11 +0000148 TensorShape input_gate_shape = compute_transposed_shape(*recurrent_to_output_weights->info());
Jenkinsb3a371b2018-05-23 11:36:53 +0100149
Jenkins52ba29e2018-08-29 15:32:11 +0000150 _input_gate_out2.allocator()->init(TensorInfo(input_gate_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100151 _input_gate_out3.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkins52ba29e2018-08-29 15:32:11 +0000152 _input_gate_out4.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
153 _input_gate_out5.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100154
155 _memory_group.manage(&_input_gate_out1);
Jenkins52ba29e2018-08-29 15:32:11 +0000156 _fully_connected_input_gate.configure(input, lstm_params.input_to_input_weights(), lstm_params.input_gate_bias(), &_input_gate_out1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100157 _memory_group.manage(&_input_gate_out2);
Jenkins52ba29e2018-08-29 15:32:11 +0000158 _transpose_input_gate.configure(lstm_params.recurrent_to_input_weights(), &_input_gate_out2);
Jenkinsb3a371b2018-05-23 11:36:53 +0100159 _memory_group.manage(&_input_gate_out3);
Jenkins52ba29e2018-08-29 15:32:11 +0000160 _gemm_input_gate.configure(output_state_in, &_input_gate_out2, nullptr, &_input_gate_out3, 1.f, 0.f);
Jenkinsb3a371b2018-05-23 11:36:53 +0100161 _input_gate_out2.allocator()->allocate();
162 _memory_group.manage(&_input_gate_out4);
Jenkins52ba29e2018-08-29 15:32:11 +0000163 _accum_input_gate1.configure(&_input_gate_out1, &_input_gate_out3, &_input_gate_out4, ConvertPolicy::SATURATE);
164 if(_run_peephole_opt)
165 {
166 _memory_group.manage(&_input_gate_out5);
167 _pixelwise_mul_input_gate.configure(cell_state_in, lstm_params.cell_to_input_weights(), &_input_gate_out5, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
168 _accum_input_gate2.configure(&_input_gate_out4, &_input_gate_out5, &_input_gate_out1, ConvertPolicy::SATURATE);
169 _input_gate_out5.allocator()->allocate();
170 }
Jenkinsb3a371b2018-05-23 11:36:53 +0100171 _input_gate_out3.allocator()->allocate();
Jenkins52ba29e2018-08-29 15:32:11 +0000172 _input_gate_out4.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100173 _activation_input_gate.configure(&_input_gate_out1, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC));
174 }
175
Jenkins52ba29e2018-08-29 15:32:11 +0000176 // Configure block that calculates the cell state
177 // cell_state = Clip((PixelwiseMul(input_gate, Activation(input * input_to_cell_weights + output_state_in * recurrent_to_cell_weights + cell_bias)) + PixelwiseMul(forget_gate, cell_state)), cell_threshold)
Jenkinsb3a371b2018-05-23 11:36:53 +0100178 TensorShape cell_state1_shape = compute_transposed_shape(*recurrent_to_output_weights->info());
179 _cell_state_out1.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
180 _cell_state_out2.allocator()->init(TensorInfo(cell_state1_shape, 1, input->info()->data_type()));
181 _cell_state_out3.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
182 _cell_state_out4.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
183 _cell_state_out5.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
184
Jenkinsb3a371b2018-05-23 11:36:53 +0100185 _memory_group.manage(&_cell_state_out1);
Jenkins52ba29e2018-08-29 15:32:11 +0000186 _fully_connected_cell_state.configure(input, input_to_cell_weights, cell_bias, &_cell_state_out1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100187 _memory_group.manage(&_cell_state_out2);
Jenkins52ba29e2018-08-29 15:32:11 +0000188 _transpose_cell_state.configure(recurrent_to_cell_weights, &_cell_state_out2);
Jenkinsb3a371b2018-05-23 11:36:53 +0100189 _memory_group.manage(&_cell_state_out3);
Jenkins52ba29e2018-08-29 15:32:11 +0000190 _gemm_cell_state1.configure(output_state_in, &_cell_state_out2, nullptr, &_cell_state_out3, 1.f, 0.f);
Jenkinsb3a371b2018-05-23 11:36:53 +0100191 _cell_state_out2.allocator()->allocate();
192 _memory_group.manage(&_cell_state_out4);
193 _accum_cell_state1.configure(&_cell_state_out1, &_cell_state_out3, &_cell_state_out4, ConvertPolicy::SATURATE);
194 _activation_cell_state.configure(&_cell_state_out4, nullptr, activation_info);
195 _memory_group.manage(&_cell_state_out5);
196 _pixelwise_mul_cell_state1.configure(&_cell_state_out4, &_input_gate_out1, &_cell_state_out5, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
Jenkinsb3a371b2018-05-23 11:36:53 +0100197 _cell_state_out4.allocator()->allocate();
Jenkins52ba29e2018-08-29 15:32:11 +0000198 _pixelwise_mul_cell_state2.configure(&_forget_gate_out1, cell_state_in, &_cell_state_out3, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
Jenkinsb3a371b2018-05-23 11:36:53 +0100199 _forget_gate_out1.allocator()->allocate();
200 _accum_cell_state2.configure(&_cell_state_out5, &_cell_state_out3, &_cell_state_out1, ConvertPolicy::SATURATE);
201 _cell_state_out3.allocator()->allocate();
202 _cell_state_out5.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100203 // Perform clipping
204 if(cell_threshold != 0.f)
205 {
206 _perform_cell_clipping = true;
207 _cell_clip.configure(&_cell_state_out1, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, -cell_threshold, cell_threshold));
208 }
209
Jenkins52ba29e2018-08-29 15:32:11 +0000210 // Configure block that calculates the output
211 // output_state_out = Activation(input * input_to_output_weights + output_state_in * recurrent_to_output_weights + PixelWiseMul(cell_state, cell_to_output_weights) + output_gate_bias)
Jenkinsb3a371b2018-05-23 11:36:53 +0100212 TensorShape output1_shape = compute_transposed_shape(*recurrent_to_output_weights->info());
Jenkinsb3a371b2018-05-23 11:36:53 +0100213 _output1.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
214 _output2.allocator()->init(TensorInfo(output1_shape, 1, input->info()->data_type()));
215 _output3.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkins52ba29e2018-08-29 15:32:11 +0000216 _output5.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100217
Jenkinsb3a371b2018-05-23 11:36:53 +0100218 _memory_group.manage(&_output1);
Jenkins52ba29e2018-08-29 15:32:11 +0000219 _fully_connected_output.configure(input, input_to_output_weights, output_gate_bias, &_output1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100220 _memory_group.manage(&_output2);
Jenkins52ba29e2018-08-29 15:32:11 +0000221 _transpose_output.configure(recurrent_to_output_weights, &_output2);
Jenkinsb3a371b2018-05-23 11:36:53 +0100222 _memory_group.manage(&_output3);
Jenkins52ba29e2018-08-29 15:32:11 +0000223 _gemm_output.configure(output_state_in, &_output2, nullptr, &_output3, 1.f, 0.f);
Jenkinsb3a371b2018-05-23 11:36:53 +0100224 _output2.allocator()->allocate();
Jenkins52ba29e2018-08-29 15:32:11 +0000225 _memory_group.manage(&_output5);
226 _accum_output1.configure(&_output1, &_output3, &_output5, ConvertPolicy::SATURATE);
Jenkinsb3a371b2018-05-23 11:36:53 +0100227 _output3.allocator()->allocate();
Jenkins52ba29e2018-08-29 15:32:11 +0000228 CLTensor *output_gate_out = &_output5;
Jenkinsb3a371b2018-05-23 11:36:53 +0100229 if(lstm_params.has_peephole_opt())
230 {
Jenkins52ba29e2018-08-29 15:32:11 +0000231 _output4.allocator()->init(TensorInfo(_cell_state_out1.info()->tensor_shape(), 1, input->info()->data_type()));
Jenkinsb3a371b2018-05-23 11:36:53 +0100232
233 _memory_group.manage(&_output4);
Jenkins52ba29e2018-08-29 15:32:11 +0000234 _pixelwise_mul_output_state1.configure(&_cell_state_out1, lstm_params.cell_to_output_weights(), &_output4, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
235 _accum_output2.configure(&_output5, &_output4, &_output1, ConvertPolicy::SATURATE);
236 _output5.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100237 output_gate_out = &_output1;
238
239 // Allocate intermediate buffers
240 _output4.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100241 }
242 else
243 {
244 _output1.allocator()->allocate();
245 }
Jenkins52ba29e2018-08-29 15:32:11 +0000246 _activation_output.configure(output_gate_out, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC));
Jenkinsb3a371b2018-05-23 11:36:53 +0100247
Jenkinsb3a371b2018-05-23 11:36:53 +0100248 // Configure block that calculates the output state
249 /** lstm_res = PixelwiseMul(output, Activation(cell_state))
250 *
251 * -- Clip(lstm_res * projection_weights + projection_bias, projection_threshold) , if there is a projection
252 * /
253 * output_state = --
254 * \
255 * -- lstm_res , otherwise
256 */
Jenkins52ba29e2018-08-29 15:32:11 +0000257 ICLTensor *output_state_out_tmp = lstm_params.has_projection() ? &_output_state1 : output_state_out;
258 _cell_state_activation.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
259 _output_state1.allocator()->init(TensorInfo(cell_state_shape, 1, input->info()->data_type()));
260
Jenkinsb3a371b2018-05-23 11:36:53 +0100261 _memory_group.manage(&_cell_state_activation);
262 _activation_output_state.configure(&_cell_state_out1, &_cell_state_activation, activation_info);
Jenkins52ba29e2018-08-29 15:32:11 +0000263 _pixelwise_mul_output_state2.configure(&_cell_state_activation, output_gate_out, output_state_out_tmp, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN);
Jenkinsb3a371b2018-05-23 11:36:53 +0100264 _cell_state_activation.allocator()->allocate();
265
266 if(lstm_params.has_projection())
267 {
268 _has_projection_weights = true;
Jenkins52ba29e2018-08-29 15:32:11 +0000269 _fully_connected_output_state.configure(output_state_out_tmp, lstm_params.projection_weights(), lstm_params.projection_bias(), output_state_out);
270 _output_state1.allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100271 // Perform clipping
272 if(projection_threshold != 0.f)
273 {
274 _perform_projection_clipping = true;
Jenkins52ba29e2018-08-29 15:32:11 +0000275 _projection_clip.configure(output_state_out, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, -projection_threshold, projection_threshold));
Jenkinsb3a371b2018-05-23 11:36:53 +0100276 }
Jenkinsb3a371b2018-05-23 11:36:53 +0100277 }
278
279 // Copy cell state and output
Jenkins52ba29e2018-08-29 15:32:11 +0000280 _copy_cell_state.configure(&_cell_state_out1, cell_state_out);
Jenkins52ba29e2018-08-29 15:32:11 +0000281 _copy_output.configure(output_state_out, output);
Jenkinsb3a371b2018-05-23 11:36:53 +0100282
283 // Vector for holding the tensors to store in scratch buffer
284 std::vector<ICLTensor *> scratch_inputs;
Jenkinsb9abeae2018-11-22 11:58:08 +0000285 if(!lstm_params.has_cifg_opt())
Jenkinsb3a371b2018-05-23 11:36:53 +0100286 {
287 scratch_inputs.emplace_back(&_input_gate_out1);
288 }
289 scratch_inputs.emplace_back(&_cell_state_out1);
290 scratch_inputs.emplace_back(forget_gate_out);
291 scratch_inputs.emplace_back(output_gate_out);
292 _concat_scratch_buffer.configure(scratch_inputs, scratch_buffer);
Jenkinsb9abeae2018-11-22 11:58:08 +0000293 _input_gate_out1.allocator()->allocate();
294 _cell_state_out1.allocator()->allocate();
295 forget_gate_out->allocator()->allocate();
296 output_gate_out->allocator()->allocate();
Jenkinsb3a371b2018-05-23 11:36:53 +0100297}
298
Jenkins52ba29e2018-08-29 15:32:11 +0000299Status CLLSTMLayer::validate(const ITensorInfo *input,
300 const ITensorInfo *input_to_forget_weights, const ITensorInfo *input_to_cell_weights, const ITensorInfo *input_to_output_weights,
Jenkinsb3a371b2018-05-23 11:36:53 +0100301 const ITensorInfo *recurrent_to_forget_weights, const ITensorInfo *recurrent_to_cell_weights, const ITensorInfo *recurrent_to_output_weights,
302 const ITensorInfo *forget_gate_bias, const ITensorInfo *cell_bias, const ITensorInfo *output_gate_bias,
Jenkins52ba29e2018-08-29 15:32:11 +0000303 const ITensorInfo *output_state_in, const ITensorInfo *cell_state_in,
304 const ITensorInfo *scratch_buffer, const ITensorInfo *output_state_out, const ITensorInfo *cell_state_out, const ITensorInfo *output,
Jenkinsb3a371b2018-05-23 11:36:53 +0100305 const LSTMParams<ITensorInfo> &lstm_params, const ActivationLayerInfo &activation_info, float cell_threshold, float projection_threshold)
306{
Jenkins52ba29e2018-08-29 15:32:11 +0000307 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input,
308 input_to_forget_weights, input_to_cell_weights, input_to_output_weights,
309 recurrent_to_forget_weights, recurrent_to_cell_weights, recurrent_to_output_weights,
310 forget_gate_bias, cell_bias, output_gate_bias,
311 output_state_in, cell_state_in,
312 scratch_buffer, output_state_out, cell_state_out, output);
Jenkinsb3a371b2018-05-23 11:36:53 +0100313
Jenkins52ba29e2018-08-29 15:32:11 +0000314 // Check data types
315 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
316 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input,
317 input_to_forget_weights, input_to_cell_weights, input_to_output_weights,
318 recurrent_to_forget_weights, recurrent_to_cell_weights, recurrent_to_output_weights,
319 forget_gate_bias, cell_bias, output_gate_bias,
320 output_state_in, cell_state_in,
321 scratch_buffer, output_state_out, cell_state_out, output);
322
323 // Check dimensions
324 ARM_COMPUTE_RETURN_ERROR_ON(input->num_dimensions() > 2);
325 ARM_COMPUTE_RETURN_ERROR_ON(input_to_forget_weights->num_dimensions() > 2);
326 ARM_COMPUTE_RETURN_ERROR_ON(input_to_cell_weights->num_dimensions() > 2);
327 ARM_COMPUTE_RETURN_ERROR_ON(input_to_output_weights->num_dimensions() > 2);
328 ARM_COMPUTE_RETURN_ERROR_ON(recurrent_to_forget_weights->num_dimensions() > 2);
329 ARM_COMPUTE_RETURN_ERROR_ON(recurrent_to_cell_weights->num_dimensions() > 2);
330 ARM_COMPUTE_RETURN_ERROR_ON(recurrent_to_output_weights->num_dimensions() > 2);
331 ARM_COMPUTE_RETURN_ERROR_ON(forget_gate_bias->num_dimensions() > 1);
332 ARM_COMPUTE_RETURN_ERROR_ON(cell_bias->num_dimensions() > 1);
333 ARM_COMPUTE_RETURN_ERROR_ON(output_gate_bias->num_dimensions() > 1);
334 ARM_COMPUTE_RETURN_ERROR_ON(output_state_in->num_dimensions() > 2);
335 ARM_COMPUTE_RETURN_ERROR_ON(cell_state_in->num_dimensions() > 2);
336 ARM_COMPUTE_RETURN_ERROR_ON(scratch_buffer->num_dimensions() > 2);
337 ARM_COMPUTE_RETURN_ERROR_ON(output_state_out->num_dimensions() > 2);
338 ARM_COMPUTE_RETURN_ERROR_ON(cell_state_out->num_dimensions() > 2);
339 ARM_COMPUTE_RETURN_ERROR_ON(output->num_dimensions() > 2);
340 ARM_COMPUTE_RETURN_ERROR_ON(cell_bias->dimension(0) * 4 != scratch_buffer->dimension(0)
341 && cell_bias->dimension(0) * 3 != scratch_buffer->dimension(0));
342
343 const unsigned int num_batches = input->dimension(1);
344 const unsigned int num_cells = input_to_output_weights->dimension(1);
345
346 // Check peephole optimization
Jenkinsb3a371b2018-05-23 11:36:53 +0100347 if(lstm_params.has_peephole_opt())
348 {
Jenkins52ba29e2018-08-29 15:32:11 +0000349 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lstm_params.cell_to_output_weights(), lstm_params.cell_to_forget_weights());
350 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.cell_to_forget_weights()->num_dimensions() > 1);
351 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.cell_to_output_weights()->num_dimensions() > 1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100352 }
353
354 TensorShape units_out_transposed_shape = compute_transposed_shape(*recurrent_to_output_weights);
Jenkinsb3a371b2018-05-23 11:36:53 +0100355 TensorShape num_units_transposed_shape = compute_transposed_shape(*forget_gate_bias);
356 const TensorInfo units_out_transposed_info = TensorInfo(units_out_transposed_shape, 1, input->data_type());
Jenkinsb3a371b2018-05-23 11:36:53 +0100357 const TensorInfo num_units_transposed_info = TensorInfo(num_units_transposed_shape, 1, input->data_type());
358
Jenkins52ba29e2018-08-29 15:32:11 +0000359 TensorInfo input_gate = TensorInfo(TensorShape(num_cells, num_batches), 1, input->data_type());
360 TensorInfo forget_gate = TensorInfo(TensorShape(num_cells, num_batches), 1, input->data_type());
361 TensorInfo output_gate_tmp = TensorInfo(TensorShape(num_cells, num_batches), 1, input->data_type());
362 TensorInfo cell_state_tmp = TensorInfo(TensorShape(num_cells, num_batches), 1, input->data_type());
363
Jenkinsb3a371b2018-05-23 11:36:53 +0100364 // Validate forget gate
Jenkins52ba29e2018-08-29 15:32:11 +0000365 ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayer::validate(input, input_to_forget_weights, forget_gate_bias, &forget_gate));
366 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMM::validate(output_state_in, &units_out_transposed_info, nullptr, &forget_gate, 1.f, 0.f, GEMMInfo()));
367 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAdditionKernel::validate(&forget_gate, &forget_gate, &forget_gate, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100368 if(lstm_params.has_peephole_opt())
369 {
Jenkins52ba29e2018-08-29 15:32:11 +0000370 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(cell_state_in, lstm_params.cell_to_forget_weights(), &forget_gate, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN));
371 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&forget_gate, &forget_gate, &forget_gate, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100372 }
Jenkins52ba29e2018-08-29 15:32:11 +0000373 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&forget_gate, &forget_gate, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC)));
Jenkinsb3a371b2018-05-23 11:36:53 +0100374
375 // Validate input gate
376 if(!lstm_params.has_cifg_opt())
377 {
Jenkins52ba29e2018-08-29 15:32:11 +0000378 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lstm_params.input_to_input_weights(),
379 lstm_params.recurrent_to_input_weights(),
380 lstm_params.input_gate_bias());
381 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.input_to_input_weights()->num_dimensions() > 2);
382 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.recurrent_to_input_weights()->num_dimensions() > 2);
383 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.input_gate_bias()->num_dimensions() > 1);
384
385 ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayer::validate(input, lstm_params.input_to_input_weights(), lstm_params.input_gate_bias(), &input_gate));
386 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMM::validate(output_state_in, &units_out_transposed_info, nullptr, &input_gate, 1.f, 0.f, GEMMInfo()));
387 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&input_gate, &input_gate, &input_gate, ConvertPolicy::SATURATE));
388 if(lstm_params.has_peephole_opt())
389 {
390 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lstm_params.cell_to_input_weights());
391 ARM_COMPUTE_RETURN_ERROR_ON(lstm_params.cell_to_input_weights()->num_dimensions() > 1);
392 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(cell_state_in, lstm_params.cell_to_input_weights(), &input_gate, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN));
393 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&input_gate, &input_gate, &input_gate, ConvertPolicy::SATURATE));
394 }
395 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&input_gate, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC)));
Jenkinsb3a371b2018-05-23 11:36:53 +0100396 }
397 else
398 {
Jenkins52ba29e2018-08-29 15:32:11 +0000399 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticSubtractionKernel::validate(&forget_gate, &forget_gate, &forget_gate, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100400 }
401
402 // Validate cell state
Jenkins52ba29e2018-08-29 15:32:11 +0000403 ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayer::validate(input, input_to_cell_weights, cell_bias, &cell_state_tmp));
404 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMM::validate(output_state_in, &units_out_transposed_info, nullptr, &cell_state_tmp, 1.f, 0.f, GEMMInfo()));
405 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&cell_state_tmp, &cell_state_tmp, &cell_state_tmp, ConvertPolicy::SATURATE));
406 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&cell_state_tmp, nullptr, activation_info));
407 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(&cell_state_tmp, &input_gate, &cell_state_tmp, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN));
408 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(&cell_state_tmp, &forget_gate, &cell_state_tmp, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN));
409 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&cell_state_tmp, &cell_state_tmp, &cell_state_tmp, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100410 if(cell_threshold != 0.f)
411 {
Jenkins52ba29e2018-08-29 15:32:11 +0000412 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&cell_state_tmp, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, -cell_threshold,
413 cell_threshold)));
Jenkinsb3a371b2018-05-23 11:36:53 +0100414 }
415
Jenkins52ba29e2018-08-29 15:32:11 +0000416 // Validate output gate tmp
417 ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayer::validate(input, input_to_output_weights, output_gate_bias, &output_gate_tmp));
418 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMM::validate(output_state_in, &units_out_transposed_info, nullptr, &output_gate_tmp, 1.f, 0.f, GEMMInfo()));
419 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&output_gate_tmp, &output_gate_tmp, &output_gate_tmp, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100420 if(lstm_params.has_peephole_opt())
421 {
Jenkins52ba29e2018-08-29 15:32:11 +0000422 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(&cell_state_tmp, lstm_params.cell_to_output_weights(), &output_gate_tmp, 1, ConvertPolicy::SATURATE,
423 RoundingPolicy::TO_NEAREST_EVEN));
424 ARM_COMPUTE_RETURN_ON_ERROR(CLArithmeticAddition::validate(&output_gate_tmp, &output_gate_tmp, &output_gate_tmp, ConvertPolicy::SATURATE));
Jenkinsb3a371b2018-05-23 11:36:53 +0100425 }
Jenkins52ba29e2018-08-29 15:32:11 +0000426 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&output_gate_tmp, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC)));
Jenkinsb3a371b2018-05-23 11:36:53 +0100427
428 // Validate output state
Jenkins52ba29e2018-08-29 15:32:11 +0000429 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(&cell_state_tmp, &cell_state_tmp, activation_info));
430 ARM_COMPUTE_RETURN_ON_ERROR(CLPixelWiseMultiplicationKernel::validate(&cell_state_tmp, &output_gate_tmp, &output_gate_tmp, 1, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_EVEN));
Jenkinsb3a371b2018-05-23 11:36:53 +0100431 if(lstm_params.has_projection())
432 {
Jenkins52ba29e2018-08-29 15:32:11 +0000433 ARM_COMPUTE_RETURN_ON_ERROR(CLFullyConnectedLayer::validate(&output_gate_tmp, lstm_params.projection_weights(), lstm_params.projection_bias(), output_state_out));
Jenkinsb3a371b2018-05-23 11:36:53 +0100434 if(projection_threshold != 0.f)
435 {
Jenkins52ba29e2018-08-29 15:32:11 +0000436 ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayerKernel::validate(output_state_out, output_state_out,
437 ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, -projection_threshold, projection_threshold)));
Jenkinsb3a371b2018-05-23 11:36:53 +0100438 }
439 }
440
Jenkins52ba29e2018-08-29 15:32:11 +0000441 // Validate copy kernel
442 ARM_COMPUTE_RETURN_ON_ERROR(CLCopyKernel::validate(&cell_state_tmp, cell_state_out));
443 ARM_COMPUTE_RETURN_ON_ERROR(CLCopyKernel::validate(output_state_out, output));
444
445 // Validate scratch concatenation
446 std::vector<ITensorInfo *> inputs_vector_info_raw;
Jenkinsb9abeae2018-11-22 11:58:08 +0000447 if(!lstm_params.has_cifg_opt())
Jenkinsb3a371b2018-05-23 11:36:53 +0100448 {
Jenkins52ba29e2018-08-29 15:32:11 +0000449 inputs_vector_info_raw.push_back(&input_gate);
Jenkinsb3a371b2018-05-23 11:36:53 +0100450 }
Jenkins52ba29e2018-08-29 15:32:11 +0000451 inputs_vector_info_raw.push_back(&cell_state_tmp);
452 inputs_vector_info_raw.push_back(&forget_gate);
453 inputs_vector_info_raw.push_back(&output_gate_tmp);
Jenkinsb3a371b2018-05-23 11:36:53 +0100454
455 ARM_COMPUTE_RETURN_ON_ERROR(CLWidthConcatenateLayer::validate(inputs_vector_info_raw, scratch_buffer));
456 return Status{};
457}
458
459void CLLSTMLayer::run()
460{
461 _memory_group.acquire();
462
463 _fully_connected_forget_gate.run();
Jenkins52ba29e2018-08-29 15:32:11 +0000464 CLScheduler::get().enqueue(_transpose_forget_gate);
465 _gemm_forget_gate.run();
Jenkinsb3a371b2018-05-23 11:36:53 +0100466 CLScheduler::get().enqueue(_accum_forget_gate1);
467
468 if(_run_peephole_opt)
469 {
Jenkins52ba29e2018-08-29 15:32:11 +0000470 CLScheduler::get().enqueue(_pixelwise_mul_forget_gate);
Jenkinsb3a371b2018-05-23 11:36:53 +0100471 _accum_forget_gate2.run();
472 }
473 CLScheduler::get().enqueue(_activation_forget_gate);
474
475 if(_run_cifg_opt)
476 {
477 _ones.map(true);
Jenkins52ba29e2018-08-29 15:32:11 +0000478 if(_ones.info()->data_type() == DataType::F16)
479 {
480 std::fill_n(reinterpret_cast<half *>(_ones.buffer()), _ones.info()->total_size() / _ones.info()->element_size(), 1);
481 }
482 else
483 {
484 std::fill_n(reinterpret_cast<float *>(_ones.buffer()), _ones.info()->total_size() / _ones.info()->element_size(), 1);
485 }
Jenkinsb3a371b2018-05-23 11:36:53 +0100486 _ones.unmap();
487 CLScheduler::get().enqueue(_subtract_input_gate);
488 }
489 else
490 {
491 _fully_connected_input_gate.run();
Jenkins52ba29e2018-08-29 15:32:11 +0000492 CLScheduler::get().enqueue(_transpose_input_gate);
493 _gemm_input_gate.run();
Jenkinsb3a371b2018-05-23 11:36:53 +0100494 CLScheduler::get().enqueue(_accum_input_gate1);
Jenkins52ba29e2018-08-29 15:32:11 +0000495 if(_run_peephole_opt)
496 {
497 CLScheduler::get().enqueue(_pixelwise_mul_input_gate);
498 _accum_input_gate2.run();
499 }
Jenkinsb3a371b2018-05-23 11:36:53 +0100500 CLScheduler::get().enqueue(_activation_input_gate);
501 }
502
503 _fully_connected_cell_state.run();
Jenkins52ba29e2018-08-29 15:32:11 +0000504 CLScheduler::get().enqueue(_transpose_cell_state);
Jenkinsb3a371b2018-05-23 11:36:53 +0100505 _gemm_cell_state1.run();
506 CLScheduler::get().enqueue(_accum_cell_state1);
507 CLScheduler::get().enqueue(_activation_cell_state);
508 CLScheduler::get().enqueue(_pixelwise_mul_cell_state1);
509 CLScheduler::get().enqueue(_pixelwise_mul_cell_state2);
510 CLScheduler::get().enqueue(_accum_cell_state2);
511
512 if(_perform_cell_clipping)
513 {
514 CLScheduler::get().enqueue(_cell_clip);
515 }
516
517 _fully_connected_output.run();
Jenkins52ba29e2018-08-29 15:32:11 +0000518 CLScheduler::get().enqueue(_transpose_output);
519 _gemm_output.run();
Jenkinsb3a371b2018-05-23 11:36:53 +0100520 CLScheduler::get().enqueue(_accum_output1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100521
522 if(_run_peephole_opt)
523 {
Jenkins52ba29e2018-08-29 15:32:11 +0000524 CLScheduler::get().enqueue(_pixelwise_mul_output_state1);
Jenkinsb3a371b2018-05-23 11:36:53 +0100525 _accum_output2.run();
526 }
527 CLScheduler::get().enqueue(_activation_output);
528
529 CLScheduler::get().enqueue(_activation_output_state);
Jenkins52ba29e2018-08-29 15:32:11 +0000530 CLScheduler::get().enqueue(_pixelwise_mul_output_state2);
Jenkinsb3a371b2018-05-23 11:36:53 +0100531
532 if(_has_projection_weights)
533 {
534 _fully_connected_output_state.run();
535 if(_perform_projection_clipping)
536 {
537 CLScheduler::get().enqueue(_projection_clip);
538 }
539 }
540
541 CLScheduler::get().enqueue(_copy_cell_state);
542 CLScheduler::get().enqueue(_copy_output);
543
544 _concat_scratch_buffer.run();
545
546 _memory_group.release();
547}