XNNPACK Team | b455b12 | 2019-09-27 18:10:33 -0700 | [diff] [blame] | 1 | // Copyright 2019 Google LLC |
| 2 | // |
| 3 | // This source code is licensed under the BSD-style license found in the |
| 4 | // LICENSE file in the root directory of this source tree. |
| 5 | |
| 6 | #include <assert.h> |
| 7 | #include <math.h> |
| 8 | #include <stdbool.h> |
| 9 | #include <stddef.h> |
| 10 | #include <stdint.h> |
| 11 | #include <stdlib.h> |
| 12 | #include <string.h> |
| 13 | |
| 14 | #include <xnnpack.h> |
| 15 | #include <xnnpack/allocator.h> |
| 16 | #include <xnnpack/operator.h> |
| 17 | #include <xnnpack/log.h> |
| 18 | #include <xnnpack/common.h> |
| 19 | #include <xnnpack/math.h> |
| 20 | #include <xnnpack/params.h> |
| 21 | #include <xnnpack/indirection.h> |
| 22 | |
| 23 | |
| 24 | static inline size_t compute_output_dimension( |
| 25 | size_t input_dimension, |
| 26 | size_t input_padding_dimension, |
| 27 | size_t kernel_dimension) |
| 28 | { |
| 29 | return doz(kernel_dimension * input_dimension, input_padding_dimension); |
| 30 | } |
| 31 | |
| 32 | enum xnn_status xnn_create_unpooling2d_nhwc_x32( |
| 33 | uint32_t input_padding_top, |
| 34 | uint32_t input_padding_right, |
| 35 | uint32_t input_padding_bottom, |
| 36 | uint32_t input_padding_left, |
| 37 | uint32_t pooling_height, |
| 38 | uint32_t pooling_width, |
| 39 | size_t channels, |
| 40 | size_t input_pixel_stride, |
| 41 | size_t output_pixel_stride, |
| 42 | uint32_t flags, |
| 43 | xnn_operator_t* unpooling_op_out) |
| 44 | { |
| 45 | xnn_operator_t unpooling_op = NULL; |
| 46 | enum xnn_status status = xnn_status_uninitialized; |
| 47 | |
| 48 | if (!xnn_params.initialized) { |
| 49 | xnn_log_error("failed to create Unpooling operator: XNNPACK is not initialized"); |
| 50 | goto error; |
| 51 | } |
| 52 | |
| 53 | status = xnn_status_invalid_parameter; |
| 54 | |
| 55 | const uint32_t pooling_size = pooling_height * pooling_width; |
| 56 | if (pooling_size == 0) { |
| 57 | xnn_log_error( |
| 58 | "failed to create Unpooling operator with %" PRIu32 "x%" PRIu32 " pooling size: " |
| 59 | "pooling size dimensions must be non-zero", |
| 60 | pooling_width, pooling_height); |
| 61 | goto error; |
| 62 | } |
| 63 | |
| 64 | if (pooling_size == 1) { |
| 65 | xnn_log_error( |
| 66 | "failed to create Unpooling operator with 1 pooling element: 1x1 unpooling is meaningless"); |
| 67 | goto error; |
| 68 | } |
| 69 | |
| 70 | if (channels == 0) { |
| 71 | xnn_log_error( |
| 72 | "failed to create Unpooling operator with %zu channels: number of channels must be non-zero", |
| 73 | channels); |
| 74 | goto error; |
| 75 | } |
| 76 | |
| 77 | if (input_pixel_stride < channels) { |
| 78 | xnn_log_error( |
| 79 | "failed to create Unpooling operator with input pixel stride of %zu: " |
| 80 | "stride must be at least as large as the number of channels (%zu)", |
| 81 | input_pixel_stride, channels); |
| 82 | goto error; |
| 83 | } |
| 84 | |
| 85 | if (output_pixel_stride < channels) { |
| 86 | xnn_log_error( |
| 87 | "failed to create Unpooling operator with output pixel stride of %zu: " |
| 88 | "stride must be at least as large as the number of channels (%zu)", |
| 89 | output_pixel_stride, channels); |
| 90 | goto error; |
| 91 | } |
| 92 | |
| 93 | status = xnn_status_out_of_memory; |
| 94 | |
Marat Dukhan | 04f03be | 2019-11-19 12:36:47 -0800 | [diff] [blame] | 95 | unpooling_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator)); |
XNNPACK Team | b455b12 | 2019-09-27 18:10:33 -0700 | [diff] [blame] | 96 | if (unpooling_op == NULL) { |
| 97 | xnn_log_error("failed to allocate %zu bytes for Unpooling operator descriptor", sizeof(struct xnn_operator)); |
| 98 | goto error; |
| 99 | } |
| 100 | |
| 101 | unpooling_op->padding_top = input_padding_top; |
| 102 | unpooling_op->padding_right = input_padding_right; |
| 103 | unpooling_op->padding_bottom = input_padding_bottom; |
| 104 | unpooling_op->padding_left = input_padding_left; |
| 105 | |
| 106 | unpooling_op->kernel_height = pooling_height; |
| 107 | unpooling_op->kernel_width = pooling_width; |
| 108 | unpooling_op->channels = channels; |
| 109 | unpooling_op->input_pixel_stride = input_pixel_stride; |
| 110 | unpooling_op->output_pixel_stride = output_pixel_stride; |
| 111 | |
Marat Dukhan | efc47b8 | 2019-11-18 09:25:38 -0800 | [diff] [blame] | 112 | unpooling_op->type = xnn_operator_type_unpooling_nhwc_x32; |
XNNPACK Team | b455b12 | 2019-09-27 18:10:33 -0700 | [diff] [blame] | 113 | unpooling_op->ukernel.type = xnn_ukernel_type_unpooling; |
| 114 | |
| 115 | unpooling_op->state = xnn_run_state_invalid; |
| 116 | |
| 117 | *unpooling_op_out = unpooling_op; |
| 118 | return xnn_status_success; |
| 119 | |
| 120 | error: |
| 121 | xnn_delete_operator(unpooling_op); |
| 122 | return status; |
| 123 | } |
| 124 | |
| 125 | enum xnn_status xnn_setup_unpooling2d_nhwc_x32( |
| 126 | xnn_operator_t unpooling_op, |
| 127 | size_t batch_size, |
| 128 | size_t input_height, |
| 129 | size_t input_width, |
| 130 | const void* input, |
| 131 | const uint32_t* index, |
| 132 | void* output, |
| 133 | pthreadpool_t threadpool) |
| 134 | { |
Marat Dukhan | efc47b8 | 2019-11-18 09:25:38 -0800 | [diff] [blame] | 135 | if (unpooling_op->type != xnn_operator_type_unpooling_nhwc_x32) { |
XNNPACK Team | b455b12 | 2019-09-27 18:10:33 -0700 | [diff] [blame] | 136 | xnn_log_error("failed to setup Unpooling (X32) operator: operator type mismatch"); |
| 137 | return xnn_status_invalid_parameter; |
| 138 | } |
| 139 | unpooling_op->state = xnn_run_state_invalid; |
| 140 | |
| 141 | if (!xnn_params.initialized) { |
| 142 | xnn_log_error("failed to setup Unpooling operator: XNNPACK is not initialized"); |
| 143 | return xnn_status_uninitialized; |
| 144 | } |
| 145 | |
| 146 | if (input_width == 0 || input_height == 0) { |
| 147 | xnn_log_error( |
| 148 | "failed to setup Unpooling operator with %zux%zu input: input dimensions must be non-zero", |
| 149 | input_width, input_height); |
| 150 | return xnn_status_invalid_parameter; |
| 151 | } |
| 152 | |
| 153 | if (batch_size == 0) { |
| 154 | unpooling_op->state = xnn_run_state_skip; |
| 155 | return xnn_status_success; |
| 156 | } |
| 157 | |
| 158 | unpooling_op->batch_size = batch_size; |
| 159 | unpooling_op->input_height = input_height; |
| 160 | unpooling_op->input_width = input_width; |
| 161 | unpooling_op->input = input; |
| 162 | |
| 163 | unpooling_op->output_height = compute_output_dimension( |
| 164 | input_height, unpooling_op->padding_top + unpooling_op->padding_bottom, |
| 165 | unpooling_op->kernel_height); |
| 166 | unpooling_op->output_width = compute_output_dimension( |
| 167 | input_width, unpooling_op->padding_left + unpooling_op->padding_right, |
| 168 | unpooling_op->kernel_width); |
| 169 | unpooling_op->output = output; |
| 170 | |
| 171 | size_t valid_batch_size = 0; |
| 172 | if (output == unpooling_op->last_output && |
| 173 | input_height == unpooling_op->last_input_height && |
| 174 | input_width == unpooling_op->last_input_width) |
| 175 | { |
| 176 | valid_batch_size = unpooling_op->valid_batch_size; |
| 177 | if (batch_size <= valid_batch_size) { |
| 178 | unpooling_op->compute.range[0] = batch_size * input_height; |
| 179 | unpooling_op->state = xnn_run_state_ready; |
| 180 | return xnn_status_success; |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | const size_t pooling_height = unpooling_op->kernel_height; |
| 185 | const size_t pooling_width = unpooling_op->kernel_width; |
| 186 | const size_t pooling_size = pooling_height * pooling_width; |
| 187 | |
| 188 | const size_t indirection_buffer_size = sizeof(void*) * (batch_size * input_height * input_width * pooling_size); |
| 189 | |
Marat Dukhan | 04f03be | 2019-11-19 12:36:47 -0800 | [diff] [blame] | 190 | void** indirection_buffer = (void**) xnn_reallocate_memory(unpooling_op->indirection_buffer, indirection_buffer_size); |
XNNPACK Team | b455b12 | 2019-09-27 18:10:33 -0700 | [diff] [blame] | 191 | if (indirection_buffer == NULL) { |
| 192 | xnn_log_error("failed to allocate %zu bytes for indirection buffer", indirection_buffer_size); |
| 193 | return xnn_status_out_of_memory; |
| 194 | } |
| 195 | unpooling_op->indirection_buffer = (const void**) indirection_buffer; |
| 196 | |
| 197 | xnn_indirection_init_unpool2d(unpooling_op, valid_batch_size, 2 /* log2(sizeof(type32)) */); |
| 198 | |
| 199 | const size_t channels = unpooling_op->channels; |
| 200 | const size_t input_pixel_stride_in_bytes = unpooling_op->input_pixel_stride * sizeof(float); |
| 201 | unpooling_op->context.unpooling = (struct unpooling_context) { |
| 202 | .input = input, |
| 203 | .input_height_stride = input_width * input_pixel_stride_in_bytes, |
| 204 | .input_width_stride = input_pixel_stride_in_bytes, |
| 205 | .index = index, |
| 206 | .index_height_stride = input_width * channels * sizeof(uint32_t), |
| 207 | .index_width_stride = channels * sizeof(uint32_t), |
| 208 | .indirect_output = indirection_buffer, |
| 209 | .indirect_output_height_stride = input_width * pooling_size * sizeof(void*), |
| 210 | .indirect_output_width_stride = pooling_size * sizeof(void*), |
| 211 | .pooling_size = pooling_size, |
| 212 | .channels = channels, |
| 213 | .fill_value = 0, |
| 214 | .ukernel = xnn_params.x32.unpool, |
| 215 | }; |
| 216 | unpooling_op->compute.type = xnn_parallelization_type_2d; |
| 217 | unpooling_op->compute.task_2d = (pthreadpool_task_2d_t) xnn_compute_unpooling; |
| 218 | unpooling_op->compute.range[0] = batch_size * input_height; |
| 219 | unpooling_op->compute.range[1] = input_width; |
| 220 | unpooling_op->state = xnn_run_state_ready; |
| 221 | |
| 222 | unpooling_op->last_output = output; |
| 223 | unpooling_op->last_input_height = input_height; |
| 224 | unpooling_op->last_input_width = input_width; |
| 225 | unpooling_op->valid_batch_size = max(valid_batch_size, batch_size); |
| 226 | |
| 227 | return xnn_status_success; |
| 228 | } |