Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 36 | #include "compiler/nir/nir_builder.h" |
| 37 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 38 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 39 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 40 | #include "util/u_dynarray.h" |
| 41 | #include "util/list.h" |
| 42 | #include "main/mtypes.h" |
| 43 | |
| 44 | #include "midgard.h" |
| 45 | #include "midgard_nir.h" |
| 46 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 47 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 48 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 49 | #include "compiler.h" |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 50 | #include "midgard_quirks.h" |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 51 | #include "panfrost-quirks.h" |
| 52 | #include "panfrost/util/pan_lower_framebuffer.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 53 | |
| 54 | #include "disassemble.h" |
| 55 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 56 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 57 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 58 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 59 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 60 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 61 | }; |
| 62 | |
| 63 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 64 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 65 | unsigned SHADER_DB_COUNT = 0; |
| 66 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 67 | int midgard_debug = 0; |
| 68 | |
| 69 | #define DBG(fmt, ...) \ |
| 70 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 71 | fprintf(stderr, "%s:%d: "fmt, \ |
| 72 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 73 | static midgard_block * |
| 74 | create_empty_block(compiler_context *ctx) |
| 75 | { |
| 76 | midgard_block *blk = rzalloc(ctx, midgard_block); |
| 77 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 78 | blk->base.predecessors = _mesa_set_create(blk, |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 79 | _mesa_hash_pointer, |
| 80 | _mesa_key_pointer_equal); |
| 81 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 82 | blk->base.name = ctx->block_source_count++; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 83 | |
| 84 | return blk; |
| 85 | } |
| 86 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 87 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 88 | schedule_barrier(compiler_context *ctx) |
| 89 | { |
| 90 | midgard_block *temp = ctx->after_block; |
| 91 | ctx->after_block = create_empty_block(ctx); |
| 92 | ctx->block_count++; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 93 | list_addtail(&ctx->after_block->base.link, &ctx->blocks); |
| 94 | list_inithead(&ctx->after_block->base.instructions); |
| 95 | pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 96 | ctx->current_block = ctx->after_block; |
| 97 | ctx->after_block = temp; |
| 98 | } |
| 99 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 100 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 101 | * driver seems to do it that way */ |
| 102 | |
| 103 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 104 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 105 | #define M_LOAD_STORE(name, store, T) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 106 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 107 | midgard_instruction i = { \ |
| 108 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 109 | .mask = 0xF, \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 110 | .dest = ~0, \ |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 111 | .src = { ~0, ~0, ~0, ~0 }, \ |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 112 | .swizzle = SWIZZLE_IDENTITY_4, \ |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 113 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 114 | .load_store = { \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 115 | .address = address \ |
| 116 | } \ |
| 117 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 118 | \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 119 | if (store) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 120 | i.src[0] = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 121 | i.src_types[0] = T; \ |
Alyssa Rosenzweig | 9915bb2 | 2020-05-07 10:12:38 -0400 | [diff] [blame] | 122 | i.dest_type = T; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 123 | } else { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 124 | i.dest = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 125 | i.dest_type = T; \ |
| 126 | } \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 127 | return i; \ |
| 128 | } |
| 129 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 130 | #define M_LOAD(name, T) M_LOAD_STORE(name, false, T) |
| 131 | #define M_STORE(name, T) M_LOAD_STORE(name, true, T) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 132 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 133 | M_LOAD(ld_attr_32, nir_type_uint32); |
| 134 | M_LOAD(ld_vary_32, nir_type_uint32); |
| 135 | M_LOAD(ld_ubo_int4, nir_type_uint32); |
| 136 | M_LOAD(ld_int4, nir_type_uint32); |
| 137 | M_STORE(st_int4, nir_type_uint32); |
| 138 | M_LOAD(ld_color_buffer_32u, nir_type_uint32); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 139 | M_LOAD(ld_color_buffer_as_fp16, nir_type_float16); |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 140 | M_LOAD(ld_color_buffer_as_fp32, nir_type_float32); |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 141 | M_STORE(st_vary_32, nir_type_uint32); |
| 142 | M_LOAD(ld_cubemap_coords, nir_type_uint32); |
| 143 | M_LOAD(ld_compute_id, nir_type_uint32); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 144 | |
| 145 | static midgard_instruction |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 146 | v_branch(bool conditional, bool invert) |
| 147 | { |
| 148 | midgard_instruction ins = { |
| 149 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 150 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 151 | .compact_branch = true, |
| 152 | .branch = { |
| 153 | .conditional = conditional, |
| 154 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 155 | }, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 156 | .dest = ~0, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 157 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 158 | }; |
| 159 | |
| 160 | return ins; |
| 161 | } |
| 162 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 163 | static void |
| 164 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 165 | { |
| 166 | ins->has_constants = true; |
| 167 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 171 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | { |
| 173 | return glsl_count_attribute_slots(type, false); |
| 174 | } |
| 175 | |
| 176 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
| 177 | static void |
| 178 | midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu) |
| 179 | { |
| 180 | if (alu->op != nir_op_fdot2) |
| 181 | return; |
| 182 | |
| 183 | b->cursor = nir_before_instr(&alu->instr); |
| 184 | |
| 185 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 186 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 187 | |
| 188 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 189 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 190 | nir_ssa_def *sum = nir_fadd(b, |
| 191 | nir_channel(b, product, 0), |
| 192 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 193 | |
| 194 | /* Replace the fdot2 with this sum */ |
| 195 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
| 196 | } |
| 197 | |
| 198 | static bool |
| 199 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 200 | { |
| 201 | bool progress = false; |
| 202 | |
| 203 | nir_foreach_function(function, shader) { |
| 204 | if (!function->impl) continue; |
| 205 | |
| 206 | nir_builder _b; |
| 207 | nir_builder *b = &_b; |
| 208 | nir_builder_init(b, function->impl); |
| 209 | |
| 210 | nir_foreach_block(block, function->impl) { |
| 211 | nir_foreach_instr_safe(instr, block) { |
| 212 | if (instr->type != nir_instr_type_alu) continue; |
| 213 | |
| 214 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 215 | midgard_nir_lower_fdot2_body(b, alu); |
| 216 | |
| 217 | progress |= true; |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 222 | |
| 223 | } |
| 224 | |
| 225 | return progress; |
| 226 | } |
| 227 | |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 228 | static const nir_variable * |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 229 | search_var(nir_shader *nir, nir_variable_mode mode, unsigned driver_loc) |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 230 | { |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 231 | nir_foreach_variable_with_modes(var, nir, mode) { |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 232 | if (var->data.driver_location == driver_loc) |
| 233 | return var; |
| 234 | } |
| 235 | |
| 236 | return NULL; |
| 237 | } |
| 238 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 239 | /* Midgard can write all of color, depth and stencil in a single writeout |
| 240 | * operation, so we merge depth/stencil stores with color stores. |
| 241 | * If there are no color stores, we add a write to the "depth RT". |
| 242 | */ |
| 243 | static bool |
| 244 | midgard_nir_lower_zs_store(nir_shader *nir) |
| 245 | { |
| 246 | if (nir->info.stage != MESA_SHADER_FRAGMENT) |
| 247 | return false; |
| 248 | |
| 249 | nir_variable *z_var = NULL, *s_var = NULL; |
| 250 | |
Jason Ekstrand | 2956d53 | 2020-07-18 18:24:25 -0500 | [diff] [blame] | 251 | nir_foreach_shader_out_variable(var, nir) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 252 | if (var->data.location == FRAG_RESULT_DEPTH) |
| 253 | z_var = var; |
| 254 | else if (var->data.location == FRAG_RESULT_STENCIL) |
| 255 | s_var = var; |
| 256 | } |
| 257 | |
| 258 | if (!z_var && !s_var) |
| 259 | return false; |
| 260 | |
| 261 | bool progress = false; |
| 262 | |
| 263 | nir_foreach_function(function, nir) { |
| 264 | if (!function->impl) continue; |
| 265 | |
| 266 | nir_intrinsic_instr *z_store = NULL, *s_store = NULL; |
| 267 | |
| 268 | nir_foreach_block(block, function->impl) { |
| 269 | nir_foreach_instr_safe(instr, block) { |
| 270 | if (instr->type != nir_instr_type_intrinsic) |
| 271 | continue; |
| 272 | |
| 273 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 274 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 275 | continue; |
| 276 | |
| 277 | if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) { |
| 278 | assert(!z_store); |
| 279 | z_store = intr; |
| 280 | } |
| 281 | |
| 282 | if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) { |
| 283 | assert(!s_store); |
| 284 | s_store = intr; |
| 285 | } |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | if (!z_store && !s_store) continue; |
| 290 | |
| 291 | bool replaced = false; |
| 292 | |
| 293 | nir_foreach_block(block, function->impl) { |
| 294 | nir_foreach_instr_safe(instr, block) { |
| 295 | if (instr->type != nir_instr_type_intrinsic) |
| 296 | continue; |
| 297 | |
| 298 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 299 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 300 | continue; |
| 301 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 302 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 303 | assert(var); |
| 304 | |
| 305 | if (var->data.location != FRAG_RESULT_COLOR && |
| 306 | var->data.location < FRAG_RESULT_DATA0) |
| 307 | continue; |
| 308 | |
Icecream95 | 334dab0 | 2020-07-10 23:28:21 +1200 | [diff] [blame] | 309 | if (var->data.index) |
| 310 | continue; |
| 311 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 312 | assert(nir_src_is_const(intr->src[1]) && "no indirect outputs"); |
| 313 | |
| 314 | nir_builder b; |
| 315 | nir_builder_init(&b, function->impl); |
| 316 | |
| 317 | assert(!z_store || z_store->instr.block == instr->block); |
| 318 | assert(!s_store || s_store->instr.block == instr->block); |
| 319 | b.cursor = nir_after_block_before_jump(instr->block); |
| 320 | |
| 321 | nir_intrinsic_instr *combined_store; |
| 322 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 323 | |
| 324 | combined_store->num_components = intr->src[0].ssa->num_components; |
| 325 | |
| 326 | nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr)); |
| 327 | |
| 328 | unsigned writeout = PAN_WRITEOUT_C; |
| 329 | if (z_store) |
| 330 | writeout |= PAN_WRITEOUT_Z; |
| 331 | if (s_store) |
| 332 | writeout |= PAN_WRITEOUT_S; |
| 333 | |
| 334 | nir_intrinsic_set_component(combined_store, writeout); |
| 335 | |
| 336 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 337 | |
| 338 | struct nir_ssa_def *src[4] = { |
| 339 | intr->src[0].ssa, |
| 340 | intr->src[1].ssa, |
| 341 | z_store ? z_store->src[0].ssa : zero, |
| 342 | s_store ? s_store->src[0].ssa : zero, |
| 343 | }; |
| 344 | |
| 345 | for (int i = 0; i < 4; ++i) |
| 346 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 347 | |
| 348 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 349 | |
| 350 | nir_instr_remove(instr); |
| 351 | |
| 352 | replaced = true; |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | /* Insert a store to the depth RT (0xff) if needed */ |
| 357 | if (!replaced) { |
| 358 | nir_builder b; |
| 359 | nir_builder_init(&b, function->impl); |
| 360 | |
| 361 | nir_block *block = NULL; |
| 362 | if (z_store && s_store) |
| 363 | assert(z_store->instr.block == s_store->instr.block); |
| 364 | |
| 365 | if (z_store) |
| 366 | block = z_store->instr.block; |
| 367 | else |
| 368 | block = s_store->instr.block; |
| 369 | |
| 370 | b.cursor = nir_after_block_before_jump(block); |
| 371 | |
| 372 | nir_intrinsic_instr *combined_store; |
| 373 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 374 | |
| 375 | combined_store->num_components = 4; |
| 376 | |
Icecream95 | 18059f4 | 2020-07-08 16:00:51 +1200 | [diff] [blame] | 377 | unsigned base; |
| 378 | if (z_store) |
| 379 | base = nir_intrinsic_base(z_store); |
| 380 | else |
| 381 | base = nir_intrinsic_base(s_store); |
| 382 | nir_intrinsic_set_base(combined_store, base); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 383 | |
| 384 | unsigned writeout = 0; |
| 385 | if (z_store) |
| 386 | writeout |= PAN_WRITEOUT_Z; |
| 387 | if (s_store) |
| 388 | writeout |= PAN_WRITEOUT_S; |
| 389 | |
| 390 | nir_intrinsic_set_component(combined_store, writeout); |
| 391 | |
| 392 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 393 | |
| 394 | struct nir_ssa_def *src[4] = { |
| 395 | nir_imm_vec4(&b, 0, 0, 0, 0), |
| 396 | zero, |
| 397 | z_store ? z_store->src[0].ssa : zero, |
| 398 | s_store ? s_store->src[0].ssa : zero, |
| 399 | }; |
| 400 | |
| 401 | for (int i = 0; i < 4; ++i) |
| 402 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 403 | |
| 404 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 405 | } |
| 406 | |
| 407 | if (z_store) |
| 408 | nir_instr_remove(&z_store->instr); |
| 409 | |
| 410 | if (s_store) |
| 411 | nir_instr_remove(&s_store->instr); |
| 412 | |
| 413 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 414 | progress = true; |
| 415 | } |
| 416 | |
| 417 | return progress; |
| 418 | } |
| 419 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 420 | /* Real writeout stores, which break execution, need to be moved to after |
| 421 | * dual-source stores, which are just standard register writes. */ |
| 422 | static bool |
| 423 | midgard_nir_reorder_writeout(nir_shader *nir) |
| 424 | { |
| 425 | bool progress = false; |
| 426 | |
| 427 | nir_foreach_function(function, nir) { |
| 428 | if (!function->impl) continue; |
| 429 | |
| 430 | nir_foreach_block(block, function->impl) { |
| 431 | nir_instr *last_writeout = NULL; |
| 432 | |
| 433 | nir_foreach_instr_reverse_safe(instr, block) { |
| 434 | if (instr->type != nir_instr_type_intrinsic) |
| 435 | continue; |
| 436 | |
| 437 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 438 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 439 | continue; |
| 440 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 441 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 442 | |
| 443 | if (var->data.index) { |
| 444 | if (!last_writeout) |
| 445 | last_writeout = instr; |
| 446 | continue; |
| 447 | } |
| 448 | |
| 449 | if (!last_writeout) |
| 450 | continue; |
| 451 | |
| 452 | /* This is a real store, so move it to after dual-source stores */ |
| 453 | exec_node_remove(&instr->node); |
| 454 | exec_node_insert_after(&last_writeout->node, &instr->node); |
| 455 | |
| 456 | progress = true; |
| 457 | } |
| 458 | } |
| 459 | } |
| 460 | |
| 461 | return progress; |
| 462 | } |
| 463 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 464 | static bool |
| 465 | mdg_is_64(const nir_instr *instr, const void *_unused) |
| 466 | { |
| 467 | const nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 468 | |
| 469 | if (nir_dest_bit_size(alu->dest.dest) == 64) |
| 470 | return true; |
| 471 | |
| 472 | switch (alu->op) { |
| 473 | case nir_op_umul_high: |
| 474 | case nir_op_imul_high: |
| 475 | return true; |
| 476 | default: |
| 477 | return false; |
| 478 | } |
| 479 | } |
| 480 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 481 | /* Flushes undefined values to zero */ |
| 482 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 483 | static void |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 484 | optimise_nir(nir_shader *nir, unsigned quirks, bool is_blend) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 485 | { |
| 486 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 487 | unsigned lower_flrp = |
| 488 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 489 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 490 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 491 | |
| 492 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
Rhys Perry | 8b98d09 | 2019-02-05 15:56:24 +0000 | [diff] [blame] | 493 | NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 494 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 495 | nir_lower_tex_options lower_tex_options = { |
| 496 | .lower_txs_lod = true, |
Alyssa Rosenzweig | 4c43b35 | 2019-11-21 13:40:00 -0500 | [diff] [blame] | 497 | .lower_txp = ~0, |
| 498 | .lower_tex_without_implicit_lod = |
| 499 | (quirks & MIDGARD_EXPLICIT_LOD), |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 500 | .lower_tg4_broadcom_swizzle = true, |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 501 | |
| 502 | /* TODO: we have native gradient.. */ |
| 503 | .lower_txd = true, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 504 | }; |
| 505 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 506 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 507 | |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 508 | /* Must lower fdot2 after tex is lowered */ |
| 509 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
| 510 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 511 | /* T720 is broken. */ |
| 512 | |
| 513 | if (quirks & MIDGARD_BROKEN_LOD) |
| 514 | NIR_PASS_V(nir, midgard_nir_lod_errata); |
| 515 | |
Alyssa Rosenzweig | c495c6c | 2020-05-12 19:07:48 -0400 | [diff] [blame] | 516 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_early); |
| 517 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 518 | do { |
| 519 | progress = false; |
| 520 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 521 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 522 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 523 | |
| 524 | NIR_PASS(progress, nir, nir_copy_prop); |
Boris Brezillon | 440b0d6 | 2020-01-06 14:31:38 +0100 | [diff] [blame] | 525 | NIR_PASS(progress, nir, nir_opt_remove_phis); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 526 | NIR_PASS(progress, nir, nir_opt_dce); |
| 527 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 528 | NIR_PASS(progress, nir, nir_opt_cse); |
| 529 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 530 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 531 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 532 | |
| 533 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 534 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 535 | NIR_PASS(lower_flrp_progress, |
| 536 | nir, |
| 537 | nir_lower_flrp, |
| 538 | lower_flrp, |
| 539 | false /* always_precise */, |
| 540 | nir->options->lower_ffma); |
| 541 | if (lower_flrp_progress) { |
| 542 | NIR_PASS(progress, nir, |
| 543 | nir_opt_constant_folding); |
| 544 | progress = true; |
| 545 | } |
| 546 | |
| 547 | /* Nothing should rematerialize any flrps, so we only |
| 548 | * need to do this lowering once. |
| 549 | */ |
| 550 | lower_flrp = 0; |
| 551 | } |
| 552 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 553 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 554 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 555 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 556 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 557 | nir_var_shader_in | |
| 558 | nir_var_shader_out | |
| 559 | nir_var_function_temp); |
| 560 | |
Eric Anholt | f25e169 | 2020-08-27 12:49:13 -0700 | [diff] [blame^] | 561 | NIR_PASS(progress, nir, nir_opt_vectorize, NULL, NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 562 | } while (progress); |
| 563 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 564 | NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_is_64, NULL); |
| 565 | |
Alyssa Rosenzweig | d838cb9 | 2020-06-16 13:07:02 -0400 | [diff] [blame] | 566 | /* Run after opts so it can hit more */ |
| 567 | if (!is_blend) |
| 568 | NIR_PASS(progress, nir, nir_fuse_io_16); |
| 569 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 570 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 571 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 572 | |
| 573 | do { |
| 574 | progress = false; |
| 575 | |
| 576 | NIR_PASS(progress, nir, nir_opt_dce); |
| 577 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 578 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 579 | NIR_PASS(progress, nir, nir_copy_prop); |
| 580 | } while (progress); |
| 581 | |
| 582 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 211dee4 | 2020-04-29 20:27:16 -0400 | [diff] [blame] | 583 | NIR_PASS(progress, nir, nir_opt_algebraic_distribute_src_mods); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 584 | |
| 585 | /* We implement booleans as 32-bit 0/~0 */ |
| 586 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 587 | |
| 588 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 589 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 590 | NIR_PASS(progress, nir, midgard_nir_cancel_inot); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 591 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 592 | NIR_PASS(progress, nir, nir_copy_prop); |
| 593 | NIR_PASS(progress, nir, nir_opt_dce); |
| 594 | |
| 595 | /* Take us out of SSA */ |
| 596 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 597 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 598 | |
| 599 | /* We are a vector architecture; write combine where possible */ |
| 600 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 601 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 602 | |
| 603 | NIR_PASS(progress, nir, nir_opt_dce); |
| 604 | } |
| 605 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 606 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 607 | |
| 608 | static void |
| 609 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 610 | { |
| 611 | nir_ssa_def def = instr->def; |
| 612 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 613 | midgard_constants *consts = rzalloc(NULL, midgard_constants); |
| 614 | |
| 615 | assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8); |
| 616 | |
| 617 | #define RAW_CONST_COPY(bits) \ |
| 618 | nir_const_value_to_array(consts->u##bits, instr->value, \ |
| 619 | instr->def.num_components, u##bits) |
| 620 | |
| 621 | switch (instr->def.bit_size) { |
| 622 | case 64: |
| 623 | RAW_CONST_COPY(64); |
| 624 | break; |
| 625 | case 32: |
| 626 | RAW_CONST_COPY(32); |
| 627 | break; |
| 628 | case 16: |
| 629 | RAW_CONST_COPY(16); |
| 630 | break; |
| 631 | case 8: |
| 632 | RAW_CONST_COPY(8); |
| 633 | break; |
| 634 | default: |
| 635 | unreachable("Invalid bit_size for load_const instruction\n"); |
| 636 | } |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 637 | |
| 638 | /* Shifted for SSA, +1 for off-by-one */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 639 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 640 | } |
| 641 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 642 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 643 | * explicitly emit a move with the constant source */ |
| 644 | |
| 645 | static void |
| 646 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 647 | { |
| 648 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 649 | |
| 650 | if (constant_value) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 651 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), to); |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 652 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 653 | emit_mir_instruction(ctx, ins); |
| 654 | } |
| 655 | } |
| 656 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 657 | static bool |
| 658 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 659 | { |
| 660 | unsigned comp = src->swizzle[0]; |
| 661 | |
| 662 | for (unsigned c = 1; c < nr_components; ++c) { |
| 663 | if (src->swizzle[c] != comp) |
| 664 | return true; |
| 665 | } |
| 666 | |
| 667 | return false; |
| 668 | } |
| 669 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 670 | #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \ |
| 671 | case nir_intrinsic_##nir: \ |
| 672 | emit_atomic(ctx, instr, is_shared, midgard_op_##op); \ |
| 673 | break; |
| 674 | |
| 675 | #define ATOMIC_CASE(ctx, instr, nir, op) \ |
| 676 | ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \ |
| 677 | ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false); |
| 678 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 679 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 680 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 681 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 682 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 683 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 684 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 685 | #define ALU_CASE_RTZ(nir, _op) \ |
| 686 | case nir_op_##nir: \ |
| 687 | op = midgard_alu_op_##_op; \ |
| 688 | roundmode = MIDGARD_RTZ; \ |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 689 | break; |
| 690 | |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 691 | #define ALU_CHECK_CMP(sext) \ |
| 692 | assert(src_bitsize == 16 || src_bitsize == 32); \ |
| 693 | assert(dst_bitsize == 16 || dst_bitsize == 32); \ |
| 694 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 695 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 696 | case nir_op_##nir: \ |
| 697 | op = midgard_alu_op_##_op; \ |
| 698 | broadcast_swizzle = count; \ |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 699 | ALU_CHECK_CMP(true); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 700 | break; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 701 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 702 | #define ALU_CASE_CMP(nir, _op, sext) \ |
| 703 | case nir_op_##nir: \ |
| 704 | op = midgard_alu_op_##_op; \ |
| 705 | ALU_CHECK_CMP(sext); \ |
| 706 | break; |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 707 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 708 | /* Compare mir_lower_invert */ |
| 709 | static bool |
| 710 | nir_accepts_inot(nir_op op, unsigned src) |
| 711 | { |
| 712 | switch (op) { |
| 713 | case nir_op_ior: |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 714 | case nir_op_iand: /* TODO: b2f16 */ |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 715 | case nir_op_ixor: |
| 716 | return true; |
| 717 | case nir_op_b32csel: |
| 718 | /* Only the condition */ |
| 719 | return (src == 0); |
| 720 | default: |
| 721 | return false; |
| 722 | } |
| 723 | } |
| 724 | |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 725 | static bool |
| 726 | mir_accept_dest_mod(compiler_context *ctx, nir_dest **dest, nir_op op) |
| 727 | { |
| 728 | if (pan_has_dest_mod(dest, op)) { |
| 729 | assert((*dest)->is_ssa); |
| 730 | BITSET_SET(ctx->already_emitted, (*dest)->ssa.index); |
| 731 | return true; |
| 732 | } |
| 733 | |
| 734 | return false; |
| 735 | } |
| 736 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 737 | /* Look for floating point mods. We have the mods fsat, fsat_signed, |
| 738 | * and fpos. We also have the relations (note 3 * 2 = 6 cases): |
| 739 | * |
| 740 | * fsat_signed(fpos(x)) = fsat(x) |
| 741 | * fsat_signed(fsat(x)) = fsat(x) |
| 742 | * fpos(fsat_signed(x)) = fsat(x) |
| 743 | * fpos(fsat(x)) = fsat(x) |
| 744 | * fsat(fsat_signed(x)) = fsat(x) |
| 745 | * fsat(fpos(x)) = fsat(x) |
| 746 | * |
| 747 | * So by cases any composition of output modifiers is equivalent to |
| 748 | * fsat alone. |
| 749 | */ |
| 750 | static unsigned |
| 751 | mir_determine_float_outmod(compiler_context *ctx, nir_dest **dest, unsigned prior_outmod) |
| 752 | { |
| 753 | bool fpos = mir_accept_dest_mod(ctx, dest, nir_op_fclamp_pos); |
| 754 | bool fsat = mir_accept_dest_mod(ctx, dest, nir_op_fsat); |
| 755 | bool ssat = mir_accept_dest_mod(ctx, dest, nir_op_fsat_signed); |
| 756 | bool prior = (prior_outmod != midgard_outmod_none); |
| 757 | int count = (int) prior + (int) fpos + (int) ssat + (int) fsat; |
| 758 | |
| 759 | return ((count > 1) || fsat) ? midgard_outmod_sat : |
| 760 | fpos ? midgard_outmod_pos : |
| 761 | ssat ? midgard_outmod_sat_signed : |
| 762 | prior_outmod; |
| 763 | } |
| 764 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 765 | static void |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 766 | mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i, unsigned to, bool *abs, bool *neg, bool *not, enum midgard_roundmode *roundmode, bool is_int, unsigned bcast_count) |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 767 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 768 | nir_alu_src src = instr->src[i]; |
Alyssa Rosenzweig | b124f53 | 2020-04-29 18:10:43 -0400 | [diff] [blame] | 769 | |
| 770 | if (!is_int) { |
| 771 | if (pan_has_source_mod(&src, nir_op_fneg)) |
| 772 | *neg = !(*neg); |
| 773 | |
| 774 | if (pan_has_source_mod(&src, nir_op_fabs)) |
| 775 | *abs = true; |
| 776 | } |
| 777 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 778 | if (nir_accepts_inot(instr->op, i) && pan_has_source_mod(&src, nir_op_inot)) |
| 779 | *not = true; |
| 780 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 781 | if (roundmode) { |
| 782 | if (pan_has_source_mod(&src, nir_op_fround_even)) |
| 783 | *roundmode = MIDGARD_RTE; |
| 784 | |
| 785 | if (pan_has_source_mod(&src, nir_op_ftrunc)) |
| 786 | *roundmode = MIDGARD_RTZ; |
| 787 | |
| 788 | if (pan_has_source_mod(&src, nir_op_ffloor)) |
| 789 | *roundmode = MIDGARD_RTN; |
| 790 | |
| 791 | if (pan_has_source_mod(&src, nir_op_fceil)) |
| 792 | *roundmode = MIDGARD_RTP; |
| 793 | } |
| 794 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 795 | unsigned bits = nir_src_bit_size(src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 796 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 797 | ins->src[to] = nir_src_index(NULL, &src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 798 | ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 799 | |
| 800 | for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 801 | ins->swizzle[to][c] = src.swizzle[ |
| 802 | (!bcast_count || c < bcast_count) ? c : |
| 803 | (bcast_count - 1)]; |
| 804 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 805 | } |
| 806 | |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 807 | /* Midgard features both fcsel and icsel, depending on whether you want int or |
| 808 | * float modifiers. NIR's csel is typeless, so we want a heuristic to guess if |
| 809 | * we should emit an int or float csel depending on what modifiers could be |
| 810 | * placed. In the absense of modifiers, this is probably arbitrary. */ |
| 811 | |
| 812 | static bool |
| 813 | mir_is_bcsel_float(nir_alu_instr *instr) |
| 814 | { |
| 815 | nir_op intmods[] = { |
| 816 | nir_op_i2i8, nir_op_i2i16, |
| 817 | nir_op_i2i32, nir_op_i2i64 |
| 818 | }; |
| 819 | |
| 820 | nir_op floatmods[] = { |
| 821 | nir_op_fabs, nir_op_fneg, |
| 822 | nir_op_f2f16, nir_op_f2f32, |
| 823 | nir_op_f2f64 |
| 824 | }; |
| 825 | |
| 826 | nir_op floatdestmods[] = { |
| 827 | nir_op_fsat, nir_op_fsat_signed, nir_op_fclamp_pos, |
| 828 | nir_op_f2f16, nir_op_f2f32 |
| 829 | }; |
| 830 | |
| 831 | signed score = 0; |
| 832 | |
| 833 | for (unsigned i = 1; i < 3; ++i) { |
| 834 | nir_alu_src s = instr->src[i]; |
| 835 | for (unsigned q = 0; q < ARRAY_SIZE(intmods); ++q) { |
| 836 | if (pan_has_source_mod(&s, intmods[q])) |
| 837 | score--; |
| 838 | } |
| 839 | } |
| 840 | |
| 841 | for (unsigned i = 1; i < 3; ++i) { |
| 842 | nir_alu_src s = instr->src[i]; |
| 843 | for (unsigned q = 0; q < ARRAY_SIZE(floatmods); ++q) { |
| 844 | if (pan_has_source_mod(&s, floatmods[q])) |
| 845 | score++; |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | for (unsigned q = 0; q < ARRAY_SIZE(floatdestmods); ++q) { |
| 850 | nir_dest *dest = &instr->dest.dest; |
| 851 | if (pan_has_dest_mod(&dest, floatdestmods[q])) |
| 852 | score++; |
| 853 | } |
| 854 | |
| 855 | return (score > 0); |
| 856 | } |
| 857 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 858 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 859 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 860 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 861 | nir_dest *dest = &instr->dest.dest; |
| 862 | |
| 863 | if (dest->is_ssa && BITSET_TEST(ctx->already_emitted, dest->ssa.index)) |
| 864 | return; |
| 865 | |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 866 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 867 | * is handled elsewhere */ |
| 868 | |
| 869 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 870 | midgard_emit_derivatives(ctx, instr); |
| 871 | return; |
| 872 | } |
| 873 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 874 | bool is_ssa = dest->is_ssa; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 875 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 876 | unsigned nr_components = nir_dest_num_components(*dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 877 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | 04f76ad | 2020-04-27 18:58:21 -0400 | [diff] [blame] | 878 | unsigned op = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 879 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 880 | /* Number of components valid to check for the instruction (the rest |
| 881 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 882 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 883 | * in Midgard */ |
| 884 | |
| 885 | unsigned broadcast_swizzle = 0; |
| 886 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 887 | /* Should we swap arguments? */ |
| 888 | bool flip_src12 = false; |
| 889 | |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 890 | ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 891 | ASSERTED unsigned dst_bitsize = nir_dest_bit_size(*dest); |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 892 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 893 | enum midgard_roundmode roundmode = MIDGARD_RTE; |
| 894 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 895 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 896 | ALU_CASE(fadd, fadd); |
| 897 | ALU_CASE(fmul, fmul); |
| 898 | ALU_CASE(fmin, fmin); |
| 899 | ALU_CASE(fmax, fmax); |
| 900 | ALU_CASE(imin, imin); |
| 901 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 902 | ALU_CASE(umin, umin); |
| 903 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 904 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 905 | ALU_CASE(fround_even, froundeven); |
| 906 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 907 | ALU_CASE(fceil, fceil); |
| 908 | ALU_CASE(fdot3, fdot3); |
| 909 | ALU_CASE(fdot4, fdot4); |
| 910 | ALU_CASE(iadd, iadd); |
| 911 | ALU_CASE(isub, isub); |
| 912 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 913 | ALU_CASE(imul_high, imul); |
| 914 | ALU_CASE(umul_high, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 915 | |
| 916 | /* Zero shoved as second-arg */ |
| 917 | ALU_CASE(iabs, iabsdiff); |
| 918 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 919 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 920 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 921 | ALU_CASE_CMP(feq32, feq, false); |
Karol Herbst | e5899c1 | 2020-08-18 19:51:57 +0200 | [diff] [blame] | 922 | ALU_CASE_CMP(fneu32, fne, false); |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 923 | ALU_CASE_CMP(flt32, flt, false); |
| 924 | ALU_CASE_CMP(ieq32, ieq, true); |
| 925 | ALU_CASE_CMP(ine32, ine, true); |
| 926 | ALU_CASE_CMP(ilt32, ilt, true); |
| 927 | ALU_CASE_CMP(ult32, ult, false); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 928 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 929 | /* We don't have a native b2f32 instruction. Instead, like many |
| 930 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 931 | * correspondingly AND |
| 932 | * by 1.0 to do the type conversion. For the moment, prime us |
| 933 | * to emit: |
| 934 | * |
| 935 | * iand [whatever], #0 |
| 936 | * |
| 937 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 938 | */ |
| 939 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 940 | ALU_CASE_CMP(b2f32, iand, true); |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 941 | ALU_CASE_CMP(b2f16, iand, true); |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 942 | ALU_CASE_CMP(b2i32, iand, true); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 943 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 944 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 945 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 946 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 947 | ALU_CASE_CMP(f2b32, fne, false); |
| 948 | ALU_CASE_CMP(i2b32, ine, true); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 949 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 950 | ALU_CASE(frcp, frcp); |
| 951 | ALU_CASE(frsq, frsqrt); |
| 952 | ALU_CASE(fsqrt, fsqrt); |
| 953 | ALU_CASE(fexp2, fexp2); |
| 954 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 955 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 956 | ALU_CASE_RTZ(f2i64, f2i_rte); |
| 957 | ALU_CASE_RTZ(f2u64, f2u_rte); |
| 958 | ALU_CASE_RTZ(i2f64, i2f_rte); |
| 959 | ALU_CASE_RTZ(u2f64, u2f_rte); |
Boris Brezillon | fcceeaf | 2020-01-20 22:05:14 +0100 | [diff] [blame] | 960 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 961 | ALU_CASE_RTZ(f2i32, f2i_rte); |
| 962 | ALU_CASE_RTZ(f2u32, f2u_rte); |
| 963 | ALU_CASE_RTZ(i2f32, i2f_rte); |
| 964 | ALU_CASE_RTZ(u2f32, u2f_rte); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 965 | |
Alyssa Rosenzweig | 0ae0141 | 2020-05-25 14:46:40 -0400 | [diff] [blame] | 966 | ALU_CASE_RTZ(f2i8, f2i_rte); |
| 967 | ALU_CASE_RTZ(f2u8, f2u_rte); |
| 968 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 969 | ALU_CASE_RTZ(f2i16, f2i_rte); |
| 970 | ALU_CASE_RTZ(f2u16, f2u_rte); |
| 971 | ALU_CASE_RTZ(i2f16, i2f_rte); |
| 972 | ALU_CASE_RTZ(u2f16, u2f_rte); |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 973 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 974 | ALU_CASE(fsin, fsin); |
| 975 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 976 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 977 | /* We'll get 0 in the second arg, so: |
| 978 | * ~a = ~(a | 0) = nor(a, 0) */ |
| 979 | ALU_CASE(inot, inor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 980 | ALU_CASE(iand, iand); |
| 981 | ALU_CASE(ior, ior); |
| 982 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 983 | ALU_CASE(ishl, ishl); |
| 984 | ALU_CASE(ishr, iasr); |
| 985 | ALU_CASE(ushr, ilsr); |
| 986 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 987 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 988 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 989 | ALU_CASE_CMP(b32all_fequal4, fball_eq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 990 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 991 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 992 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 993 | ALU_CASE_CMP(b32any_fnequal4, fbany_neq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 994 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 995 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 996 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 997 | ALU_CASE_CMP(b32all_iequal4, iball_eq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 998 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 999 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 1000 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 1001 | ALU_CASE_CMP(b32any_inequal4, ibany_neq, true); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1002 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1003 | /* Source mods will be shoved in later */ |
| 1004 | ALU_CASE(fabs, fmov); |
| 1005 | ALU_CASE(fneg, fmov); |
| 1006 | ALU_CASE(fsat, fmov); |
Alyssa Rosenzweig | 24e2e24 | 2020-05-04 16:12:41 -0400 | [diff] [blame] | 1007 | ALU_CASE(fsat_signed, fmov); |
| 1008 | ALU_CASE(fclamp_pos, fmov); |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1009 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1010 | /* For size conversion, we use a move. Ideally though we would squash |
| 1011 | * these ops together; maybe that has to happen after in NIR as part of |
| 1012 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1013 | * only / exactly one size. If stepping down, we use a dest override to |
| 1014 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 1015 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1016 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1017 | case nir_op_i2i8: |
| 1018 | case nir_op_i2i16: |
| 1019 | case nir_op_i2i32: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1020 | case nir_op_i2i64: |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1021 | case nir_op_u2u8: |
| 1022 | case nir_op_u2u16: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1023 | case nir_op_u2u32: |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1024 | case nir_op_u2u64: |
| 1025 | case nir_op_f2f16: |
Boris Brezillon | e1f9e8d | 2020-01-20 16:05:31 +0100 | [diff] [blame] | 1026 | case nir_op_f2f32: |
| 1027 | case nir_op_f2f64: { |
| 1028 | if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 || |
| 1029 | instr->op == nir_op_f2f64) |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1030 | op = midgard_alu_op_fmov; |
| 1031 | else |
| 1032 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1033 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1034 | break; |
| 1035 | } |
| 1036 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1037 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1038 | * arguments */ |
| 1039 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1040 | case nir_op_fge: |
| 1041 | case nir_op_fge32: |
| 1042 | case nir_op_ige32: |
| 1043 | case nir_op_uge32: { |
| 1044 | op = |
| 1045 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 1046 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 1047 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 1048 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 1049 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1050 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1051 | flip_src12 = true; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 1052 | ALU_CHECK_CMP(false); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1053 | break; |
| 1054 | } |
| 1055 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 1056 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 1057 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 1058 | bool is_float = mir_is_bcsel_float(instr); |
| 1059 | op = is_float ? |
| 1060 | (mixed ? midgard_alu_op_fcsel_v : midgard_alu_op_fcsel) : |
| 1061 | (mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1062 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1063 | break; |
| 1064 | } |
| 1065 | |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1066 | case nir_op_unpack_32_2x16: |
| 1067 | case nir_op_unpack_32_4x8: |
| 1068 | case nir_op_pack_32_2x16: |
| 1069 | case nir_op_pack_32_4x8: { |
| 1070 | op = midgard_alu_op_imov; |
| 1071 | break; |
| 1072 | } |
| 1073 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1074 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1075 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1076 | assert(0); |
| 1077 | return; |
| 1078 | } |
| 1079 | |
Alyssa Rosenzweig | 72c1e3a | 2020-05-21 12:31:40 -0400 | [diff] [blame] | 1080 | /* Promote imov to fmov if it might help inline a constant */ |
| 1081 | if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src) |
| 1082 | && nir_src_bit_size(instr->src[0].src) == 32 |
| 1083 | && nir_is_same_comp_swizzle(instr->src[0].swizzle, |
| 1084 | nir_src_num_components(instr->src[0].src))) { |
| 1085 | op = midgard_alu_op_fmov; |
| 1086 | } |
| 1087 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 1088 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1089 | |
| 1090 | unsigned outmod = 0; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1091 | bool is_int = midgard_is_integer_op(op); |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 1092 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 1093 | if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) { |
| 1094 | outmod = midgard_outmod_int_high; |
| 1095 | } else if (midgard_is_integer_out_op(op)) { |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1096 | outmod = midgard_outmod_int_wrap; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1097 | } else if (instr->op == nir_op_fsat) { |
| 1098 | outmod = midgard_outmod_sat; |
| 1099 | } else if (instr->op == nir_op_fsat_signed) { |
| 1100 | outmod = midgard_outmod_sat_signed; |
| 1101 | } else if (instr->op == nir_op_fclamp_pos) { |
| 1102 | outmod = midgard_outmod_pos; |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1103 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1104 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1105 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 1106 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1107 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1108 | |
Italo Nicola | 2096903 | 2020-07-13 16:19:08 +0000 | [diff] [blame] | 1109 | if (!midgard_is_integer_out_op(op)) { |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1110 | outmod = mir_determine_float_outmod(ctx, &dest, outmod); |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 1111 | } |
| 1112 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1113 | midgard_instruction ins = { |
| 1114 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1115 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ecf9466 | 2020-04-27 18:57:34 -0400 | [diff] [blame] | 1116 | .dest_type = nir_op_infos[instr->op].output_type |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1117 | | nir_dest_bit_size(*dest), |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 1118 | .roundmode = roundmode, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1119 | }; |
| 1120 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1121 | enum midgard_roundmode *roundptr = (opcode_props & MIDGARD_ROUNDS) ? |
| 1122 | &ins.roundmode : NULL; |
| 1123 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1124 | for (unsigned i = nr_inputs; i < ARRAY_SIZE(ins.src); ++i) |
| 1125 | ins.src[i] = ~0; |
| 1126 | |
| 1127 | if (quirk_flipped_r24) { |
| 1128 | ins.src[0] = ~0; |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1129 | mir_copy_src(&ins, instr, 0, 1, &ins.src_abs[1], &ins.src_neg[1], &ins.src_invert[1], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1130 | } else { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1131 | for (unsigned i = 0; i < nr_inputs; ++i) { |
| 1132 | unsigned to = i; |
| 1133 | |
| 1134 | if (instr->op == nir_op_b32csel) { |
| 1135 | /* The condition is the first argument; move |
| 1136 | * the other arguments up one to be a binary |
| 1137 | * instruction for Midgard with the condition |
| 1138 | * last */ |
| 1139 | |
| 1140 | if (i == 0) |
| 1141 | to = 2; |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1142 | else if (flip_src12) |
| 1143 | to = 2 - i; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1144 | else |
| 1145 | to = i - 1; |
| 1146 | } else if (flip_src12) { |
| 1147 | to = 1 - to; |
| 1148 | } |
| 1149 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1150 | mir_copy_src(&ins, instr, i, to, &ins.src_abs[to], &ins.src_neg[to], &ins.src_invert[to], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1151 | |
| 1152 | /* (!c) ? a : b = c ? b : a */ |
| 1153 | if (instr->op == nir_op_b32csel && ins.src_invert[2]) { |
| 1154 | ins.src_invert[2] = false; |
| 1155 | flip_src12 ^= true; |
| 1156 | } |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1157 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1158 | } |
| 1159 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1160 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1161 | /* Lowered to move */ |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1162 | if (instr->op == nir_op_fneg) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1163 | ins.src_neg[1] ^= true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1164 | |
| 1165 | if (instr->op == nir_op_fabs) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1166 | ins.src_abs[1] = true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1167 | } |
| 1168 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1169 | ins.mask = mask_of(nr_components); |
| 1170 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1171 | /* Apply writemask if non-SSA, keeping in mind that we can't write to |
| 1172 | * components that don't exist. Note modifier => SSA => !reg => no |
| 1173 | * writemask, so we don't have to worry about writemasks here.*/ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1174 | |
| 1175 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1176 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1177 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1178 | ins.op = op; |
Italo Nicola | 5011373 | 2020-07-15 18:43:18 +0000 | [diff] [blame] | 1179 | ins.outmod = outmod; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1180 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1181 | /* Late fixup for emulated instructions */ |
| 1182 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1183 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1184 | /* Presently, our second argument is an inline #0 constant. |
| 1185 | * Switch over to an embedded 1.0 constant (that can't fit |
| 1186 | * inline, since we're 32-bit, not 16-bit like the inline |
| 1187 | * constants) */ |
| 1188 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1189 | ins.has_inline_constant = false; |
| 1190 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1191 | ins.src_types[1] = nir_type_float32; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1192 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1193 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1194 | if (instr->op == nir_op_b2f32) |
| 1195 | ins.constants.f32[0] = 1.0f; |
| 1196 | else |
| 1197 | ins.constants.i32[0] = 1; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1198 | |
| 1199 | for (unsigned c = 0; c < 16; ++c) |
| 1200 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 1201 | } else if (instr->op == nir_op_b2f16) { |
| 1202 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
| 1203 | ins.src_types[1] = nir_type_float16; |
| 1204 | ins.has_constants = true; |
| 1205 | ins.constants.i16[0] = _mesa_float_to_half(1.0); |
| 1206 | |
| 1207 | for (unsigned c = 0; c < 16; ++c) |
| 1208 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 1209 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 1210 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1211 | ins.has_inline_constant = false; |
| 1212 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Italo Nicola | b1b0ce0 | 2020-07-10 14:51:52 +0000 | [diff] [blame] | 1213 | ins.src_types[1] = ins.src_types[0]; |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1214 | ins.has_constants = true; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1215 | ins.constants.u32[0] = 0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1216 | |
| 1217 | for (unsigned c = 0; c < 16; ++c) |
| 1218 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1219 | } else if (instr->op == nir_op_pack_32_2x16) { |
| 1220 | ins.dest_type = nir_type_uint16; |
| 1221 | ins.mask = mask_of(nr_components * 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1222 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1223 | } else if (instr->op == nir_op_pack_32_4x8) { |
| 1224 | ins.dest_type = nir_type_uint8; |
| 1225 | ins.mask = mask_of(nr_components * 4); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1226 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1227 | } else if (instr->op == nir_op_unpack_32_2x16) { |
| 1228 | ins.dest_type = nir_type_uint32; |
| 1229 | ins.mask = mask_of(nr_components >> 1); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1230 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1231 | } else if (instr->op == nir_op_unpack_32_4x8) { |
| 1232 | ins.dest_type = nir_type_uint32; |
| 1233 | ins.mask = mask_of(nr_components >> 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1234 | ins.is_pack = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1235 | } |
| 1236 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1237 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 1238 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 1239 | * instructions can only operate as if they were scalars. Lower |
| 1240 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1241 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1242 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1243 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1244 | unsigned swizzle_back[MIR_VEC_COMPONENTS]; |
| 1245 | memcpy(&swizzle_back, ins.swizzle[0], sizeof(swizzle_back)); |
| 1246 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1247 | midgard_instruction ins_split[MIR_VEC_COMPONENTS]; |
| 1248 | unsigned ins_count = 0; |
| 1249 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1250 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1251 | /* Mask the associated component, dropping the |
| 1252 | * instruction if needed */ |
| 1253 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1254 | ins.mask = 1 << i; |
| 1255 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1256 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1257 | for (unsigned j = 0; j < ins_count; ++j) { |
| 1258 | if (swizzle_back[i] == ins_split[j].swizzle[0][0]) { |
| 1259 | ins_split[j].mask |= ins.mask; |
| 1260 | ins.mask = 0; |
| 1261 | break; |
| 1262 | } |
| 1263 | } |
| 1264 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1265 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1266 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1267 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1268 | for (unsigned j = 0; j < MIR_VEC_COMPONENTS; ++j) |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1269 | ins.swizzle[0][j] = swizzle_back[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1270 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1271 | ins_split[ins_count] = ins; |
| 1272 | |
| 1273 | ++ins_count; |
| 1274 | } |
| 1275 | |
| 1276 | for (unsigned i = 0; i < ins_count; ++i) { |
| 1277 | emit_mir_instruction(ctx, ins_split[i]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1278 | } |
| 1279 | } else { |
| 1280 | emit_mir_instruction(ctx, ins); |
| 1281 | } |
| 1282 | } |
| 1283 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1284 | #undef ALU_CASE |
| 1285 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1286 | static void |
| 1287 | mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read) |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1288 | { |
| 1289 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1290 | unsigned nir_mask = 0; |
| 1291 | unsigned dsize = 0; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1292 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1293 | if (is_read) { |
| 1294 | nir_mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1295 | dsize = nir_dest_bit_size(intr->dest); |
| 1296 | } else { |
| 1297 | nir_mask = nir_intrinsic_write_mask(intr); |
| 1298 | dsize = 32; |
| 1299 | } |
| 1300 | |
| 1301 | /* Once we have the NIR mask, we need to normalize to work in 32-bit space */ |
Alyssa Rosenzweig | 9b8cb9f | 2020-03-09 20:19:29 -0400 | [diff] [blame] | 1302 | unsigned bytemask = pan_to_bytemask(dsize, nir_mask); |
Alyssa Rosenzweig | b91d715 | 2020-05-11 15:06:53 -0400 | [diff] [blame] | 1303 | ins->dest_type = nir_type_uint | dsize; |
Italo Nicola | 1101261 | 2020-08-26 14:56:13 +0000 | [diff] [blame] | 1304 | mir_set_bytemask(ins, bytemask); |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1305 | } |
| 1306 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1307 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1308 | * optimized) versions of UBO #0 */ |
| 1309 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1310 | static midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1311 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1312 | compiler_context *ctx, |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1313 | nir_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1314 | unsigned dest, |
| 1315 | unsigned offset, |
| 1316 | nir_src *indirect_offset, |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1317 | unsigned indirect_shift, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1318 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1319 | { |
| 1320 | /* TODO: half-floats */ |
| 1321 | |
Alyssa Rosenzweig | bc9a7d0 | 2019-11-15 14:19:34 -0500 | [diff] [blame] | 1322 | midgard_instruction ins = m_ld_ubo_int4(dest, 0); |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1323 | ins.constants.u32[0] = offset; |
Alyssa Rosenzweig | da73651 | 2019-12-19 11:12:25 -0500 | [diff] [blame] | 1324 | |
| 1325 | if (instr->type == nir_instr_type_intrinsic) |
| 1326 | mir_set_intr_mask(instr, &ins, true); |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1327 | |
| 1328 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1329 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1330 | ins.src_types[2] = nir_type_uint32; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1331 | ins.load_store.arg_2 = (indirect_shift << 5); |
Alyssa Rosenzweig | 797fa87 | 2020-07-06 10:57:04 -0400 | [diff] [blame] | 1332 | |
| 1333 | /* X component for the whole swizzle to prevent register |
| 1334 | * pressure from ballooning from the extra components */ |
| 1335 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[2]); ++i) |
| 1336 | ins.swizzle[2][i] = 0; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1337 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1338 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1339 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1340 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1341 | ins.load_store.arg_1 = index; |
| 1342 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1343 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1344 | } |
| 1345 | |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1346 | /* Globals are like UBOs if you squint. And shared memory is like globals if |
| 1347 | * you squint even harder */ |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1348 | |
| 1349 | static void |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1350 | emit_global( |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1351 | compiler_context *ctx, |
| 1352 | nir_instr *instr, |
| 1353 | bool is_read, |
| 1354 | unsigned srcdest, |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1355 | nir_src *offset, |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1356 | bool is_shared) |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1357 | { |
| 1358 | /* TODO: types */ |
| 1359 | |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 1360 | midgard_instruction ins; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1361 | |
| 1362 | if (is_read) |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1363 | ins = m_ld_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1364 | else |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1365 | ins = m_st_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1366 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1367 | mir_set_offset(ctx, &ins, offset, is_shared); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1368 | mir_set_intr_mask(instr, &ins, is_read); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1369 | |
Alyssa Rosenzweig | 41184f8 | 2020-08-27 15:13:19 -0400 | [diff] [blame] | 1370 | /* Set a valid swizzle for masked out components */ |
| 1371 | assert(ins.mask); |
| 1372 | unsigned first_component = __builtin_ffs(ins.mask) - 1; |
| 1373 | |
| 1374 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) { |
| 1375 | if (!(ins.mask & (1 << i))) |
| 1376 | ins.swizzle[0][i] = first_component; |
| 1377 | } |
| 1378 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1379 | emit_mir_instruction(ctx, ins); |
| 1380 | } |
| 1381 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 1382 | /* If is_shared is off, the only other possible value are globals, since |
| 1383 | * SSBO's are being lowered to globals through a NIR pass. */ |
| 1384 | static void |
| 1385 | emit_atomic( |
| 1386 | compiler_context *ctx, |
| 1387 | nir_intrinsic_instr *instr, |
| 1388 | bool is_shared, |
| 1389 | midgard_load_store_op op) |
| 1390 | { |
| 1391 | unsigned bitsize = nir_src_bit_size(instr->src[1]); |
| 1392 | nir_alu_type type = |
| 1393 | (op == midgard_op_atomic_imin || op == midgard_op_atomic_imax) ? |
| 1394 | nir_type_int : nir_type_uint; |
| 1395 | |
| 1396 | unsigned dest = nir_dest_index(&instr->dest); |
| 1397 | unsigned val = nir_src_index(ctx, &instr->src[1]); |
| 1398 | emit_explicit_constant(ctx, val, val); |
| 1399 | |
| 1400 | midgard_instruction ins = { |
| 1401 | .type = TAG_LOAD_STORE_4, |
| 1402 | .mask = 0xF, |
| 1403 | .dest = dest, |
| 1404 | .src = { ~0, ~0, ~0, val }, |
| 1405 | .src_types = { 0, 0, 0, type | bitsize }, |
| 1406 | .op = op |
| 1407 | }; |
| 1408 | |
| 1409 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1410 | |
| 1411 | /* cmpxchg takes an extra value in arg_2, so we don't use it for the offset */ |
| 1412 | if (op == midgard_op_atomic_cmpxchg) { |
| 1413 | unsigned addr = nir_src_index(ctx, src_offset); |
| 1414 | |
| 1415 | ins.src[1] = addr; |
| 1416 | ins.src_types[1] = nir_type_uint | nir_src_bit_size(*src_offset); |
| 1417 | |
| 1418 | unsigned xchg_val = nir_src_index(ctx, &instr->src[2]); |
| 1419 | emit_explicit_constant(ctx, xchg_val, xchg_val); |
| 1420 | |
| 1421 | ins.src[2] = val; |
| 1422 | ins.src_types[2] = type | bitsize; |
| 1423 | ins.src[3] = xchg_val; |
| 1424 | |
| 1425 | if (is_shared) |
| 1426 | ins.load_store.arg_1 |= 0x6E; |
| 1427 | } else { |
| 1428 | mir_set_offset(ctx, &ins, src_offset, is_shared); |
| 1429 | } |
| 1430 | |
| 1431 | mir_set_intr_mask(&instr->instr, &ins, true); |
| 1432 | |
| 1433 | emit_mir_instruction(ctx, ins); |
| 1434 | } |
| 1435 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1436 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1437 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1438 | compiler_context *ctx, |
| 1439 | unsigned dest, unsigned offset, |
| 1440 | unsigned nr_comp, unsigned component, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1441 | nir_src *indirect_offset, nir_alu_type type, bool flat) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1442 | { |
| 1443 | /* XXX: Half-floats? */ |
| 1444 | /* TODO: swizzle, mask */ |
| 1445 | |
| 1446 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1447 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 2d16883 | 2020-06-04 11:32:59 -0400 | [diff] [blame] | 1448 | ins.dest_type = type; |
| 1449 | |
| 1450 | if (type == nir_type_float16) { |
| 1451 | /* Ensure we are aligned so we can pack it later */ |
| 1452 | ins.mask = mask_of(ALIGN_POT(nr_comp, 2)); |
| 1453 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1454 | |
| 1455 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) |
| 1456 | ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1457 | |
| 1458 | midgard_varying_parameter p = { |
| 1459 | .is_varying = 1, |
| 1460 | .interpolation = midgard_interp_default, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1461 | .flat = flat, |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1462 | }; |
| 1463 | |
| 1464 | unsigned u; |
| 1465 | memcpy(&u, &p, sizeof(p)); |
| 1466 | ins.load_store.varying_parameters = u; |
| 1467 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1468 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1469 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1470 | ins.src_types[2] = nir_type_uint32; |
| 1471 | } else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1472 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1473 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1474 | ins.load_store.arg_1 = 0x9E; |
| 1475 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1476 | /* Use the type appropriate load */ |
| 1477 | switch (type) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1478 | case nir_type_uint32: |
| 1479 | case nir_type_bool32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1480 | ins.op = midgard_op_ld_vary_32u; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1481 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1482 | case nir_type_int32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1483 | ins.op = midgard_op_ld_vary_32i; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1484 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1485 | case nir_type_float32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1486 | ins.op = midgard_op_ld_vary_32; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1487 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1488 | case nir_type_float16: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1489 | ins.op = midgard_op_ld_vary_16; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1490 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1491 | default: |
| 1492 | unreachable("Attempted to load unknown type"); |
| 1493 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1494 | } |
| 1495 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1496 | emit_mir_instruction(ctx, ins); |
| 1497 | } |
| 1498 | |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1499 | static void |
| 1500 | emit_attr_read( |
| 1501 | compiler_context *ctx, |
| 1502 | unsigned dest, unsigned offset, |
| 1503 | unsigned nr_comp, nir_alu_type t) |
| 1504 | { |
| 1505 | midgard_instruction ins = m_ld_attr_32(dest, offset); |
| 1506 | ins.load_store.arg_1 = 0x1E; |
| 1507 | ins.load_store.arg_2 = 0x1E; |
| 1508 | ins.mask = mask_of(nr_comp); |
| 1509 | |
| 1510 | /* Use the type appropriate load */ |
| 1511 | switch (t) { |
| 1512 | case nir_type_uint: |
| 1513 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1514 | ins.op = midgard_op_ld_attr_32u; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1515 | break; |
| 1516 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1517 | ins.op = midgard_op_ld_attr_32i; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1518 | break; |
| 1519 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1520 | ins.op = midgard_op_ld_attr_32; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1521 | break; |
| 1522 | default: |
| 1523 | unreachable("Attempted to load unknown type"); |
| 1524 | break; |
| 1525 | } |
| 1526 | |
| 1527 | emit_mir_instruction(ctx, ins); |
| 1528 | } |
| 1529 | |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1530 | static void |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1531 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1532 | unsigned nr_components, unsigned offset) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1533 | { |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1534 | nir_dest nir_dest; |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1535 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1536 | /* Figure out which uniform this is */ |
Alyssa Rosenzweig | e610267 | 2020-03-10 16:06:30 -0400 | [diff] [blame] | 1537 | int sysval = panfrost_sysval_for_instr(instr, &nir_dest); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1538 | void *val = _mesa_hash_table_u64_search(ctx->sysvals.sysval_to_id, sysval); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1539 | |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1540 | unsigned dest = nir_dest_index(&nir_dest); |
| 1541 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1542 | /* Sysvals are prefix uniforms */ |
| 1543 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1544 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1545 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1546 | midgard_instruction *ins = |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1547 | emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0, 0); |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1548 | |
| 1549 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1550 | } |
| 1551 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1552 | static unsigned |
| 1553 | compute_builtin_arg(nir_op op) |
| 1554 | { |
| 1555 | switch (op) { |
| 1556 | case nir_intrinsic_load_work_group_id: |
| 1557 | return 0x14; |
| 1558 | case nir_intrinsic_load_local_invocation_id: |
| 1559 | return 0x10; |
| 1560 | default: |
| 1561 | unreachable("Invalid compute paramater loaded"); |
| 1562 | } |
| 1563 | } |
| 1564 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1565 | static void |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1566 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z, unsigned src_s, enum midgard_rt_id rt) |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1567 | { |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1568 | assert(rt < ARRAY_SIZE(ctx->writeout_branch)); |
| 1569 | |
| 1570 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 1571 | |
| 1572 | assert(!br); |
| 1573 | |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 1574 | emit_explicit_constant(ctx, src, src); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1575 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1576 | struct midgard_instruction ins = |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1577 | v_branch(false, false); |
| 1578 | |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1579 | bool depth_only = (rt == MIDGARD_ZS_RT); |
| 1580 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1581 | ins.writeout = depth_only ? 0 : PAN_WRITEOUT_C; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1582 | |
| 1583 | /* Add dependencies */ |
Alyssa Rosenzweig | 7652983 | 2019-08-30 11:01:15 -0700 | [diff] [blame] | 1584 | ins.src[0] = src; |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1585 | ins.src_types[0] = nir_type_uint32; |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1586 | ins.constants.u32[0] = depth_only ? 0xFF : (rt - MIDGARD_COLOR_RT0) * 0x100; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 1587 | for (int i = 0; i < 4; ++i) |
| 1588 | ins.swizzle[0][i] = i; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1589 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1590 | if (~src_z) { |
| 1591 | emit_explicit_constant(ctx, src_z, src_z); |
| 1592 | ins.src[2] = src_z; |
| 1593 | ins.src_types[2] = nir_type_uint32; |
| 1594 | ins.writeout |= PAN_WRITEOUT_Z; |
| 1595 | } |
| 1596 | if (~src_s) { |
| 1597 | emit_explicit_constant(ctx, src_s, src_s); |
| 1598 | ins.src[3] = src_s; |
| 1599 | ins.src_types[3] = nir_type_uint32; |
| 1600 | ins.writeout |= PAN_WRITEOUT_S; |
| 1601 | } |
| 1602 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1603 | /* Emit the branch */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1604 | br = emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 281cc6f | 2019-11-23 12:43:55 -0500 | [diff] [blame] | 1605 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 1606 | ctx->writeout_branch[rt] = br; |
| 1607 | |
| 1608 | /* Push our current location = current block count - 1 = where we'll |
| 1609 | * jump to. Maybe a bit too clever for my own good */ |
| 1610 | |
| 1611 | br->branch.target_block = ctx->block_count - 1; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1612 | } |
| 1613 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1614 | static void |
| 1615 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1616 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1617 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1618 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1619 | ins.mask = mask_of(3); |
Alyssa Rosenzweig | d3747fb | 2020-02-12 08:39:29 -0500 | [diff] [blame] | 1620 | ins.swizzle[0][3] = COMPONENT_X; /* xyzx */ |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1621 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1622 | emit_mir_instruction(ctx, ins); |
| 1623 | } |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1624 | |
| 1625 | static unsigned |
| 1626 | vertex_builtin_arg(nir_op op) |
| 1627 | { |
| 1628 | switch (op) { |
| 1629 | case nir_intrinsic_load_vertex_id: |
| 1630 | return PAN_VERTEX_ID; |
| 1631 | case nir_intrinsic_load_instance_id: |
| 1632 | return PAN_INSTANCE_ID; |
| 1633 | default: |
| 1634 | unreachable("Invalid vertex builtin"); |
| 1635 | } |
| 1636 | } |
| 1637 | |
| 1638 | static void |
| 1639 | emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1640 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1641 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1642 | emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int); |
| 1643 | } |
| 1644 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1645 | static void |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1646 | emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx) |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1647 | { |
| 1648 | unsigned reg = nir_dest_index(&instr->dest); |
| 1649 | |
| 1650 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1651 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1652 | ld.load_store.address = idx; |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1653 | ld.load_store.arg_2 = 0x1E; |
| 1654 | |
| 1655 | for (int i = 0; i < 4; ++i) |
| 1656 | ld.swizzle[0][i] = COMPONENT_X; |
| 1657 | |
| 1658 | emit_mir_instruction(ctx, ld); |
| 1659 | } |
| 1660 | |
| 1661 | static void |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1662 | emit_control_barrier(compiler_context *ctx) |
| 1663 | { |
| 1664 | midgard_instruction ins = { |
| 1665 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | fde1f2b | 2020-05-13 11:05:34 -0400 | [diff] [blame] | 1666 | .dest = ~0, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1667 | .src = { ~0, ~0, ~0, ~0 }, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 1668 | .op = TEXTURE_OP_BARRIER, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1669 | }; |
| 1670 | |
| 1671 | emit_mir_instruction(ctx, ins); |
| 1672 | } |
| 1673 | |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1674 | static unsigned |
| 1675 | mir_get_branch_cond(nir_src *src, bool *invert) |
| 1676 | { |
| 1677 | /* Wrap it. No swizzle since it's a scalar */ |
| 1678 | |
| 1679 | nir_alu_src alu = { |
| 1680 | .src = *src |
| 1681 | }; |
| 1682 | |
| 1683 | *invert = pan_has_source_mod(&alu, nir_op_inot); |
| 1684 | return nir_src_index(NULL, &alu.src); |
| 1685 | } |
| 1686 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1687 | static uint8_t |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1688 | output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr) |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1689 | { |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1690 | if (ctx->is_blend) |
| 1691 | return ctx->blend_rt; |
| 1692 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1693 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1694 | var = search_var(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr)); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1695 | assert(var); |
| 1696 | |
| 1697 | unsigned loc = var->data.location; |
| 1698 | |
| 1699 | if (loc == FRAG_RESULT_COLOR) |
| 1700 | loc = FRAG_RESULT_DATA0; |
| 1701 | |
| 1702 | if (loc >= FRAG_RESULT_DATA0) |
| 1703 | return loc - FRAG_RESULT_DATA0; |
| 1704 | |
| 1705 | if (loc == FRAG_RESULT_DEPTH) |
| 1706 | return 0x1F; |
| 1707 | if (loc == FRAG_RESULT_STENCIL) |
| 1708 | return 0x1E; |
| 1709 | |
Icecream95 | 6493d29 | 2020-07-14 15:06:09 +1200 | [diff] [blame] | 1710 | unreachable("Invalid RT to load from"); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1711 | } |
| 1712 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1713 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1714 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1715 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1716 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1717 | |
| 1718 | switch (instr->intrinsic) { |
| 1719 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1720 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1721 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1722 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1723 | discard.branch.target_type = TARGET_DISCARD; |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1724 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1725 | if (conditional) { |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1726 | discard.src[0] = mir_get_branch_cond(&instr->src[0], |
| 1727 | &discard.branch.invert_conditional); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1728 | discard.src_types[0] = nir_type_uint32; |
| 1729 | } |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1730 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1731 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1732 | schedule_barrier(ctx); |
| 1733 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1734 | break; |
| 1735 | } |
| 1736 | |
| 1737 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1738 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1739 | case nir_intrinsic_load_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1740 | case nir_intrinsic_load_shared: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1741 | case nir_intrinsic_load_input: |
| 1742 | case nir_intrinsic_load_interpolated_input: { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1743 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1744 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1745 | bool is_global = instr->intrinsic == nir_intrinsic_load_global; |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1746 | bool is_shared = instr->intrinsic == nir_intrinsic_load_shared; |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1747 | bool is_flat = instr->intrinsic == nir_intrinsic_load_input; |
| 1748 | bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1749 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1750 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1751 | /* TODO: Infer type? Does it matter? */ |
| 1752 | nir_alu_type t = |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1753 | (is_ubo || is_global || is_shared) ? nir_type_uint : |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1754 | (is_interp) ? nir_type_float : |
| 1755 | nir_intrinsic_type(instr); |
| 1756 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1757 | t = nir_alu_type_get_base_type(t); |
| 1758 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1759 | if (!(is_ubo || is_global)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1760 | offset = nir_intrinsic_base(instr); |
| 1761 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1762 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1763 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1764 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1765 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1766 | |
| 1767 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1768 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1769 | |
| 1770 | if (direct) |
| 1771 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1772 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1773 | /* We may need to apply a fractional offset */ |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1774 | int component = (is_flat || is_interp) ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1775 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1776 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1777 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1778 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1779 | emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysvals.sysval_count + offset) * 16, indirect_offset, 4, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1780 | } else if (is_ubo) { |
| 1781 | nir_src index = instr->src[0]; |
| 1782 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1783 | /* TODO: Is indirect block number possible? */ |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1784 | assert(nir_src_is_const(index)); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1785 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1786 | uint32_t uindex = nir_src_as_uint(index) + 1; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1787 | emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1788 | } else if (is_global || is_shared) { |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1789 | emit_global(ctx, &instr->instr, true, reg, src_offset, is_shared); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1790 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1791 | emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1792 | } else if (ctx->is_blend) { |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1793 | /* ctx->blend_input will be precoloured to r0/r2, where |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1794 | * the input is preloaded */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1795 | |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1796 | unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input; |
| 1797 | |
| 1798 | if (*input == ~0) |
| 1799 | *input = reg; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1800 | else |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1801 | emit_mir_instruction(ctx, v_mov(*input, reg)); |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1802 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
| 1803 | emit_attr_read(ctx, reg, offset, nr_comp, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1804 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1805 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1806 | assert(0); |
| 1807 | } |
| 1808 | |
| 1809 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1810 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1811 | |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1812 | /* Artefact of load_interpolated_input. TODO: other barycentric modes */ |
| 1813 | case nir_intrinsic_load_barycentric_pixel: |
Tomeu Vizoso | 2504206 | 2020-01-03 09:42:11 +0100 | [diff] [blame] | 1814 | case nir_intrinsic_load_barycentric_centroid: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1815 | break; |
| 1816 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1817 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1818 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1819 | case nir_intrinsic_load_raw_output_pan: { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1820 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1821 | |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1822 | /* T720 and below use different blend opcodes with slightly |
| 1823 | * different semantics than T760 and up */ |
| 1824 | |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 1825 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1826 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1827 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1828 | |
Icecream95 | c20d166 | 2020-07-16 14:16:11 +1200 | [diff] [blame] | 1829 | if (nir_src_is_const(instr->src[0])) { |
| 1830 | ld.load_store.arg_1 = nir_src_as_uint(instr->src[0]); |
| 1831 | } else { |
| 1832 | ld.load_store.varying_parameters = 2; |
| 1833 | ld.src[1] = nir_src_index(ctx, &instr->src[0]); |
| 1834 | ld.src_types[1] = nir_type_int32; |
| 1835 | } |
| 1836 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1837 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1838 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 5a175e4 | 2020-05-29 21:11:11 -0400 | [diff] [blame] | 1839 | ld.load_store.address = 16; |
| 1840 | ld.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1841 | } |
| 1842 | |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1843 | emit_mir_instruction(ctx, ld); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1844 | break; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1845 | } |
| 1846 | |
| 1847 | case nir_intrinsic_load_output: { |
| 1848 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1849 | |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1850 | unsigned bits = nir_dest_bit_size(instr->dest); |
| 1851 | |
| 1852 | midgard_instruction ld; |
| 1853 | if (bits == 16) |
| 1854 | ld = m_ld_color_buffer_as_fp16(reg, 0); |
| 1855 | else |
| 1856 | ld = m_ld_color_buffer_as_fp32(reg, 0); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1857 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1858 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1859 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1860 | for (unsigned c = 4; c < 16; ++c) |
| 1861 | ld.swizzle[0][c] = 0; |
| 1862 | |
| 1863 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1864 | if (bits == 16) |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1865 | ld.op = midgard_op_ld_color_buffer_as_fp16_old; |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1866 | else |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1867 | ld.op = midgard_op_ld_color_buffer_as_fp32_old; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1868 | ld.load_store.address = 1; |
| 1869 | ld.load_store.arg_2 = 0x1E; |
| 1870 | } |
| 1871 | |
| 1872 | emit_mir_instruction(ctx, ld); |
| 1873 | break; |
| 1874 | } |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1875 | |
| 1876 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1877 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1878 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1879 | |
| 1880 | /* Blend constants are embedded directly in the shader and |
| 1881 | * patched in, so we use some magic routing */ |
| 1882 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1883 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1884 | ins.has_constants = true; |
| 1885 | ins.has_blend_constant = true; |
| 1886 | emit_mir_instruction(ctx, ins); |
| 1887 | break; |
| 1888 | } |
| 1889 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1890 | case nir_intrinsic_store_output: |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1891 | case nir_intrinsic_store_combined_output_pan: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1892 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1893 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1894 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1895 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1896 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1897 | |
| 1898 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1899 | bool combined = instr->intrinsic == |
| 1900 | nir_intrinsic_store_combined_output_pan; |
| 1901 | |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1902 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1903 | var = search_var(ctx->nir, nir_var_shader_out, |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1904 | nir_intrinsic_base(instr)); |
| 1905 | assert(var); |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1906 | |
| 1907 | /* Dual-source blend writeout is done by leaving the |
| 1908 | * value in r2 for the blend shader to use. */ |
| 1909 | if (var->data.index) { |
| 1910 | if (instr->src[0].is_ssa) { |
| 1911 | emit_explicit_constant(ctx, reg, reg); |
| 1912 | |
| 1913 | unsigned out = make_compiler_temp(ctx); |
| 1914 | |
| 1915 | midgard_instruction ins = v_mov(reg, out); |
| 1916 | emit_mir_instruction(ctx, ins); |
| 1917 | |
| 1918 | ctx->blend_src1 = out; |
| 1919 | } else { |
| 1920 | ctx->blend_src1 = reg; |
| 1921 | } |
| 1922 | |
| 1923 | break; |
| 1924 | } |
| 1925 | |
| 1926 | enum midgard_rt_id rt; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1927 | if (var->data.location == FRAG_RESULT_COLOR) |
| 1928 | rt = MIDGARD_COLOR_RT0; |
| 1929 | else if (var->data.location >= FRAG_RESULT_DATA0) |
| 1930 | rt = MIDGARD_COLOR_RT0 + var->data.location - |
| 1931 | FRAG_RESULT_DATA0; |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1932 | else if (combined) |
| 1933 | rt = MIDGARD_ZS_RT; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1934 | else |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 1935 | unreachable("bad rt"); |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1936 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1937 | unsigned reg_z = ~0, reg_s = ~0; |
| 1938 | if (combined) { |
| 1939 | unsigned writeout = nir_intrinsic_component(instr); |
| 1940 | if (writeout & PAN_WRITEOUT_Z) |
| 1941 | reg_z = nir_src_index(ctx, &instr->src[2]); |
| 1942 | if (writeout & PAN_WRITEOUT_S) |
| 1943 | reg_s = nir_src_index(ctx, &instr->src[3]); |
| 1944 | } |
| 1945 | |
| 1946 | emit_fragment_store(ctx, reg, reg_z, reg_s, rt); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1947 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1948 | assert(instr->intrinsic == nir_intrinsic_store_output); |
| 1949 | |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1950 | /* We should have been vectorized, though we don't |
| 1951 | * currently check that st_vary is emitted only once |
| 1952 | * per slot (this is relevant, since there's not a mask |
| 1953 | * parameter available on the store [set to 0 by the |
| 1954 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1955 | * swizzle. If this is a constant source, we'll need to |
| 1956 | * emit that explicitly. */ |
| 1957 | |
| 1958 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1959 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1960 | unsigned dst_component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | 2788721 | 2019-08-15 16:53:03 -0700 | [diff] [blame] | 1961 | unsigned nr_comp = nir_src_num_components(instr->src[0]); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1962 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1963 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1964 | st.load_store.arg_1 = 0x9E; |
| 1965 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1966 | |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1967 | switch (nir_alu_type_get_base_type(nir_intrinsic_type(instr))) { |
| 1968 | case nir_type_uint: |
| 1969 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1970 | st.op = midgard_op_st_vary_32u; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1971 | break; |
| 1972 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1973 | st.op = midgard_op_st_vary_32i; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1974 | break; |
| 1975 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1976 | st.op = midgard_op_st_vary_32; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1977 | break; |
| 1978 | default: |
| 1979 | unreachable("Attempted to store unknown type"); |
| 1980 | break; |
| 1981 | } |
| 1982 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1983 | /* nir_intrinsic_component(store_intr) encodes the |
| 1984 | * destination component start. Source component offset |
| 1985 | * adjustment is taken care of in |
| 1986 | * install_registers_instr(), when offset_swizzle() is |
| 1987 | * called. |
| 1988 | */ |
| 1989 | unsigned src_component = COMPONENT_X; |
| 1990 | |
| 1991 | assert(nr_comp > 0); |
| 1992 | for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) { |
| 1993 | st.swizzle[0][i] = src_component; |
| 1994 | if (i >= dst_component && i < dst_component + nr_comp - 1) |
| 1995 | src_component++; |
| 1996 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1997 | |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1998 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1999 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2000 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2001 | assert(0); |
| 2002 | } |
| 2003 | |
| 2004 | break; |
| 2005 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 2006 | /* Special case of store_output for lowered blend shaders */ |
| 2007 | case nir_intrinsic_store_raw_output_pan: |
| 2008 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 2009 | reg = nir_src_index(ctx, &instr->src[0]); |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 2010 | emit_fragment_store(ctx, reg, ~0, ~0, ctx->blend_rt); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 2011 | break; |
| 2012 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2013 | case nir_intrinsic_store_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2014 | case nir_intrinsic_store_shared: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2015 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2016 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2017 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 2018 | emit_global(ctx, &instr->instr, false, reg, &instr->src[1], instr->intrinsic == nir_intrinsic_store_shared); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2019 | break; |
| 2020 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2021 | case nir_intrinsic_load_ssbo_address: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2022 | emit_sysval_read(ctx, &instr->instr, 1, 0); |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 2023 | break; |
| 2024 | |
| 2025 | case nir_intrinsic_get_buffer_size: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2026 | emit_sysval_read(ctx, &instr->instr, 1, 8); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2027 | break; |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 2028 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2029 | case nir_intrinsic_load_viewport_scale: |
| 2030 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 2031 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 2032 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2033 | emit_sysval_read(ctx, &instr->instr, 3, 0); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2034 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2035 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 2036 | case nir_intrinsic_load_work_group_id: |
| 2037 | case nir_intrinsic_load_local_invocation_id: |
| 2038 | emit_compute_builtin(ctx, instr); |
| 2039 | break; |
| 2040 | |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 2041 | case nir_intrinsic_load_vertex_id: |
| 2042 | case nir_intrinsic_load_instance_id: |
| 2043 | emit_vertex_builtin(ctx, instr); |
| 2044 | break; |
| 2045 | |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2046 | case nir_intrinsic_load_sample_mask_in: |
| 2047 | emit_special(ctx, instr, 96); |
| 2048 | break; |
| 2049 | |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2050 | case nir_intrinsic_load_sample_id: |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2051 | emit_special(ctx, instr, 97); |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2052 | break; |
| 2053 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 2054 | case nir_intrinsic_memory_barrier_buffer: |
| 2055 | case nir_intrinsic_memory_barrier_shared: |
| 2056 | break; |
| 2057 | |
| 2058 | case nir_intrinsic_control_barrier: |
| 2059 | schedule_barrier(ctx); |
| 2060 | emit_control_barrier(ctx); |
| 2061 | schedule_barrier(ctx); |
| 2062 | break; |
| 2063 | |
Italo Nicola | d7b6d2e | 2020-08-31 17:32:30 +0000 | [diff] [blame] | 2064 | ATOMIC_CASE(ctx, instr, add, add); |
| 2065 | ATOMIC_CASE(ctx, instr, and, and); |
| 2066 | ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg); |
| 2067 | ATOMIC_CASE(ctx, instr, exchange, xchg); |
| 2068 | ATOMIC_CASE(ctx, instr, imax, imax); |
| 2069 | ATOMIC_CASE(ctx, instr, imin, imin); |
| 2070 | ATOMIC_CASE(ctx, instr, or, or); |
| 2071 | ATOMIC_CASE(ctx, instr, umax, umax); |
| 2072 | ATOMIC_CASE(ctx, instr, umin, umin); |
| 2073 | ATOMIC_CASE(ctx, instr, xor, xor); |
| 2074 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2075 | default: |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2076 | fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2077 | assert(0); |
| 2078 | break; |
| 2079 | } |
| 2080 | } |
| 2081 | |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2082 | /* Returns dimension with 0 special casing cubemaps */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2083 | static unsigned |
| 2084 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 2085 | { |
| 2086 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2087 | case GLSL_SAMPLER_DIM_1D: |
| 2088 | case GLSL_SAMPLER_DIM_BUF: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2089 | return 1; |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2090 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2091 | case GLSL_SAMPLER_DIM_2D: |
Alyssa Rosenzweig | a2748d4 | 2020-06-30 15:31:30 -0400 | [diff] [blame] | 2092 | case GLSL_SAMPLER_DIM_MS: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2093 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 2094 | case GLSL_SAMPLER_DIM_RECT: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2095 | return 2; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2096 | |
| 2097 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2098 | return 3; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2099 | |
| 2100 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2101 | return 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2102 | |
| 2103 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2104 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2105 | assert(0); |
| 2106 | return 0; |
| 2107 | } |
| 2108 | } |
| 2109 | |
Alyssa Rosenzweig | c6c906e | 2020-05-21 18:02:38 -0400 | [diff] [blame] | 2110 | /* Tries to attach an explicit LOD or bias as a constant. Returns whether this |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2111 | * was successful */ |
| 2112 | |
| 2113 | static bool |
| 2114 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2115 | compiler_context *ctx, |
| 2116 | nir_src lod, |
| 2117 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2118 | { |
| 2119 | /* To attach as constant, it has to *be* constant */ |
| 2120 | |
| 2121 | if (!nir_src_is_const(lod)) |
| 2122 | return false; |
| 2123 | |
| 2124 | float f = nir_src_as_float(lod); |
| 2125 | |
| 2126 | /* Break into fixed-point */ |
| 2127 | signed lod_int = f; |
| 2128 | float lod_frac = f - lod_int; |
| 2129 | |
| 2130 | /* Carry over negative fractions */ |
| 2131 | if (lod_frac < 0.0) { |
| 2132 | lod_int--; |
| 2133 | lod_frac += 1.0; |
| 2134 | } |
| 2135 | |
| 2136 | /* Encode */ |
| 2137 | word->bias = float_to_ubyte(lod_frac); |
| 2138 | word->bias_int = lod_int; |
| 2139 | |
| 2140 | return true; |
| 2141 | } |
| 2142 | |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2143 | static enum mali_texture_mode |
| 2144 | mdg_texture_mode(nir_tex_instr *instr) |
| 2145 | { |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2146 | if (instr->op == nir_texop_tg4 && instr->is_shadow) |
| 2147 | return TEXTURE_GATHER_SHADOW; |
| 2148 | else if (instr->op == nir_texop_tg4) |
| 2149 | return TEXTURE_GATHER_X + instr->component; |
| 2150 | else if (instr->is_shadow) |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2151 | return TEXTURE_SHADOW; |
| 2152 | else |
| 2153 | return TEXTURE_NORMAL; |
| 2154 | } |
| 2155 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2156 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2157 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2158 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2159 | { |
| 2160 | /* TODO */ |
| 2161 | //assert (!instr->sampler); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2162 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2163 | nir_dest *dest = &instr->dest; |
| 2164 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2165 | int texture_index = instr->texture_index; |
| 2166 | int sampler_index = texture_index; |
| 2167 | |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2168 | nir_alu_type dest_base = nir_alu_type_get_base_type(instr->dest_type); |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2169 | nir_alu_type dest_type = dest_base | nir_dest_bit_size(*dest); |
| 2170 | |
| 2171 | /* texture instructions support float outmods */ |
| 2172 | unsigned outmod = midgard_outmod_none; |
| 2173 | if (dest_base == nir_type_float) { |
| 2174 | outmod = mir_determine_float_outmod(ctx, &dest, 0); |
| 2175 | } |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2176 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2177 | midgard_instruction ins = { |
| 2178 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 2179 | .mask = 0xF, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2180 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2181 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2182 | .dest_type = dest_type, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2183 | .swizzle = SWIZZLE_IDENTITY_4, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2184 | .outmod = outmod, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 2185 | .op = midgard_texop, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2186 | .texture = { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2187 | .format = midgard_tex_format(instr->sampler_dim), |
| 2188 | .texture_handle = texture_index, |
| 2189 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2190 | .mode = mdg_texture_mode(instr) |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2191 | } |
| 2192 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 2193 | |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2194 | if (instr->is_shadow && !instr->is_new_style_shadow && instr->op != nir_texop_tg4) |
Icecream95 | d1290e7 | 2020-05-12 10:16:31 +1200 | [diff] [blame] | 2195 | for (int i = 0; i < 4; ++i) |
| 2196 | ins.swizzle[0][i] = COMPONENT_X; |
| 2197 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2198 | /* We may need a temporary for the coordinate */ |
| 2199 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2200 | bool needs_temp_coord = |
| 2201 | (midgard_texop == TEXTURE_OP_TEXEL_FETCH) || |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2202 | (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) || |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2203 | (instr->is_shadow); |
| 2204 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2205 | unsigned coords = needs_temp_coord ? make_compiler_temp_reg(ctx) : 0; |
| 2206 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2207 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2208 | int index = nir_src_index(ctx, &instr->src[i].src); |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2209 | unsigned nr_components = nir_src_num_components(instr->src[i].src); |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2210 | unsigned sz = nir_src_bit_size(instr->src[i].src); |
| 2211 | nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2212 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2213 | switch (instr->src[i].src_type) { |
| 2214 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2215 | emit_explicit_constant(ctx, index, index); |
| 2216 | |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2217 | unsigned coord_mask = mask_of(instr->coord_components); |
| 2218 | |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2219 | bool flip_zw = (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) && (coord_mask & (1 << COMPONENT_Z)); |
| 2220 | |
| 2221 | if (flip_zw) |
| 2222 | coord_mask ^= ((1 << COMPONENT_Z) | (1 << COMPONENT_W)); |
| 2223 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2224 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
| 2225 | /* texelFetch is undefined on samplerCube */ |
| 2226 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 2227 | |
| 2228 | /* For cubemaps, we use a special ld/st op to |
| 2229 | * select the face and copy the xy into the |
| 2230 | * texture register */ |
| 2231 | |
| 2232 | midgard_instruction ld = m_ld_cubemap_coords(coords, 0); |
| 2233 | ld.src[1] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2234 | ld.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2235 | ld.mask = 0x3; /* xy */ |
| 2236 | ld.load_store.arg_1 = 0x20; |
| 2237 | ld.swizzle[1][3] = COMPONENT_X; |
| 2238 | emit_mir_instruction(ctx, ld); |
| 2239 | |
| 2240 | /* xyzw -> xyxx */ |
| 2241 | ins.swizzle[1][2] = instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
| 2242 | ins.swizzle[1][3] = COMPONENT_X; |
| 2243 | } else if (needs_temp_coord) { |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2244 | /* mov coord_temp, coords */ |
| 2245 | midgard_instruction mov = v_mov(index, coords); |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2246 | mov.mask = coord_mask; |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2247 | |
| 2248 | if (flip_zw) |
| 2249 | mov.swizzle[1][COMPONENT_W] = COMPONENT_Z; |
| 2250 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2251 | emit_mir_instruction(ctx, mov); |
| 2252 | } else { |
| 2253 | coords = index; |
| 2254 | } |
| 2255 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2256 | ins.src[1] = coords; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2257 | ins.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2258 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2259 | /* Texelfetch coordinates uses all four elements |
| 2260 | * (xyz/index) regardless of texture dimensionality, |
| 2261 | * which means it's necessary to zero the unused |
| 2262 | * components to keep everything happy */ |
| 2263 | |
| 2264 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2265 | /* mov index.zw, #0, or generalized */ |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2266 | midgard_instruction mov = |
| 2267 | v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), coords); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2268 | mov.has_constants = true; |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2269 | mov.mask = coord_mask ^ 0xF; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2270 | emit_mir_instruction(ctx, mov); |
| 2271 | } |
| 2272 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2273 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2274 | /* Array component in w but NIR wants it in z, |
| 2275 | * but if we have a temp coord we already fixed |
| 2276 | * that up */ |
| 2277 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2278 | if (nr_components == 3) { |
| 2279 | ins.swizzle[1][2] = COMPONENT_Z; |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2280 | ins.swizzle[1][3] = needs_temp_coord ? COMPONENT_W : COMPONENT_Z; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2281 | } else if (nr_components == 2) { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2282 | ins.swizzle[1][2] = |
| 2283 | instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2284 | ins.swizzle[1][3] = COMPONENT_X; |
| 2285 | } else |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2286 | unreachable("Invalid texture 2D components"); |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 2287 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2288 | |
Alyssa Rosenzweig | 64b2fe9 | 2019-12-20 12:38:24 -0500 | [diff] [blame] | 2289 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 2290 | /* We zeroed */ |
| 2291 | ins.swizzle[1][2] = COMPONENT_Z; |
| 2292 | ins.swizzle[1][3] = COMPONENT_W; |
| 2293 | } |
| 2294 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2295 | break; |
| 2296 | } |
| 2297 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 2298 | case nir_tex_src_bias: |
| 2299 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2300 | /* Try as a constant if we can */ |
| 2301 | |
| 2302 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 2303 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 2304 | break; |
| 2305 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2306 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2307 | ins.src[2] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2308 | ins.src_types[2] = T; |
Alyssa Rosenzweig | 72e5749 | 2019-12-20 12:34:20 -0500 | [diff] [blame] | 2309 | |
| 2310 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2311 | ins.swizzle[2][c] = COMPONENT_X; |
| 2312 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2313 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2314 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2315 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2316 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2317 | |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2318 | case nir_tex_src_offset: { |
| 2319 | ins.texture.offset_register = true; |
| 2320 | ins.src[3] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2321 | ins.src_types[3] = T; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2322 | |
| 2323 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2324 | ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c; |
| 2325 | |
| 2326 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | 4ec1f95 | 2019-12-20 12:58:10 -0500 | [diff] [blame] | 2327 | break; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2328 | }; |
| 2329 | |
Alyssa Rosenzweig | 6d9f951 | 2020-06-30 15:31:39 -0400 | [diff] [blame] | 2330 | case nir_tex_src_comparator: |
| 2331 | case nir_tex_src_ms_index: { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2332 | unsigned comp = COMPONENT_Z; |
| 2333 | |
| 2334 | /* mov coord_temp.foo, coords */ |
| 2335 | midgard_instruction mov = v_mov(index, coords); |
| 2336 | mov.mask = 1 << comp; |
| 2337 | |
| 2338 | for (unsigned i = 0; i < MIR_VEC_COMPONENTS; ++i) |
| 2339 | mov.swizzle[1][i] = COMPONENT_X; |
| 2340 | |
| 2341 | emit_mir_instruction(ctx, mov); |
| 2342 | break; |
| 2343 | } |
| 2344 | |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2345 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2346 | fprintf(stderr, "Unknown texture source type: %d\n", instr->src[i].src_type); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2347 | assert(0); |
| 2348 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2349 | } |
| 2350 | } |
| 2351 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2352 | emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2353 | } |
| 2354 | |
| 2355 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2356 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 2357 | { |
| 2358 | switch (instr->op) { |
| 2359 | case nir_texop_tex: |
| 2360 | case nir_texop_txb: |
| 2361 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 2362 | break; |
| 2363 | case nir_texop_txl: |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2364 | case nir_texop_tg4: |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2365 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 2366 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2367 | case nir_texop_txf: |
Alyssa Rosenzweig | 63a8722 | 2020-06-30 15:32:01 -0400 | [diff] [blame] | 2368 | case nir_texop_txf_ms: |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2369 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 2370 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2371 | case nir_texop_txs: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2372 | emit_sysval_read(ctx, &instr->instr, 4, 0); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2373 | break; |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2374 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2375 | fprintf(stderr, "Unhandled texture op: %d\n", instr->op); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2376 | assert(0); |
| 2377 | } |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2378 | } |
| 2379 | } |
| 2380 | |
| 2381 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2382 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 2383 | { |
| 2384 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2385 | case nir_jump_break: { |
| 2386 | /* Emit a branch out of the loop */ |
| 2387 | struct midgard_instruction br = v_branch(false, false); |
| 2388 | br.branch.target_type = TARGET_BREAK; |
| 2389 | br.branch.target_break = ctx->current_loop_depth; |
| 2390 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2391 | break; |
| 2392 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2393 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2394 | default: |
| 2395 | DBG("Unknown jump type %d\n", instr->type); |
| 2396 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2397 | } |
| 2398 | } |
| 2399 | |
| 2400 | static void |
| 2401 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 2402 | { |
| 2403 | switch (instr->type) { |
| 2404 | case nir_instr_type_load_const: |
| 2405 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 2406 | break; |
| 2407 | |
| 2408 | case nir_instr_type_intrinsic: |
| 2409 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 2410 | break; |
| 2411 | |
| 2412 | case nir_instr_type_alu: |
| 2413 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 2414 | break; |
| 2415 | |
| 2416 | case nir_instr_type_tex: |
| 2417 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 2418 | break; |
| 2419 | |
| 2420 | case nir_instr_type_jump: |
| 2421 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 2422 | break; |
| 2423 | |
| 2424 | case nir_instr_type_ssa_undef: |
| 2425 | /* Spurious */ |
| 2426 | break; |
| 2427 | |
| 2428 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2429 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2430 | break; |
| 2431 | } |
| 2432 | } |
| 2433 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2434 | |
| 2435 | /* ALU instructions can inline or embed constants, which decreases register |
| 2436 | * pressure and saves space. */ |
| 2437 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2438 | #define CONDITIONAL_ATTACH(idx) { \ |
| 2439 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2440 | \ |
| 2441 | if (entry) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2442 | attach_constants(ctx, alu, entry, alu->src[idx] + 1); \ |
| 2443 | alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2444 | } \ |
| 2445 | } |
| 2446 | |
| 2447 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2448 | inline_alu_constants(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2449 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2450 | mir_foreach_instr_in_block(block, alu) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2451 | /* Other instructions cannot inline constants */ |
| 2452 | if (alu->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 2453 | if (alu->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2454 | |
| 2455 | /* If there is already a constant here, we can do nothing */ |
| 2456 | if (alu->has_constants) continue; |
| 2457 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2458 | CONDITIONAL_ATTACH(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2459 | |
| 2460 | if (!alu->has_constants) { |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2461 | CONDITIONAL_ATTACH(1) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2462 | } else if (!alu->inline_constant) { |
| 2463 | /* Corner case: _two_ vec4 constants, for instance with a |
| 2464 | * csel. For this case, we can only use a constant |
| 2465 | * register for one, we'll have to emit a move for the |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2466 | * other. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2467 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2468 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1); |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2469 | unsigned scratch = make_compiler_temp(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2470 | |
| 2471 | if (entry) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 2472 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch); |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2473 | attach_constants(ctx, &ins, entry, alu->src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2474 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2475 | /* Set the source */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2476 | alu->src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2477 | |
| 2478 | /* Inject us -before- the last instruction which set r31 */ |
Boris Brezillon | 938c5b0 | 2019-08-28 09:17:21 +0200 | [diff] [blame] | 2479 | mir_insert_instruction_before(ctx, mir_prev_op(alu), ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2480 | } |
| 2481 | } |
| 2482 | } |
| 2483 | } |
| 2484 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2485 | unsigned |
| 2486 | max_bitsize_for_alu(midgard_instruction *ins) |
| 2487 | { |
| 2488 | unsigned max_bitsize = 0; |
| 2489 | for (int i = 0; i < MIR_SRC_COUNT; i++) { |
| 2490 | if (ins->src[i] == ~0) continue; |
| 2491 | unsigned src_bitsize = nir_alu_type_get_type_size(ins->src_types[i]); |
| 2492 | max_bitsize = MAX2(src_bitsize, max_bitsize); |
| 2493 | } |
| 2494 | unsigned dst_bitsize = nir_alu_type_get_type_size(ins->dest_type); |
| 2495 | max_bitsize = MAX2(dst_bitsize, max_bitsize); |
| 2496 | |
| 2497 | /* We don't have fp16 LUTs, so we'll want to emit code like: |
| 2498 | * |
| 2499 | * vlut.fsinr hr0, hr0 |
| 2500 | * |
| 2501 | * where both input and output are 16-bit but the operation is carried |
| 2502 | * out in 32-bit |
| 2503 | */ |
| 2504 | |
| 2505 | switch (ins->op) { |
| 2506 | case midgard_alu_op_fsqrt: |
| 2507 | case midgard_alu_op_frcp: |
| 2508 | case midgard_alu_op_frsqrt: |
| 2509 | case midgard_alu_op_fsin: |
| 2510 | case midgard_alu_op_fcos: |
| 2511 | case midgard_alu_op_fexp2: |
| 2512 | case midgard_alu_op_flog2: |
| 2513 | max_bitsize = MAX2(max_bitsize, 32); |
| 2514 | break; |
| 2515 | |
| 2516 | default: |
| 2517 | break; |
| 2518 | } |
| 2519 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 2520 | /* High implies computing at a higher bitsize, e.g umul_high of 32-bit |
| 2521 | * requires computing at 64-bit */ |
| 2522 | if (midgard_is_integer_out_op(ins->op) && ins->outmod == midgard_outmod_int_high) { |
| 2523 | max_bitsize *= 2; |
| 2524 | assert(max_bitsize <= 64); |
| 2525 | } |
| 2526 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2527 | return max_bitsize; |
| 2528 | } |
| 2529 | |
| 2530 | midgard_reg_mode |
| 2531 | reg_mode_for_bitsize(unsigned bitsize) |
| 2532 | { |
| 2533 | switch (bitsize) { |
| 2534 | /* use 16 pipe for 8 since we don't support vec16 yet */ |
| 2535 | case 8: |
| 2536 | case 16: |
| 2537 | return midgard_reg_mode_16; |
| 2538 | case 32: |
| 2539 | return midgard_reg_mode_32; |
| 2540 | case 64: |
| 2541 | return midgard_reg_mode_64; |
| 2542 | default: |
| 2543 | unreachable("invalid bit size"); |
| 2544 | } |
| 2545 | } |
| 2546 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2547 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2548 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2549 | * constants can be demoted to inline constants, for space savings and |
| 2550 | * sometimes a performance boost */ |
| 2551 | |
| 2552 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2553 | embedded_to_inline_constant(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2554 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2555 | mir_foreach_instr_in_block(block, ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2556 | if (!ins->has_constants) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2557 | if (ins->has_inline_constant) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2558 | |
| 2559 | /* Blend constants must not be inlined by definition */ |
| 2560 | if (ins->has_blend_constant) continue; |
| 2561 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2562 | unsigned max_bitsize = max_bitsize_for_alu(ins); |
| 2563 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2564 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2565 | bool is_16 = max_bitsize == 16; |
| 2566 | bool is_32 = max_bitsize == 32; |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2567 | |
| 2568 | if (!(is_16 || is_32)) |
| 2569 | continue; |
| 2570 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2571 | /* src1 cannot be an inline constant due to encoding |
| 2572 | * restrictions. So, if possible we try to flip the arguments |
| 2573 | * in that case */ |
| 2574 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2575 | int op = ins->op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2576 | |
Alyssa Rosenzweig | ba9f3d1 | 2020-04-30 13:11:52 -0400 | [diff] [blame] | 2577 | if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT) && |
| 2578 | alu_opcode_props[op].props & OP_COMMUTES) { |
| 2579 | mir_flip(ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2580 | } |
| 2581 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2582 | if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2583 | /* Component is from the swizzle. Take a nonzero component */ |
| 2584 | assert(ins->mask); |
| 2585 | unsigned first_comp = ffs(ins->mask) - 1; |
| 2586 | unsigned component = ins->swizzle[1][first_comp]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2587 | |
| 2588 | /* Scale constant appropriately, if we can legally */ |
Icecream95 | d97aaad | 2020-06-05 20:17:27 +1200 | [diff] [blame] | 2589 | int16_t scaled_constant = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2590 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2591 | if (is_16) { |
| 2592 | scaled_constant = ins->constants.u16[component]; |
| 2593 | } else if (midgard_is_integer_op(op)) { |
| 2594 | scaled_constant = ins->constants.u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2595 | |
| 2596 | /* Constant overflow after resize */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2597 | if (scaled_constant != ins->constants.u32[component]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2598 | continue; |
| 2599 | } else { |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2600 | float original = ins->constants.f32[component]; |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2601 | scaled_constant = _mesa_float_to_half(original); |
| 2602 | |
| 2603 | /* Check for loss of precision. If this is |
| 2604 | * mediump, we don't care, but for a highp |
| 2605 | * shader, we need to pay attention. NIR |
| 2606 | * doesn't yet tell us which mode we're in! |
| 2607 | * Practically this prevents most constants |
| 2608 | * from being inlined, sadly. */ |
| 2609 | |
| 2610 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2611 | |
| 2612 | if (fp32 != original) |
| 2613 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2614 | } |
| 2615 | |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 2616 | /* Should've been const folded */ |
| 2617 | if (ins->src_abs[1] || ins->src_neg[1]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2618 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2619 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2620 | /* Make sure that the constant is not itself a vector |
| 2621 | * by checking if all accessed values are the same. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2622 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2623 | const midgard_constants *cons = &ins->constants; |
| 2624 | uint32_t value = is_16 ? cons->u16[component] : cons->u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2625 | |
| 2626 | bool is_vector = false; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2627 | unsigned mask = effective_writemask(ins->op, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2628 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2629 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2630 | /* We only care if this component is actually used */ |
| 2631 | if (!(mask & (1 << c))) |
| 2632 | continue; |
| 2633 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2634 | uint32_t test = is_16 ? |
| 2635 | cons->u16[ins->swizzle[1][c]] : |
| 2636 | cons->u32[ins->swizzle[1][c]]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2637 | |
| 2638 | if (test != value) { |
| 2639 | is_vector = true; |
| 2640 | break; |
| 2641 | } |
| 2642 | } |
| 2643 | |
| 2644 | if (is_vector) |
| 2645 | continue; |
| 2646 | |
| 2647 | /* Get rid of the embedded constant */ |
| 2648 | ins->has_constants = false; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2649 | ins->src[1] = ~0; |
| 2650 | ins->has_inline_constant = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2651 | ins->inline_constant = scaled_constant; |
| 2652 | } |
| 2653 | } |
| 2654 | } |
| 2655 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2656 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2657 | * per block is legal semantically */ |
| 2658 | |
| 2659 | static void |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 2660 | midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2661 | { |
| 2662 | bool branched = false; |
| 2663 | |
| 2664 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2665 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2666 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2667 | if (branched) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2668 | mir_remove_instruction(ins); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2669 | |
| 2670 | branched = true; |
| 2671 | } |
| 2672 | } |
| 2673 | |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2674 | /* We want to force the invert on AND/OR to the second slot to legalize into |
| 2675 | * iandnot/iornot. The relevant patterns are for AND (and OR respectively) |
| 2676 | * |
| 2677 | * ~a & #b = ~a & ~(#~b) |
| 2678 | * ~a & b = b & ~a |
| 2679 | */ |
| 2680 | |
| 2681 | static void |
| 2682 | midgard_legalize_invert(compiler_context *ctx, midgard_block *block) |
| 2683 | { |
| 2684 | mir_foreach_instr_in_block(block, ins) { |
| 2685 | if (ins->type != TAG_ALU_4) continue; |
| 2686 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2687 | if (ins->op != midgard_alu_op_iand && |
| 2688 | ins->op != midgard_alu_op_ior) continue; |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2689 | |
| 2690 | if (ins->src_invert[1] || !ins->src_invert[0]) continue; |
| 2691 | |
| 2692 | if (ins->has_inline_constant) { |
| 2693 | /* ~(#~a) = ~(~#a) = a, so valid, and forces both |
| 2694 | * inverts on */ |
| 2695 | ins->inline_constant = ~ins->inline_constant; |
| 2696 | ins->src_invert[1] = true; |
| 2697 | } else { |
| 2698 | /* Flip to the right invert order. Note |
| 2699 | * has_inline_constant false by assumption on the |
| 2700 | * branch, so flipping makes sense. */ |
| 2701 | mir_flip(ins); |
| 2702 | } |
| 2703 | } |
| 2704 | } |
| 2705 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2706 | static unsigned |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2707 | emit_fragment_epilogue(compiler_context *ctx, unsigned rt) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2708 | { |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2709 | /* Loop to ourselves */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2710 | midgard_instruction *br = ctx->writeout_branch[rt]; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2711 | struct midgard_instruction ins = v_branch(false, false); |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 2712 | ins.writeout = br->writeout; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2713 | ins.branch.target_block = ctx->block_count - 1; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2714 | ins.constants.u32[0] = br->constants.u32[0]; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 2715 | memcpy(&ins.src_types, &br->src_types, sizeof(ins.src_types)); |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2716 | emit_mir_instruction(ctx, ins); |
| 2717 | |
Alyssa Rosenzweig | 3448b26 | 2019-12-03 10:37:01 -0500 | [diff] [blame] | 2718 | ctx->current_block->epilogue = true; |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2719 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2720 | return ins.branch.target_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2721 | } |
| 2722 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2723 | static midgard_block * |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2724 | emit_block_init(compiler_context *ctx) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2725 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2726 | midgard_block *this_block = ctx->after_block; |
| 2727 | ctx->after_block = NULL; |
| 2728 | |
| 2729 | if (!this_block) |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2730 | this_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2731 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2732 | list_addtail(&this_block->base.link, &ctx->blocks); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2733 | |
Alyssa Rosenzweig | c5dd1d5 | 2020-03-11 08:22:08 -0400 | [diff] [blame] | 2734 | this_block->scheduled = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2735 | ++ctx->block_count; |
| 2736 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2737 | /* Set up current block */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2738 | list_inithead(&this_block->base.instructions); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2739 | ctx->current_block = this_block; |
| 2740 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2741 | return this_block; |
| 2742 | } |
| 2743 | |
| 2744 | static midgard_block * |
| 2745 | emit_block(compiler_context *ctx, nir_block *block) |
| 2746 | { |
| 2747 | midgard_block *this_block = emit_block_init(ctx); |
| 2748 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2749 | nir_foreach_instr(instr, block) { |
| 2750 | emit_instr(ctx, instr); |
| 2751 | ++ctx->instruction_count; |
| 2752 | } |
| 2753 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2754 | return this_block; |
| 2755 | } |
| 2756 | |
| 2757 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2758 | |
| 2759 | static void |
| 2760 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2761 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2762 | midgard_block *before_block = ctx->current_block; |
| 2763 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2764 | /* Speculatively emit the branch, but we can't fill it in until later */ |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2765 | bool inv = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2766 | EMIT(branch, true, true); |
| 2767 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2768 | then_branch->src[0] = mir_get_branch_cond(&nif->condition, &inv); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 2769 | then_branch->src_types[0] = nir_type_uint32; |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2770 | then_branch->branch.invert_conditional = !inv; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2771 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2772 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2773 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2774 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2775 | |
| 2776 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2777 | EMIT(branch, false, false); |
| 2778 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2779 | |
| 2780 | /* Emit second block, and check if it's empty */ |
| 2781 | |
| 2782 | int else_idx = ctx->block_count; |
| 2783 | int count_in = ctx->instruction_count; |
| 2784 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2785 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2786 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2787 | |
| 2788 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2789 | |
| 2790 | assert(then_block); |
| 2791 | assert(else_block); |
| 2792 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2793 | if (ctx->instruction_count == count_in) { |
| 2794 | /* The else block is empty, so don't emit an exit jump */ |
| 2795 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2796 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2797 | } else { |
| 2798 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2799 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2800 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2801 | |
| 2802 | /* Wire up the successors */ |
| 2803 | |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2804 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2805 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2806 | pan_block_add_successor(&before_block->base, &then_block->base); |
| 2807 | pan_block_add_successor(&before_block->base, &else_block->base); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2808 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2809 | pan_block_add_successor(&end_then_block->base, &ctx->after_block->base); |
| 2810 | pan_block_add_successor(&end_else_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2811 | } |
| 2812 | |
| 2813 | static void |
| 2814 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2815 | { |
| 2816 | /* Remember where we are */ |
| 2817 | midgard_block *start_block = ctx->current_block; |
| 2818 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2819 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2820 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2821 | |
| 2822 | /* Get index from before the body so we can loop back later */ |
| 2823 | int start_idx = ctx->block_count; |
| 2824 | |
| 2825 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2826 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2827 | |
| 2828 | /* Branch back to loop back */ |
| 2829 | struct midgard_instruction br_back = v_branch(false, false); |
| 2830 | br_back.branch.target_block = start_idx; |
| 2831 | emit_mir_instruction(ctx, br_back); |
| 2832 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2833 | /* Mark down that branch in the graph. */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2834 | pan_block_add_successor(&start_block->base, &loop_block->base); |
| 2835 | pan_block_add_successor(&ctx->current_block->base, &loop_block->base); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2836 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2837 | /* Find the index of the block about to follow us (note: we don't add |
| 2838 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2839 | int break_block_idx = ctx->block_count; |
| 2840 | |
| 2841 | /* Fix up the break statements we emitted to point to the right place, |
| 2842 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2843 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2844 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2845 | mir_foreach_block_from(ctx, start_block, _block) { |
| 2846 | mir_foreach_instr_in_block(((midgard_block *) _block), ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2847 | if (ins->type != TAG_ALU_4) continue; |
| 2848 | if (!ins->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2849 | |
| 2850 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2851 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2852 | |
| 2853 | /* It's a break! Check if it's our break */ |
| 2854 | if (ins->branch.target_break != loop_idx) continue; |
| 2855 | |
| 2856 | /* Okay, cool, we're breaking out of this loop. |
| 2857 | * Rewrite from a break to a goto */ |
| 2858 | |
| 2859 | ins->branch.target_type = TARGET_GOTO; |
| 2860 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2861 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2862 | pan_block_add_successor(_block, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2863 | } |
| 2864 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2865 | |
| 2866 | /* Now that we've finished emitting the loop, free up the depth again |
| 2867 | * so we play nice with recursion amid nested loops */ |
| 2868 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2869 | |
| 2870 | /* Dump loop stats */ |
| 2871 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2872 | } |
| 2873 | |
| 2874 | static midgard_block * |
| 2875 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2876 | { |
| 2877 | midgard_block *start_block = NULL; |
| 2878 | |
| 2879 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2880 | switch (node->type) { |
| 2881 | case nir_cf_node_block: { |
| 2882 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2883 | |
| 2884 | if (!start_block) |
| 2885 | start_block = block; |
| 2886 | |
| 2887 | break; |
| 2888 | } |
| 2889 | |
| 2890 | case nir_cf_node_if: |
| 2891 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2892 | break; |
| 2893 | |
| 2894 | case nir_cf_node_loop: |
| 2895 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2896 | break; |
| 2897 | |
| 2898 | case nir_cf_node_function: |
| 2899 | assert(0); |
| 2900 | break; |
| 2901 | } |
| 2902 | } |
| 2903 | |
| 2904 | return start_block; |
| 2905 | } |
| 2906 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2907 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2908 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2909 | * until we find one that 'works' */ |
| 2910 | |
Italo Nicola | 8150c1d | 2020-07-29 20:14:55 +0000 | [diff] [blame] | 2911 | unsigned |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2912 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2913 | { |
| 2914 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2915 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2916 | mir_foreach_block_from(ctx, initial_block, _v) { |
| 2917 | midgard_block *v = (midgard_block *) _v; |
Alyssa Rosenzweig | 45ac8ea | 2019-11-04 10:32:49 -0500 | [diff] [blame] | 2918 | if (v->quadword_count) { |
| 2919 | midgard_bundle *initial_bundle = |
| 2920 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2921 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2922 | return initial_bundle->tag; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2923 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2924 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2925 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2926 | /* Default to a tag 1 which will break from the shader, in case we jump |
| 2927 | * to the exit block (i.e. `return` in a compute shader) */ |
| 2928 | |
| 2929 | return 1; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2930 | } |
| 2931 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2932 | /* For each fragment writeout instruction, generate a writeout loop to |
| 2933 | * associate with it */ |
| 2934 | |
| 2935 | static void |
| 2936 | mir_add_writeout_loops(compiler_context *ctx) |
| 2937 | { |
| 2938 | for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) { |
| 2939 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 2940 | if (!br) continue; |
| 2941 | |
| 2942 | unsigned popped = br->branch.target_block; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2943 | pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base), &ctx->current_block->base); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2944 | br->branch.target_block = emit_fragment_epilogue(ctx, rt); |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2945 | br->branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2946 | |
| 2947 | /* If we have more RTs, we'll need to restore back after our |
| 2948 | * loop terminates */ |
| 2949 | |
| 2950 | if ((rt + 1) < ARRAY_SIZE(ctx->writeout_branch) && ctx->writeout_branch[rt + 1]) { |
| 2951 | midgard_instruction uncond = v_branch(false, false); |
| 2952 | uncond.branch.target_block = popped; |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2953 | uncond.branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2954 | emit_mir_instruction(ctx, uncond); |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2955 | pan_block_add_successor(&ctx->current_block->base, &(mir_get_block(ctx, popped)->base)); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2956 | schedule_barrier(ctx); |
| 2957 | } else { |
| 2958 | /* We're last, so we can terminate here */ |
| 2959 | br->last_writeout = true; |
| 2960 | } |
| 2961 | } |
| 2962 | } |
| 2963 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2964 | int |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 2965 | midgard_compile_shader_nir(nir_shader *nir, panfrost_program *program, bool is_blend, unsigned blend_rt, unsigned gpu_id, bool shaderdb, bool silent) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2966 | { |
| 2967 | struct util_dynarray *compiled = &program->compiled; |
| 2968 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2969 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2970 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2971 | /* TODO: Bound against what? */ |
| 2972 | compiler_context *ctx = rzalloc(NULL, compiler_context); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2973 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2974 | ctx->nir = nir; |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2975 | ctx->stage = nir->info.stage; |
| 2976 | ctx->is_blend = is_blend; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2977 | ctx->blend_rt = MIDGARD_COLOR_RT0 + blend_rt; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 2978 | ctx->blend_input = ~0; |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 2979 | ctx->blend_src1 = ~0; |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 2980 | ctx->quirks = midgard_get_quirks(gpu_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2981 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2982 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2983 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2984 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2985 | ctx->uniform_cutoff = 8; |
| 2986 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2987 | /* Initialize at a global (not block) level hash tables */ |
| 2988 | |
| 2989 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2990 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2991 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2992 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2993 | * messed with our I/O quite a bit already) */ |
| 2994 | |
| 2995 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2996 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2997 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2998 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | 2023716 | 2019-08-26 12:14:11 -0700 | [diff] [blame] | 2999 | NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 3000 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3001 | |
| 3002 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3003 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 3004 | NIR_PASS_V(nir, nir_split_var_copies); |
| 3005 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3006 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 3007 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3008 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 3009 | |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3010 | unsigned pan_quirks = panfrost_get_quirks(gpu_id); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3011 | NIR_PASS_V(nir, pan_lower_framebuffer, |
Icecream95 | 2fa60b7 | 2020-07-06 19:34:49 +1200 | [diff] [blame] | 3012 | program->rt_formats, is_blend, pan_quirks); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3013 | |
Jason Ekstrand | b019b22 | 2020-06-10 17:54:25 -0500 | [diff] [blame] | 3014 | NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out, |
| 3015 | glsl_type_size, 0); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 3016 | NIR_PASS_V(nir, nir_lower_ssbo); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 3017 | NIR_PASS_V(nir, midgard_nir_lower_zs_store); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3018 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3019 | /* Optimisation passes */ |
| 3020 | |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 3021 | optimise_nir(nir, ctx->quirks, is_blend); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3022 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 3023 | NIR_PASS_V(nir, midgard_nir_reorder_writeout); |
| 3024 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3025 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !silent) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3026 | nir_print_shader(nir, stdout); |
| 3027 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3028 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 3029 | /* Assign sysvals and counts, now that we're sure |
| 3030 | * (post-optimisation) */ |
| 3031 | |
Alyssa Rosenzweig | 680fb05 | 2020-08-18 08:31:42 -0400 | [diff] [blame] | 3032 | panfrost_nir_assign_sysvals(&ctx->sysvals, ctx, nir); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 3033 | program->sysval_count = ctx->sysvals.sysval_count; |
| 3034 | memcpy(program->sysvals, ctx->sysvals.sysvals, sizeof(ctx->sysvals.sysvals[0]) * ctx->sysvals.sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3035 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3036 | nir_foreach_function(func, nir) { |
| 3037 | if (!func->impl) |
| 3038 | continue; |
| 3039 | |
| 3040 | list_inithead(&ctx->blocks); |
| 3041 | ctx->block_count = 0; |
| 3042 | ctx->func = func; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3043 | ctx->already_emitted = calloc(BITSET_WORDS(func->impl->ssa_alloc), sizeof(BITSET_WORD)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3044 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 3045 | if (nir->info.outputs_read && !is_blend) { |
| 3046 | emit_block_init(ctx); |
| 3047 | |
| 3048 | struct midgard_instruction wait = v_branch(false, false); |
| 3049 | wait.branch.target_type = TARGET_TILEBUF_WAIT; |
| 3050 | |
| 3051 | emit_mir_instruction(ctx, wait); |
| 3052 | |
| 3053 | ++ctx->instruction_count; |
| 3054 | } |
| 3055 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3056 | emit_cf_list(ctx, &func->impl->body); |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3057 | free(ctx->already_emitted); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3058 | break; /* TODO: Multi-function shaders */ |
| 3059 | } |
| 3060 | |
| 3061 | util_dynarray_init(compiled, NULL); |
| 3062 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3063 | /* Per-block lowering before opts */ |
| 3064 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3065 | mir_foreach_block(ctx, _block) { |
| 3066 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3067 | inline_alu_constants(ctx, block); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3068 | embedded_to_inline_constant(ctx, block); |
| 3069 | } |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3070 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3071 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3072 | bool progress = false; |
| 3073 | |
| 3074 | do { |
| 3075 | progress = false; |
Alyssa Rosenzweig | fc06b8b | 2020-05-06 17:34:09 -0400 | [diff] [blame] | 3076 | progress |= midgard_opt_dead_code_eliminate(ctx); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3077 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3078 | mir_foreach_block(ctx, _block) { |
| 3079 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3080 | progress |= midgard_opt_copy_prop(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 3081 | progress |= midgard_opt_combine_projection(ctx, block); |
| 3082 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3083 | } |
| 3084 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3085 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3086 | mir_foreach_block(ctx, _block) { |
| 3087 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 3088 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 3089 | midgard_legalize_invert(ctx, block); |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 3090 | midgard_cull_dead_branch(ctx, block); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 3091 | } |
| 3092 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3093 | if (ctx->stage == MESA_SHADER_FRAGMENT) |
| 3094 | mir_add_writeout_loops(ctx); |
| 3095 | |
Alyssa Rosenzweig | 9a7f0e2 | 2020-05-12 13:26:32 -0400 | [diff] [blame] | 3096 | /* Analyze now that the code is known but before scheduling creates |
| 3097 | * pipeline registers which are harder to track */ |
| 3098 | mir_analyze_helper_terminate(ctx); |
| 3099 | mir_analyze_helper_requirements(ctx); |
| 3100 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3101 | /* Schedule! */ |
Robert Foss | 62adb65 | 2020-01-15 01:14:16 +0100 | [diff] [blame] | 3102 | midgard_schedule_program(ctx); |
Alyssa Rosenzweig | 9dc3b18 | 2019-12-06 09:32:38 -0500 | [diff] [blame] | 3103 | mir_ra(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3104 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3105 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 3106 | * sequence. Save instruction boundaries such that lookahead tags can |
| 3107 | * be assigned easily */ |
| 3108 | |
| 3109 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 3110 | |
| 3111 | int bundle_count = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3112 | mir_foreach_block(ctx, _block) { |
| 3113 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3114 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 3115 | } |
| 3116 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 3117 | int bundle_idx = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3118 | mir_foreach_block(ctx, _block) { |
| 3119 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3120 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 3121 | source_order_bundles[bundle_idx++] = bundle; |
| 3122 | } |
| 3123 | } |
| 3124 | |
| 3125 | int current_bundle = 0; |
| 3126 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3127 | /* Midgard prefetches instruction types, so during emission we |
| 3128 | * need to lookahead. Unless this is the last instruction, in |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3129 | * which we return 1. */ |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3130 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3131 | mir_foreach_block(ctx, _block) { |
| 3132 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 3133 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3134 | int lookahead = 1; |
| 3135 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3136 | if (!bundle->last_writeout && (current_bundle + 1 < bundle_count)) |
| 3137 | lookahead = source_order_bundles[current_bundle + 1]->tag; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3138 | |
Alyssa Rosenzweig | 30a393f | 2020-05-21 19:14:23 -0400 | [diff] [blame] | 3139 | emit_binary_bundle(ctx, block, bundle, compiled, lookahead); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3140 | ++current_bundle; |
| 3141 | } |
| 3142 | |
| 3143 | /* TODO: Free deeper */ |
| 3144 | //util_dynarray_fini(&block->instructions); |
| 3145 | } |
| 3146 | |
| 3147 | free(source_order_bundles); |
| 3148 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 3149 | /* Report the very first tag executed */ |
| 3150 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3151 | |
| 3152 | /* Deal with off-by-one related to the fencepost problem */ |
| 3153 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3154 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 3155 | |
| 3156 | program->blend_patch_offset = ctx->blend_constant_offset; |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 3157 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3158 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3159 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !silent) |
Icecream95 | 968f36d | 2020-01-23 09:42:12 +1300 | [diff] [blame] | 3160 | disassemble_midgard(stdout, program->compiled.data, program->compiled.size, gpu_id, ctx->stage); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3161 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3162 | if ((midgard_debug & MIDGARD_DBG_SHADERDB || shaderdb) && !silent) { |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3163 | unsigned nr_bundles = 0, nr_ins = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3164 | |
| 3165 | /* Count instructions and bundles */ |
| 3166 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3167 | mir_foreach_block(ctx, _block) { |
| 3168 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3169 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3170 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 3171 | |
Alyssa Rosenzweig | 67909c8 | 2019-08-30 13:08:16 -0700 | [diff] [blame] | 3172 | mir_foreach_bundle_in_block(block, bun) |
| 3173 | nr_ins += bun->instruction_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3174 | } |
| 3175 | |
| 3176 | /* Calculate thread count. There are certain cutoffs by |
| 3177 | * register count for thread count */ |
| 3178 | |
| 3179 | unsigned nr_registers = program->work_register_count; |
| 3180 | |
| 3181 | unsigned nr_threads = |
| 3182 | (nr_registers <= 4) ? 4 : |
| 3183 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3184 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3185 | |
| 3186 | /* Dump stats */ |
| 3187 | |
| 3188 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3189 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3190 | "%u registers, %u threads, %u loops, " |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 3191 | "%u:%u spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3192 | SHADER_DB_COUNT++, |
Alyssa Rosenzweig | 014d2e4 | 2020-05-25 13:19:43 -0400 | [diff] [blame] | 3193 | ctx->is_blend ? "PAN_SHADER_BLEND" : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3194 | gl_shader_stage_name(ctx->stage), |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3195 | nr_ins, nr_bundles, ctx->quadword_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3196 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3197 | ctx->loop_count, |
| 3198 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3199 | } |
| 3200 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 3201 | ralloc_free(ctx); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3202 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3203 | return 0; |
| 3204 | } |