Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 36 | #include "compiler/nir/nir_builder.h" |
| 37 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 38 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 39 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 40 | #include "util/u_dynarray.h" |
| 41 | #include "util/list.h" |
| 42 | #include "main/mtypes.h" |
| 43 | |
| 44 | #include "midgard.h" |
| 45 | #include "midgard_nir.h" |
| 46 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 47 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 48 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 49 | #include "compiler.h" |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 50 | #include "midgard_quirks.h" |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 51 | #include "panfrost-quirks.h" |
| 52 | #include "panfrost/util/pan_lower_framebuffer.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 53 | |
| 54 | #include "disassemble.h" |
| 55 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 56 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 57 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 58 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 59 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 60 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 61 | }; |
| 62 | |
| 63 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 64 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 65 | unsigned SHADER_DB_COUNT = 0; |
| 66 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 67 | int midgard_debug = 0; |
| 68 | |
| 69 | #define DBG(fmt, ...) \ |
| 70 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 71 | fprintf(stderr, "%s:%d: "fmt, \ |
| 72 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 73 | static midgard_block * |
| 74 | create_empty_block(compiler_context *ctx) |
| 75 | { |
| 76 | midgard_block *blk = rzalloc(ctx, midgard_block); |
| 77 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 78 | blk->base.predecessors = _mesa_set_create(blk, |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 79 | _mesa_hash_pointer, |
| 80 | _mesa_key_pointer_equal); |
| 81 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 82 | blk->base.name = ctx->block_source_count++; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 83 | |
| 84 | return blk; |
| 85 | } |
| 86 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 87 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 88 | schedule_barrier(compiler_context *ctx) |
| 89 | { |
| 90 | midgard_block *temp = ctx->after_block; |
| 91 | ctx->after_block = create_empty_block(ctx); |
| 92 | ctx->block_count++; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 93 | list_addtail(&ctx->after_block->base.link, &ctx->blocks); |
| 94 | list_inithead(&ctx->after_block->base.instructions); |
| 95 | pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 96 | ctx->current_block = ctx->after_block; |
| 97 | ctx->after_block = temp; |
| 98 | } |
| 99 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 100 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 101 | * driver seems to do it that way */ |
| 102 | |
| 103 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 104 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 105 | #define M_LOAD_STORE(name, store, T) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 106 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 107 | midgard_instruction i = { \ |
| 108 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 109 | .mask = 0xF, \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 110 | .dest = ~0, \ |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 111 | .src = { ~0, ~0, ~0, ~0 }, \ |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 112 | .swizzle = SWIZZLE_IDENTITY_4, \ |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 113 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 114 | .load_store = { \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 115 | .address = address \ |
| 116 | } \ |
| 117 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 118 | \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 119 | if (store) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 120 | i.src[0] = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 121 | i.src_types[0] = T; \ |
Alyssa Rosenzweig | 9915bb2 | 2020-05-07 10:12:38 -0400 | [diff] [blame] | 122 | i.dest_type = T; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 123 | } else { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 124 | i.dest = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 125 | i.dest_type = T; \ |
| 126 | } \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 127 | return i; \ |
| 128 | } |
| 129 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 130 | #define M_LOAD(name, T) M_LOAD_STORE(name, false, T) |
| 131 | #define M_STORE(name, T) M_LOAD_STORE(name, true, T) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 132 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 133 | M_LOAD(ld_attr_32, nir_type_uint32); |
| 134 | M_LOAD(ld_vary_32, nir_type_uint32); |
| 135 | M_LOAD(ld_ubo_int4, nir_type_uint32); |
| 136 | M_LOAD(ld_int4, nir_type_uint32); |
| 137 | M_STORE(st_int4, nir_type_uint32); |
| 138 | M_LOAD(ld_color_buffer_32u, nir_type_uint32); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 139 | M_LOAD(ld_color_buffer_as_fp16, nir_type_float16); |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 140 | M_LOAD(ld_color_buffer_as_fp32, nir_type_float32); |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 141 | M_STORE(st_vary_32, nir_type_uint32); |
| 142 | M_LOAD(ld_cubemap_coords, nir_type_uint32); |
| 143 | M_LOAD(ld_compute_id, nir_type_uint32); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 144 | |
| 145 | static midgard_instruction |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 146 | v_branch(bool conditional, bool invert) |
| 147 | { |
| 148 | midgard_instruction ins = { |
| 149 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 150 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 151 | .compact_branch = true, |
| 152 | .branch = { |
| 153 | .conditional = conditional, |
| 154 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 155 | }, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 156 | .dest = ~0, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 157 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 158 | }; |
| 159 | |
| 160 | return ins; |
| 161 | } |
| 162 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 163 | static void |
| 164 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 165 | { |
| 166 | ins->has_constants = true; |
| 167 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 171 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | { |
| 173 | return glsl_count_attribute_slots(type, false); |
| 174 | } |
| 175 | |
| 176 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
| 177 | static void |
| 178 | midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu) |
| 179 | { |
| 180 | if (alu->op != nir_op_fdot2) |
| 181 | return; |
| 182 | |
| 183 | b->cursor = nir_before_instr(&alu->instr); |
| 184 | |
| 185 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 186 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 187 | |
| 188 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 189 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 190 | nir_ssa_def *sum = nir_fadd(b, |
| 191 | nir_channel(b, product, 0), |
| 192 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 193 | |
| 194 | /* Replace the fdot2 with this sum */ |
| 195 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
| 196 | } |
| 197 | |
| 198 | static bool |
| 199 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 200 | { |
| 201 | bool progress = false; |
| 202 | |
| 203 | nir_foreach_function(function, shader) { |
| 204 | if (!function->impl) continue; |
| 205 | |
| 206 | nir_builder _b; |
| 207 | nir_builder *b = &_b; |
| 208 | nir_builder_init(b, function->impl); |
| 209 | |
| 210 | nir_foreach_block(block, function->impl) { |
| 211 | nir_foreach_instr_safe(instr, block) { |
| 212 | if (instr->type != nir_instr_type_alu) continue; |
| 213 | |
| 214 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 215 | midgard_nir_lower_fdot2_body(b, alu); |
| 216 | |
| 217 | progress |= true; |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 222 | |
| 223 | } |
| 224 | |
| 225 | return progress; |
| 226 | } |
| 227 | |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 228 | static const nir_variable * |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 229 | search_var(nir_shader *nir, nir_variable_mode mode, unsigned driver_loc) |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 230 | { |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 231 | nir_foreach_variable_with_modes(var, nir, mode) { |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 232 | if (var->data.driver_location == driver_loc) |
| 233 | return var; |
| 234 | } |
| 235 | |
| 236 | return NULL; |
| 237 | } |
| 238 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 239 | /* Midgard can write all of color, depth and stencil in a single writeout |
| 240 | * operation, so we merge depth/stencil stores with color stores. |
| 241 | * If there are no color stores, we add a write to the "depth RT". |
| 242 | */ |
| 243 | static bool |
| 244 | midgard_nir_lower_zs_store(nir_shader *nir) |
| 245 | { |
| 246 | if (nir->info.stage != MESA_SHADER_FRAGMENT) |
| 247 | return false; |
| 248 | |
| 249 | nir_variable *z_var = NULL, *s_var = NULL; |
| 250 | |
Jason Ekstrand | 2956d53 | 2020-07-18 18:24:25 -0500 | [diff] [blame] | 251 | nir_foreach_shader_out_variable(var, nir) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 252 | if (var->data.location == FRAG_RESULT_DEPTH) |
| 253 | z_var = var; |
| 254 | else if (var->data.location == FRAG_RESULT_STENCIL) |
| 255 | s_var = var; |
| 256 | } |
| 257 | |
| 258 | if (!z_var && !s_var) |
| 259 | return false; |
| 260 | |
| 261 | bool progress = false; |
| 262 | |
| 263 | nir_foreach_function(function, nir) { |
| 264 | if (!function->impl) continue; |
| 265 | |
| 266 | nir_intrinsic_instr *z_store = NULL, *s_store = NULL; |
| 267 | |
| 268 | nir_foreach_block(block, function->impl) { |
| 269 | nir_foreach_instr_safe(instr, block) { |
| 270 | if (instr->type != nir_instr_type_intrinsic) |
| 271 | continue; |
| 272 | |
| 273 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 274 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 275 | continue; |
| 276 | |
| 277 | if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) { |
| 278 | assert(!z_store); |
| 279 | z_store = intr; |
| 280 | } |
| 281 | |
| 282 | if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) { |
| 283 | assert(!s_store); |
| 284 | s_store = intr; |
| 285 | } |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | if (!z_store && !s_store) continue; |
| 290 | |
| 291 | bool replaced = false; |
| 292 | |
| 293 | nir_foreach_block(block, function->impl) { |
| 294 | nir_foreach_instr_safe(instr, block) { |
| 295 | if (instr->type != nir_instr_type_intrinsic) |
| 296 | continue; |
| 297 | |
| 298 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 299 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 300 | continue; |
| 301 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 302 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 303 | assert(var); |
| 304 | |
| 305 | if (var->data.location != FRAG_RESULT_COLOR && |
| 306 | var->data.location < FRAG_RESULT_DATA0) |
| 307 | continue; |
| 308 | |
Icecream95 | 334dab0 | 2020-07-10 23:28:21 +1200 | [diff] [blame] | 309 | if (var->data.index) |
| 310 | continue; |
| 311 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 312 | assert(nir_src_is_const(intr->src[1]) && "no indirect outputs"); |
| 313 | |
| 314 | nir_builder b; |
| 315 | nir_builder_init(&b, function->impl); |
| 316 | |
| 317 | assert(!z_store || z_store->instr.block == instr->block); |
| 318 | assert(!s_store || s_store->instr.block == instr->block); |
| 319 | b.cursor = nir_after_block_before_jump(instr->block); |
| 320 | |
| 321 | nir_intrinsic_instr *combined_store; |
| 322 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 323 | |
| 324 | combined_store->num_components = intr->src[0].ssa->num_components; |
| 325 | |
| 326 | nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr)); |
| 327 | |
| 328 | unsigned writeout = PAN_WRITEOUT_C; |
| 329 | if (z_store) |
| 330 | writeout |= PAN_WRITEOUT_Z; |
| 331 | if (s_store) |
| 332 | writeout |= PAN_WRITEOUT_S; |
| 333 | |
| 334 | nir_intrinsic_set_component(combined_store, writeout); |
| 335 | |
| 336 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 337 | |
| 338 | struct nir_ssa_def *src[4] = { |
| 339 | intr->src[0].ssa, |
| 340 | intr->src[1].ssa, |
| 341 | z_store ? z_store->src[0].ssa : zero, |
| 342 | s_store ? s_store->src[0].ssa : zero, |
| 343 | }; |
| 344 | |
| 345 | for (int i = 0; i < 4; ++i) |
| 346 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 347 | |
| 348 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 349 | |
| 350 | nir_instr_remove(instr); |
| 351 | |
| 352 | replaced = true; |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | /* Insert a store to the depth RT (0xff) if needed */ |
| 357 | if (!replaced) { |
| 358 | nir_builder b; |
| 359 | nir_builder_init(&b, function->impl); |
| 360 | |
| 361 | nir_block *block = NULL; |
| 362 | if (z_store && s_store) |
| 363 | assert(z_store->instr.block == s_store->instr.block); |
| 364 | |
| 365 | if (z_store) |
| 366 | block = z_store->instr.block; |
| 367 | else |
| 368 | block = s_store->instr.block; |
| 369 | |
| 370 | b.cursor = nir_after_block_before_jump(block); |
| 371 | |
| 372 | nir_intrinsic_instr *combined_store; |
| 373 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 374 | |
| 375 | combined_store->num_components = 4; |
| 376 | |
Icecream95 | 18059f4 | 2020-07-08 16:00:51 +1200 | [diff] [blame] | 377 | unsigned base; |
| 378 | if (z_store) |
| 379 | base = nir_intrinsic_base(z_store); |
| 380 | else |
| 381 | base = nir_intrinsic_base(s_store); |
| 382 | nir_intrinsic_set_base(combined_store, base); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 383 | |
| 384 | unsigned writeout = 0; |
| 385 | if (z_store) |
| 386 | writeout |= PAN_WRITEOUT_Z; |
| 387 | if (s_store) |
| 388 | writeout |= PAN_WRITEOUT_S; |
| 389 | |
| 390 | nir_intrinsic_set_component(combined_store, writeout); |
| 391 | |
| 392 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 393 | |
| 394 | struct nir_ssa_def *src[4] = { |
| 395 | nir_imm_vec4(&b, 0, 0, 0, 0), |
| 396 | zero, |
| 397 | z_store ? z_store->src[0].ssa : zero, |
| 398 | s_store ? s_store->src[0].ssa : zero, |
| 399 | }; |
| 400 | |
| 401 | for (int i = 0; i < 4; ++i) |
| 402 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 403 | |
| 404 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 405 | } |
| 406 | |
| 407 | if (z_store) |
| 408 | nir_instr_remove(&z_store->instr); |
| 409 | |
| 410 | if (s_store) |
| 411 | nir_instr_remove(&s_store->instr); |
| 412 | |
| 413 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 414 | progress = true; |
| 415 | } |
| 416 | |
| 417 | return progress; |
| 418 | } |
| 419 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 420 | /* Real writeout stores, which break execution, need to be moved to after |
| 421 | * dual-source stores, which are just standard register writes. */ |
| 422 | static bool |
| 423 | midgard_nir_reorder_writeout(nir_shader *nir) |
| 424 | { |
| 425 | bool progress = false; |
| 426 | |
| 427 | nir_foreach_function(function, nir) { |
| 428 | if (!function->impl) continue; |
| 429 | |
| 430 | nir_foreach_block(block, function->impl) { |
| 431 | nir_instr *last_writeout = NULL; |
| 432 | |
| 433 | nir_foreach_instr_reverse_safe(instr, block) { |
| 434 | if (instr->type != nir_instr_type_intrinsic) |
| 435 | continue; |
| 436 | |
| 437 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 438 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 439 | continue; |
| 440 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 441 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 442 | |
| 443 | if (var->data.index) { |
| 444 | if (!last_writeout) |
| 445 | last_writeout = instr; |
| 446 | continue; |
| 447 | } |
| 448 | |
| 449 | if (!last_writeout) |
| 450 | continue; |
| 451 | |
| 452 | /* This is a real store, so move it to after dual-source stores */ |
| 453 | exec_node_remove(&instr->node); |
| 454 | exec_node_insert_after(&last_writeout->node, &instr->node); |
| 455 | |
| 456 | progress = true; |
| 457 | } |
| 458 | } |
| 459 | } |
| 460 | |
| 461 | return progress; |
| 462 | } |
| 463 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 464 | static bool |
| 465 | mdg_is_64(const nir_instr *instr, const void *_unused) |
| 466 | { |
| 467 | const nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 468 | |
| 469 | if (nir_dest_bit_size(alu->dest.dest) == 64) |
| 470 | return true; |
| 471 | |
| 472 | switch (alu->op) { |
| 473 | case nir_op_umul_high: |
| 474 | case nir_op_imul_high: |
| 475 | return true; |
| 476 | default: |
| 477 | return false; |
| 478 | } |
| 479 | } |
| 480 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 481 | /* Flushes undefined values to zero */ |
| 482 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 483 | static void |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 484 | optimise_nir(nir_shader *nir, unsigned quirks, bool is_blend) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 485 | { |
| 486 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 487 | unsigned lower_flrp = |
| 488 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 489 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 490 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 491 | |
| 492 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
Rhys Perry | 8b98d09 | 2019-02-05 15:56:24 +0000 | [diff] [blame] | 493 | NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 494 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 495 | nir_lower_tex_options lower_tex_options = { |
| 496 | .lower_txs_lod = true, |
Alyssa Rosenzweig | 4c43b35 | 2019-11-21 13:40:00 -0500 | [diff] [blame] | 497 | .lower_txp = ~0, |
| 498 | .lower_tex_without_implicit_lod = |
| 499 | (quirks & MIDGARD_EXPLICIT_LOD), |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 500 | .lower_tg4_broadcom_swizzle = true, |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 501 | |
| 502 | /* TODO: we have native gradient.. */ |
| 503 | .lower_txd = true, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 504 | }; |
| 505 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 506 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 507 | |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 508 | /* Must lower fdot2 after tex is lowered */ |
| 509 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
| 510 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 511 | /* T720 is broken. */ |
| 512 | |
| 513 | if (quirks & MIDGARD_BROKEN_LOD) |
| 514 | NIR_PASS_V(nir, midgard_nir_lod_errata); |
| 515 | |
Alyssa Rosenzweig | c495c6c | 2020-05-12 19:07:48 -0400 | [diff] [blame] | 516 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_early); |
| 517 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 518 | do { |
| 519 | progress = false; |
| 520 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 521 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 522 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 523 | |
| 524 | NIR_PASS(progress, nir, nir_copy_prop); |
Boris Brezillon | 440b0d6 | 2020-01-06 14:31:38 +0100 | [diff] [blame] | 525 | NIR_PASS(progress, nir, nir_opt_remove_phis); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 526 | NIR_PASS(progress, nir, nir_opt_dce); |
| 527 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 528 | NIR_PASS(progress, nir, nir_opt_cse); |
| 529 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 530 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 531 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 532 | |
| 533 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 534 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 535 | NIR_PASS(lower_flrp_progress, |
| 536 | nir, |
| 537 | nir_lower_flrp, |
| 538 | lower_flrp, |
Marek Olšák | ac55b1a | 2020-07-22 22:13:16 -0400 | [diff] [blame^] | 539 | false /* always_precise */); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 540 | if (lower_flrp_progress) { |
| 541 | NIR_PASS(progress, nir, |
| 542 | nir_opt_constant_folding); |
| 543 | progress = true; |
| 544 | } |
| 545 | |
| 546 | /* Nothing should rematerialize any flrps, so we only |
| 547 | * need to do this lowering once. |
| 548 | */ |
| 549 | lower_flrp = 0; |
| 550 | } |
| 551 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 552 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 553 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 554 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 555 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 556 | nir_var_shader_in | |
| 557 | nir_var_shader_out | |
| 558 | nir_var_function_temp); |
| 559 | |
Eric Anholt | f25e169 | 2020-08-27 12:49:13 -0700 | [diff] [blame] | 560 | NIR_PASS(progress, nir, nir_opt_vectorize, NULL, NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 561 | } while (progress); |
| 562 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 563 | NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_is_64, NULL); |
| 564 | |
Alyssa Rosenzweig | d838cb9 | 2020-06-16 13:07:02 -0400 | [diff] [blame] | 565 | /* Run after opts so it can hit more */ |
| 566 | if (!is_blend) |
| 567 | NIR_PASS(progress, nir, nir_fuse_io_16); |
| 568 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 569 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 570 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 571 | |
| 572 | do { |
| 573 | progress = false; |
| 574 | |
| 575 | NIR_PASS(progress, nir, nir_opt_dce); |
| 576 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 577 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 578 | NIR_PASS(progress, nir, nir_copy_prop); |
| 579 | } while (progress); |
| 580 | |
| 581 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 211dee4 | 2020-04-29 20:27:16 -0400 | [diff] [blame] | 582 | NIR_PASS(progress, nir, nir_opt_algebraic_distribute_src_mods); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 583 | |
| 584 | /* We implement booleans as 32-bit 0/~0 */ |
| 585 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 586 | |
| 587 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 588 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 589 | NIR_PASS(progress, nir, midgard_nir_cancel_inot); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 590 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 591 | NIR_PASS(progress, nir, nir_copy_prop); |
| 592 | NIR_PASS(progress, nir, nir_opt_dce); |
| 593 | |
| 594 | /* Take us out of SSA */ |
| 595 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 596 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 597 | |
| 598 | /* We are a vector architecture; write combine where possible */ |
| 599 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 600 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 601 | |
| 602 | NIR_PASS(progress, nir, nir_opt_dce); |
| 603 | } |
| 604 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 605 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 606 | |
| 607 | static void |
| 608 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 609 | { |
| 610 | nir_ssa_def def = instr->def; |
| 611 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 612 | midgard_constants *consts = rzalloc(NULL, midgard_constants); |
| 613 | |
| 614 | assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8); |
| 615 | |
| 616 | #define RAW_CONST_COPY(bits) \ |
| 617 | nir_const_value_to_array(consts->u##bits, instr->value, \ |
| 618 | instr->def.num_components, u##bits) |
| 619 | |
| 620 | switch (instr->def.bit_size) { |
| 621 | case 64: |
| 622 | RAW_CONST_COPY(64); |
| 623 | break; |
| 624 | case 32: |
| 625 | RAW_CONST_COPY(32); |
| 626 | break; |
| 627 | case 16: |
| 628 | RAW_CONST_COPY(16); |
| 629 | break; |
| 630 | case 8: |
| 631 | RAW_CONST_COPY(8); |
| 632 | break; |
| 633 | default: |
| 634 | unreachable("Invalid bit_size for load_const instruction\n"); |
| 635 | } |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 636 | |
| 637 | /* Shifted for SSA, +1 for off-by-one */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 638 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 639 | } |
| 640 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 641 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 642 | * explicitly emit a move with the constant source */ |
| 643 | |
| 644 | static void |
| 645 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 646 | { |
| 647 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 648 | |
| 649 | if (constant_value) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 650 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), to); |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 651 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 652 | emit_mir_instruction(ctx, ins); |
| 653 | } |
| 654 | } |
| 655 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 656 | static bool |
| 657 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 658 | { |
| 659 | unsigned comp = src->swizzle[0]; |
| 660 | |
| 661 | for (unsigned c = 1; c < nr_components; ++c) { |
| 662 | if (src->swizzle[c] != comp) |
| 663 | return true; |
| 664 | } |
| 665 | |
| 666 | return false; |
| 667 | } |
| 668 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 669 | #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \ |
| 670 | case nir_intrinsic_##nir: \ |
| 671 | emit_atomic(ctx, instr, is_shared, midgard_op_##op); \ |
| 672 | break; |
| 673 | |
| 674 | #define ATOMIC_CASE(ctx, instr, nir, op) \ |
| 675 | ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \ |
| 676 | ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false); |
| 677 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 678 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 679 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 680 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 681 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 682 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 683 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 684 | #define ALU_CASE_RTZ(nir, _op) \ |
| 685 | case nir_op_##nir: \ |
| 686 | op = midgard_alu_op_##_op; \ |
| 687 | roundmode = MIDGARD_RTZ; \ |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 688 | break; |
| 689 | |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 690 | #define ALU_CHECK_CMP(sext) \ |
| 691 | assert(src_bitsize == 16 || src_bitsize == 32); \ |
| 692 | assert(dst_bitsize == 16 || dst_bitsize == 32); \ |
| 693 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 694 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 695 | case nir_op_##nir: \ |
| 696 | op = midgard_alu_op_##_op; \ |
| 697 | broadcast_swizzle = count; \ |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 698 | ALU_CHECK_CMP(true); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 699 | break; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 700 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 701 | #define ALU_CASE_CMP(nir, _op, sext) \ |
| 702 | case nir_op_##nir: \ |
| 703 | op = midgard_alu_op_##_op; \ |
| 704 | ALU_CHECK_CMP(sext); \ |
| 705 | break; |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 706 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 707 | /* Compare mir_lower_invert */ |
| 708 | static bool |
| 709 | nir_accepts_inot(nir_op op, unsigned src) |
| 710 | { |
| 711 | switch (op) { |
| 712 | case nir_op_ior: |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 713 | case nir_op_iand: /* TODO: b2f16 */ |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 714 | case nir_op_ixor: |
| 715 | return true; |
| 716 | case nir_op_b32csel: |
| 717 | /* Only the condition */ |
| 718 | return (src == 0); |
| 719 | default: |
| 720 | return false; |
| 721 | } |
| 722 | } |
| 723 | |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 724 | static bool |
| 725 | mir_accept_dest_mod(compiler_context *ctx, nir_dest **dest, nir_op op) |
| 726 | { |
| 727 | if (pan_has_dest_mod(dest, op)) { |
| 728 | assert((*dest)->is_ssa); |
| 729 | BITSET_SET(ctx->already_emitted, (*dest)->ssa.index); |
| 730 | return true; |
| 731 | } |
| 732 | |
| 733 | return false; |
| 734 | } |
| 735 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 736 | /* Look for floating point mods. We have the mods fsat, fsat_signed, |
| 737 | * and fpos. We also have the relations (note 3 * 2 = 6 cases): |
| 738 | * |
| 739 | * fsat_signed(fpos(x)) = fsat(x) |
| 740 | * fsat_signed(fsat(x)) = fsat(x) |
| 741 | * fpos(fsat_signed(x)) = fsat(x) |
| 742 | * fpos(fsat(x)) = fsat(x) |
| 743 | * fsat(fsat_signed(x)) = fsat(x) |
| 744 | * fsat(fpos(x)) = fsat(x) |
| 745 | * |
| 746 | * So by cases any composition of output modifiers is equivalent to |
| 747 | * fsat alone. |
| 748 | */ |
| 749 | static unsigned |
| 750 | mir_determine_float_outmod(compiler_context *ctx, nir_dest **dest, unsigned prior_outmod) |
| 751 | { |
| 752 | bool fpos = mir_accept_dest_mod(ctx, dest, nir_op_fclamp_pos); |
| 753 | bool fsat = mir_accept_dest_mod(ctx, dest, nir_op_fsat); |
| 754 | bool ssat = mir_accept_dest_mod(ctx, dest, nir_op_fsat_signed); |
| 755 | bool prior = (prior_outmod != midgard_outmod_none); |
| 756 | int count = (int) prior + (int) fpos + (int) ssat + (int) fsat; |
| 757 | |
| 758 | return ((count > 1) || fsat) ? midgard_outmod_sat : |
| 759 | fpos ? midgard_outmod_pos : |
| 760 | ssat ? midgard_outmod_sat_signed : |
| 761 | prior_outmod; |
| 762 | } |
| 763 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 764 | static void |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 765 | mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i, unsigned to, bool *abs, bool *neg, bool *not, enum midgard_roundmode *roundmode, bool is_int, unsigned bcast_count) |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 766 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 767 | nir_alu_src src = instr->src[i]; |
Alyssa Rosenzweig | b124f53 | 2020-04-29 18:10:43 -0400 | [diff] [blame] | 768 | |
| 769 | if (!is_int) { |
| 770 | if (pan_has_source_mod(&src, nir_op_fneg)) |
| 771 | *neg = !(*neg); |
| 772 | |
| 773 | if (pan_has_source_mod(&src, nir_op_fabs)) |
| 774 | *abs = true; |
| 775 | } |
| 776 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 777 | if (nir_accepts_inot(instr->op, i) && pan_has_source_mod(&src, nir_op_inot)) |
| 778 | *not = true; |
| 779 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 780 | if (roundmode) { |
| 781 | if (pan_has_source_mod(&src, nir_op_fround_even)) |
| 782 | *roundmode = MIDGARD_RTE; |
| 783 | |
| 784 | if (pan_has_source_mod(&src, nir_op_ftrunc)) |
| 785 | *roundmode = MIDGARD_RTZ; |
| 786 | |
| 787 | if (pan_has_source_mod(&src, nir_op_ffloor)) |
| 788 | *roundmode = MIDGARD_RTN; |
| 789 | |
| 790 | if (pan_has_source_mod(&src, nir_op_fceil)) |
| 791 | *roundmode = MIDGARD_RTP; |
| 792 | } |
| 793 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 794 | unsigned bits = nir_src_bit_size(src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 795 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 796 | ins->src[to] = nir_src_index(NULL, &src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 797 | ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 798 | |
| 799 | for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 800 | ins->swizzle[to][c] = src.swizzle[ |
| 801 | (!bcast_count || c < bcast_count) ? c : |
| 802 | (bcast_count - 1)]; |
| 803 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 804 | } |
| 805 | |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 806 | /* Midgard features both fcsel and icsel, depending on whether you want int or |
| 807 | * float modifiers. NIR's csel is typeless, so we want a heuristic to guess if |
| 808 | * we should emit an int or float csel depending on what modifiers could be |
| 809 | * placed. In the absense of modifiers, this is probably arbitrary. */ |
| 810 | |
| 811 | static bool |
| 812 | mir_is_bcsel_float(nir_alu_instr *instr) |
| 813 | { |
| 814 | nir_op intmods[] = { |
| 815 | nir_op_i2i8, nir_op_i2i16, |
| 816 | nir_op_i2i32, nir_op_i2i64 |
| 817 | }; |
| 818 | |
| 819 | nir_op floatmods[] = { |
| 820 | nir_op_fabs, nir_op_fneg, |
| 821 | nir_op_f2f16, nir_op_f2f32, |
| 822 | nir_op_f2f64 |
| 823 | }; |
| 824 | |
| 825 | nir_op floatdestmods[] = { |
| 826 | nir_op_fsat, nir_op_fsat_signed, nir_op_fclamp_pos, |
| 827 | nir_op_f2f16, nir_op_f2f32 |
| 828 | }; |
| 829 | |
| 830 | signed score = 0; |
| 831 | |
| 832 | for (unsigned i = 1; i < 3; ++i) { |
| 833 | nir_alu_src s = instr->src[i]; |
| 834 | for (unsigned q = 0; q < ARRAY_SIZE(intmods); ++q) { |
| 835 | if (pan_has_source_mod(&s, intmods[q])) |
| 836 | score--; |
| 837 | } |
| 838 | } |
| 839 | |
| 840 | for (unsigned i = 1; i < 3; ++i) { |
| 841 | nir_alu_src s = instr->src[i]; |
| 842 | for (unsigned q = 0; q < ARRAY_SIZE(floatmods); ++q) { |
| 843 | if (pan_has_source_mod(&s, floatmods[q])) |
| 844 | score++; |
| 845 | } |
| 846 | } |
| 847 | |
| 848 | for (unsigned q = 0; q < ARRAY_SIZE(floatdestmods); ++q) { |
| 849 | nir_dest *dest = &instr->dest.dest; |
| 850 | if (pan_has_dest_mod(&dest, floatdestmods[q])) |
| 851 | score++; |
| 852 | } |
| 853 | |
| 854 | return (score > 0); |
| 855 | } |
| 856 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 857 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 858 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 859 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 860 | nir_dest *dest = &instr->dest.dest; |
| 861 | |
| 862 | if (dest->is_ssa && BITSET_TEST(ctx->already_emitted, dest->ssa.index)) |
| 863 | return; |
| 864 | |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 865 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 866 | * is handled elsewhere */ |
| 867 | |
| 868 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 869 | midgard_emit_derivatives(ctx, instr); |
| 870 | return; |
| 871 | } |
| 872 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 873 | bool is_ssa = dest->is_ssa; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 874 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 875 | unsigned nr_components = nir_dest_num_components(*dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 876 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | 04f76ad | 2020-04-27 18:58:21 -0400 | [diff] [blame] | 877 | unsigned op = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 878 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 879 | /* Number of components valid to check for the instruction (the rest |
| 880 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 881 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 882 | * in Midgard */ |
| 883 | |
| 884 | unsigned broadcast_swizzle = 0; |
| 885 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 886 | /* Should we swap arguments? */ |
| 887 | bool flip_src12 = false; |
| 888 | |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 889 | ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 890 | ASSERTED unsigned dst_bitsize = nir_dest_bit_size(*dest); |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 891 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 892 | enum midgard_roundmode roundmode = MIDGARD_RTE; |
| 893 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 894 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 895 | ALU_CASE(fadd, fadd); |
| 896 | ALU_CASE(fmul, fmul); |
| 897 | ALU_CASE(fmin, fmin); |
| 898 | ALU_CASE(fmax, fmax); |
| 899 | ALU_CASE(imin, imin); |
| 900 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 901 | ALU_CASE(umin, umin); |
| 902 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 903 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 904 | ALU_CASE(fround_even, froundeven); |
| 905 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 906 | ALU_CASE(fceil, fceil); |
| 907 | ALU_CASE(fdot3, fdot3); |
| 908 | ALU_CASE(fdot4, fdot4); |
| 909 | ALU_CASE(iadd, iadd); |
| 910 | ALU_CASE(isub, isub); |
| 911 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 912 | ALU_CASE(imul_high, imul); |
| 913 | ALU_CASE(umul_high, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 914 | |
| 915 | /* Zero shoved as second-arg */ |
| 916 | ALU_CASE(iabs, iabsdiff); |
| 917 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 918 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 919 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 920 | ALU_CASE_CMP(feq32, feq, false); |
Karol Herbst | e5899c1 | 2020-08-18 19:51:57 +0200 | [diff] [blame] | 921 | ALU_CASE_CMP(fneu32, fne, false); |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 922 | ALU_CASE_CMP(flt32, flt, false); |
| 923 | ALU_CASE_CMP(ieq32, ieq, true); |
| 924 | ALU_CASE_CMP(ine32, ine, true); |
| 925 | ALU_CASE_CMP(ilt32, ilt, true); |
| 926 | ALU_CASE_CMP(ult32, ult, false); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 927 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 928 | /* We don't have a native b2f32 instruction. Instead, like many |
| 929 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 930 | * correspondingly AND |
| 931 | * by 1.0 to do the type conversion. For the moment, prime us |
| 932 | * to emit: |
| 933 | * |
| 934 | * iand [whatever], #0 |
| 935 | * |
| 936 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 937 | */ |
| 938 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 939 | ALU_CASE_CMP(b2f32, iand, true); |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 940 | ALU_CASE_CMP(b2f16, iand, true); |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 941 | ALU_CASE_CMP(b2i32, iand, true); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 942 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 943 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 944 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 945 | |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 946 | ALU_CASE_CMP(f2b32, fne, false); |
| 947 | ALU_CASE_CMP(i2b32, ine, true); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 948 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 949 | ALU_CASE(frcp, frcp); |
| 950 | ALU_CASE(frsq, frsqrt); |
| 951 | ALU_CASE(fsqrt, fsqrt); |
| 952 | ALU_CASE(fexp2, fexp2); |
| 953 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 954 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 955 | ALU_CASE_RTZ(f2i64, f2i_rte); |
| 956 | ALU_CASE_RTZ(f2u64, f2u_rte); |
| 957 | ALU_CASE_RTZ(i2f64, i2f_rte); |
| 958 | ALU_CASE_RTZ(u2f64, u2f_rte); |
Boris Brezillon | fcceeaf | 2020-01-20 22:05:14 +0100 | [diff] [blame] | 959 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 960 | ALU_CASE_RTZ(f2i32, f2i_rte); |
| 961 | ALU_CASE_RTZ(f2u32, f2u_rte); |
| 962 | ALU_CASE_RTZ(i2f32, i2f_rte); |
| 963 | ALU_CASE_RTZ(u2f32, u2f_rte); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 964 | |
Alyssa Rosenzweig | 0ae0141 | 2020-05-25 14:46:40 -0400 | [diff] [blame] | 965 | ALU_CASE_RTZ(f2i8, f2i_rte); |
| 966 | ALU_CASE_RTZ(f2u8, f2u_rte); |
| 967 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 968 | ALU_CASE_RTZ(f2i16, f2i_rte); |
| 969 | ALU_CASE_RTZ(f2u16, f2u_rte); |
| 970 | ALU_CASE_RTZ(i2f16, i2f_rte); |
| 971 | ALU_CASE_RTZ(u2f16, u2f_rte); |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 972 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 973 | ALU_CASE(fsin, fsin); |
| 974 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 975 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 976 | /* We'll get 0 in the second arg, so: |
| 977 | * ~a = ~(a | 0) = nor(a, 0) */ |
| 978 | ALU_CASE(inot, inor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 979 | ALU_CASE(iand, iand); |
| 980 | ALU_CASE(ior, ior); |
| 981 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 982 | ALU_CASE(ishl, ishl); |
| 983 | ALU_CASE(ishr, iasr); |
| 984 | ALU_CASE(ushr, ilsr); |
| 985 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 986 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 987 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 988 | ALU_CASE_CMP(b32all_fequal4, fball_eq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 989 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 990 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 991 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 992 | ALU_CASE_CMP(b32any_fnequal4, fbany_neq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 993 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 994 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 995 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 996 | ALU_CASE_CMP(b32all_iequal4, iball_eq, true); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 997 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 998 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 999 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 1000 | ALU_CASE_CMP(b32any_inequal4, ibany_neq, true); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1001 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1002 | /* Source mods will be shoved in later */ |
| 1003 | ALU_CASE(fabs, fmov); |
| 1004 | ALU_CASE(fneg, fmov); |
| 1005 | ALU_CASE(fsat, fmov); |
Alyssa Rosenzweig | 24e2e24 | 2020-05-04 16:12:41 -0400 | [diff] [blame] | 1006 | ALU_CASE(fsat_signed, fmov); |
| 1007 | ALU_CASE(fclamp_pos, fmov); |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1008 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1009 | /* For size conversion, we use a move. Ideally though we would squash |
| 1010 | * these ops together; maybe that has to happen after in NIR as part of |
| 1011 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1012 | * only / exactly one size. If stepping down, we use a dest override to |
| 1013 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 1014 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1015 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1016 | case nir_op_i2i8: |
| 1017 | case nir_op_i2i16: |
| 1018 | case nir_op_i2i32: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1019 | case nir_op_i2i64: |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1020 | case nir_op_u2u8: |
| 1021 | case nir_op_u2u16: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1022 | case nir_op_u2u32: |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1023 | case nir_op_u2u64: |
| 1024 | case nir_op_f2f16: |
Boris Brezillon | e1f9e8d | 2020-01-20 16:05:31 +0100 | [diff] [blame] | 1025 | case nir_op_f2f32: |
| 1026 | case nir_op_f2f64: { |
| 1027 | if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 || |
| 1028 | instr->op == nir_op_f2f64) |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1029 | op = midgard_alu_op_fmov; |
| 1030 | else |
| 1031 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1032 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1033 | break; |
| 1034 | } |
| 1035 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1036 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1037 | * arguments */ |
| 1038 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1039 | case nir_op_fge: |
| 1040 | case nir_op_fge32: |
| 1041 | case nir_op_ige32: |
| 1042 | case nir_op_uge32: { |
| 1043 | op = |
| 1044 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 1045 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 1046 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 1047 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 1048 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1049 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1050 | flip_src12 = true; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 1051 | ALU_CHECK_CMP(false); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1052 | break; |
| 1053 | } |
| 1054 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 1055 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 1056 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 1057 | bool is_float = mir_is_bcsel_float(instr); |
| 1058 | op = is_float ? |
| 1059 | (mixed ? midgard_alu_op_fcsel_v : midgard_alu_op_fcsel) : |
| 1060 | (mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1061 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1062 | break; |
| 1063 | } |
| 1064 | |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1065 | case nir_op_unpack_32_2x16: |
| 1066 | case nir_op_unpack_32_4x8: |
| 1067 | case nir_op_pack_32_2x16: |
| 1068 | case nir_op_pack_32_4x8: { |
| 1069 | op = midgard_alu_op_imov; |
| 1070 | break; |
| 1071 | } |
| 1072 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1073 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1074 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1075 | assert(0); |
| 1076 | return; |
| 1077 | } |
| 1078 | |
Alyssa Rosenzweig | 72c1e3a | 2020-05-21 12:31:40 -0400 | [diff] [blame] | 1079 | /* Promote imov to fmov if it might help inline a constant */ |
| 1080 | if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src) |
| 1081 | && nir_src_bit_size(instr->src[0].src) == 32 |
| 1082 | && nir_is_same_comp_swizzle(instr->src[0].swizzle, |
| 1083 | nir_src_num_components(instr->src[0].src))) { |
| 1084 | op = midgard_alu_op_fmov; |
| 1085 | } |
| 1086 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 1087 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1088 | |
| 1089 | unsigned outmod = 0; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1090 | bool is_int = midgard_is_integer_op(op); |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 1091 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 1092 | if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) { |
| 1093 | outmod = midgard_outmod_int_high; |
| 1094 | } else if (midgard_is_integer_out_op(op)) { |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1095 | outmod = midgard_outmod_int_wrap; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1096 | } else if (instr->op == nir_op_fsat) { |
| 1097 | outmod = midgard_outmod_sat; |
| 1098 | } else if (instr->op == nir_op_fsat_signed) { |
| 1099 | outmod = midgard_outmod_sat_signed; |
| 1100 | } else if (instr->op == nir_op_fclamp_pos) { |
| 1101 | outmod = midgard_outmod_pos; |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1102 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1103 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1104 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 1105 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1106 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1107 | |
Italo Nicola | 2096903 | 2020-07-13 16:19:08 +0000 | [diff] [blame] | 1108 | if (!midgard_is_integer_out_op(op)) { |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1109 | outmod = mir_determine_float_outmod(ctx, &dest, outmod); |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 1110 | } |
| 1111 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1112 | midgard_instruction ins = { |
| 1113 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1114 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ecf9466 | 2020-04-27 18:57:34 -0400 | [diff] [blame] | 1115 | .dest_type = nir_op_infos[instr->op].output_type |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1116 | | nir_dest_bit_size(*dest), |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 1117 | .roundmode = roundmode, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1118 | }; |
| 1119 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1120 | enum midgard_roundmode *roundptr = (opcode_props & MIDGARD_ROUNDS) ? |
| 1121 | &ins.roundmode : NULL; |
| 1122 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1123 | for (unsigned i = nr_inputs; i < ARRAY_SIZE(ins.src); ++i) |
| 1124 | ins.src[i] = ~0; |
| 1125 | |
| 1126 | if (quirk_flipped_r24) { |
| 1127 | ins.src[0] = ~0; |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1128 | mir_copy_src(&ins, instr, 0, 1, &ins.src_abs[1], &ins.src_neg[1], &ins.src_invert[1], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1129 | } else { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1130 | for (unsigned i = 0; i < nr_inputs; ++i) { |
| 1131 | unsigned to = i; |
| 1132 | |
| 1133 | if (instr->op == nir_op_b32csel) { |
| 1134 | /* The condition is the first argument; move |
| 1135 | * the other arguments up one to be a binary |
| 1136 | * instruction for Midgard with the condition |
| 1137 | * last */ |
| 1138 | |
| 1139 | if (i == 0) |
| 1140 | to = 2; |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1141 | else if (flip_src12) |
| 1142 | to = 2 - i; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1143 | else |
| 1144 | to = i - 1; |
| 1145 | } else if (flip_src12) { |
| 1146 | to = 1 - to; |
| 1147 | } |
| 1148 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1149 | mir_copy_src(&ins, instr, i, to, &ins.src_abs[to], &ins.src_neg[to], &ins.src_invert[to], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1150 | |
| 1151 | /* (!c) ? a : b = c ? b : a */ |
| 1152 | if (instr->op == nir_op_b32csel && ins.src_invert[2]) { |
| 1153 | ins.src_invert[2] = false; |
| 1154 | flip_src12 ^= true; |
| 1155 | } |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1156 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1157 | } |
| 1158 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1159 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1160 | /* Lowered to move */ |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1161 | if (instr->op == nir_op_fneg) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1162 | ins.src_neg[1] ^= true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1163 | |
| 1164 | if (instr->op == nir_op_fabs) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1165 | ins.src_abs[1] = true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1166 | } |
| 1167 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1168 | ins.mask = mask_of(nr_components); |
| 1169 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1170 | /* Apply writemask if non-SSA, keeping in mind that we can't write to |
| 1171 | * components that don't exist. Note modifier => SSA => !reg => no |
| 1172 | * writemask, so we don't have to worry about writemasks here.*/ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1173 | |
| 1174 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1175 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1176 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1177 | ins.op = op; |
Italo Nicola | 5011373 | 2020-07-15 18:43:18 +0000 | [diff] [blame] | 1178 | ins.outmod = outmod; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1179 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1180 | /* Late fixup for emulated instructions */ |
| 1181 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1182 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1183 | /* Presently, our second argument is an inline #0 constant. |
| 1184 | * Switch over to an embedded 1.0 constant (that can't fit |
| 1185 | * inline, since we're 32-bit, not 16-bit like the inline |
| 1186 | * constants) */ |
| 1187 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1188 | ins.has_inline_constant = false; |
| 1189 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1190 | ins.src_types[1] = nir_type_float32; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1191 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1192 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1193 | if (instr->op == nir_op_b2f32) |
| 1194 | ins.constants.f32[0] = 1.0f; |
| 1195 | else |
| 1196 | ins.constants.i32[0] = 1; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1197 | |
| 1198 | for (unsigned c = 0; c < 16; ++c) |
| 1199 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 1200 | } else if (instr->op == nir_op_b2f16) { |
| 1201 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
| 1202 | ins.src_types[1] = nir_type_float16; |
| 1203 | ins.has_constants = true; |
| 1204 | ins.constants.i16[0] = _mesa_float_to_half(1.0); |
| 1205 | |
| 1206 | for (unsigned c = 0; c < 16; ++c) |
| 1207 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 1208 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 1209 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1210 | ins.has_inline_constant = false; |
| 1211 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Italo Nicola | b1b0ce0 | 2020-07-10 14:51:52 +0000 | [diff] [blame] | 1212 | ins.src_types[1] = ins.src_types[0]; |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1213 | ins.has_constants = true; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1214 | ins.constants.u32[0] = 0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1215 | |
| 1216 | for (unsigned c = 0; c < 16; ++c) |
| 1217 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1218 | } else if (instr->op == nir_op_pack_32_2x16) { |
| 1219 | ins.dest_type = nir_type_uint16; |
| 1220 | ins.mask = mask_of(nr_components * 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1221 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1222 | } else if (instr->op == nir_op_pack_32_4x8) { |
| 1223 | ins.dest_type = nir_type_uint8; |
| 1224 | ins.mask = mask_of(nr_components * 4); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1225 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1226 | } else if (instr->op == nir_op_unpack_32_2x16) { |
| 1227 | ins.dest_type = nir_type_uint32; |
| 1228 | ins.mask = mask_of(nr_components >> 1); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1229 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1230 | } else if (instr->op == nir_op_unpack_32_4x8) { |
| 1231 | ins.dest_type = nir_type_uint32; |
| 1232 | ins.mask = mask_of(nr_components >> 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1233 | ins.is_pack = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1234 | } |
| 1235 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1236 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 1237 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 1238 | * instructions can only operate as if they were scalars. Lower |
| 1239 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1240 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1241 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1242 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1243 | unsigned swizzle_back[MIR_VEC_COMPONENTS]; |
| 1244 | memcpy(&swizzle_back, ins.swizzle[0], sizeof(swizzle_back)); |
| 1245 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1246 | midgard_instruction ins_split[MIR_VEC_COMPONENTS]; |
| 1247 | unsigned ins_count = 0; |
| 1248 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1249 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1250 | /* Mask the associated component, dropping the |
| 1251 | * instruction if needed */ |
| 1252 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1253 | ins.mask = 1 << i; |
| 1254 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1255 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1256 | for (unsigned j = 0; j < ins_count; ++j) { |
| 1257 | if (swizzle_back[i] == ins_split[j].swizzle[0][0]) { |
| 1258 | ins_split[j].mask |= ins.mask; |
| 1259 | ins.mask = 0; |
| 1260 | break; |
| 1261 | } |
| 1262 | } |
| 1263 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1264 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1265 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1266 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1267 | for (unsigned j = 0; j < MIR_VEC_COMPONENTS; ++j) |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1268 | ins.swizzle[0][j] = swizzle_back[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1269 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1270 | ins_split[ins_count] = ins; |
| 1271 | |
| 1272 | ++ins_count; |
| 1273 | } |
| 1274 | |
| 1275 | for (unsigned i = 0; i < ins_count; ++i) { |
| 1276 | emit_mir_instruction(ctx, ins_split[i]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1277 | } |
| 1278 | } else { |
| 1279 | emit_mir_instruction(ctx, ins); |
| 1280 | } |
| 1281 | } |
| 1282 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1283 | #undef ALU_CASE |
| 1284 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1285 | static void |
| 1286 | mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read) |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1287 | { |
| 1288 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1289 | unsigned nir_mask = 0; |
| 1290 | unsigned dsize = 0; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1291 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1292 | if (is_read) { |
| 1293 | nir_mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1294 | dsize = nir_dest_bit_size(intr->dest); |
| 1295 | } else { |
| 1296 | nir_mask = nir_intrinsic_write_mask(intr); |
| 1297 | dsize = 32; |
| 1298 | } |
| 1299 | |
| 1300 | /* Once we have the NIR mask, we need to normalize to work in 32-bit space */ |
Alyssa Rosenzweig | 9b8cb9f | 2020-03-09 20:19:29 -0400 | [diff] [blame] | 1301 | unsigned bytemask = pan_to_bytemask(dsize, nir_mask); |
Alyssa Rosenzweig | b91d715 | 2020-05-11 15:06:53 -0400 | [diff] [blame] | 1302 | ins->dest_type = nir_type_uint | dsize; |
Italo Nicola | 1101261 | 2020-08-26 14:56:13 +0000 | [diff] [blame] | 1303 | mir_set_bytemask(ins, bytemask); |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1304 | } |
| 1305 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1306 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1307 | * optimized) versions of UBO #0 */ |
| 1308 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1309 | static midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1310 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1311 | compiler_context *ctx, |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1312 | nir_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1313 | unsigned dest, |
| 1314 | unsigned offset, |
| 1315 | nir_src *indirect_offset, |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1316 | unsigned indirect_shift, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1317 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1318 | { |
| 1319 | /* TODO: half-floats */ |
| 1320 | |
Alyssa Rosenzweig | bc9a7d0 | 2019-11-15 14:19:34 -0500 | [diff] [blame] | 1321 | midgard_instruction ins = m_ld_ubo_int4(dest, 0); |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1322 | ins.constants.u32[0] = offset; |
Alyssa Rosenzweig | da73651 | 2019-12-19 11:12:25 -0500 | [diff] [blame] | 1323 | |
| 1324 | if (instr->type == nir_instr_type_intrinsic) |
| 1325 | mir_set_intr_mask(instr, &ins, true); |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1326 | |
| 1327 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1328 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1329 | ins.src_types[2] = nir_type_uint32; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1330 | ins.load_store.arg_2 = (indirect_shift << 5); |
Alyssa Rosenzweig | 797fa87 | 2020-07-06 10:57:04 -0400 | [diff] [blame] | 1331 | |
| 1332 | /* X component for the whole swizzle to prevent register |
| 1333 | * pressure from ballooning from the extra components */ |
| 1334 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[2]); ++i) |
| 1335 | ins.swizzle[2][i] = 0; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1336 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1337 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1338 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1339 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1340 | ins.load_store.arg_1 = index; |
| 1341 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1342 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1343 | } |
| 1344 | |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1345 | /* Globals are like UBOs if you squint. And shared memory is like globals if |
| 1346 | * you squint even harder */ |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1347 | |
| 1348 | static void |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1349 | emit_global( |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1350 | compiler_context *ctx, |
| 1351 | nir_instr *instr, |
| 1352 | bool is_read, |
| 1353 | unsigned srcdest, |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1354 | nir_src *offset, |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1355 | bool is_shared) |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1356 | { |
| 1357 | /* TODO: types */ |
| 1358 | |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 1359 | midgard_instruction ins; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1360 | |
| 1361 | if (is_read) |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1362 | ins = m_ld_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1363 | else |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1364 | ins = m_st_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1365 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1366 | mir_set_offset(ctx, &ins, offset, is_shared); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1367 | mir_set_intr_mask(instr, &ins, is_read); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1368 | |
Alyssa Rosenzweig | 41184f8 | 2020-08-27 15:13:19 -0400 | [diff] [blame] | 1369 | /* Set a valid swizzle for masked out components */ |
| 1370 | assert(ins.mask); |
| 1371 | unsigned first_component = __builtin_ffs(ins.mask) - 1; |
| 1372 | |
| 1373 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) { |
| 1374 | if (!(ins.mask & (1 << i))) |
| 1375 | ins.swizzle[0][i] = first_component; |
| 1376 | } |
| 1377 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1378 | emit_mir_instruction(ctx, ins); |
| 1379 | } |
| 1380 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 1381 | /* If is_shared is off, the only other possible value are globals, since |
| 1382 | * SSBO's are being lowered to globals through a NIR pass. */ |
| 1383 | static void |
| 1384 | emit_atomic( |
| 1385 | compiler_context *ctx, |
| 1386 | nir_intrinsic_instr *instr, |
| 1387 | bool is_shared, |
| 1388 | midgard_load_store_op op) |
| 1389 | { |
| 1390 | unsigned bitsize = nir_src_bit_size(instr->src[1]); |
| 1391 | nir_alu_type type = |
| 1392 | (op == midgard_op_atomic_imin || op == midgard_op_atomic_imax) ? |
| 1393 | nir_type_int : nir_type_uint; |
| 1394 | |
| 1395 | unsigned dest = nir_dest_index(&instr->dest); |
| 1396 | unsigned val = nir_src_index(ctx, &instr->src[1]); |
| 1397 | emit_explicit_constant(ctx, val, val); |
| 1398 | |
| 1399 | midgard_instruction ins = { |
| 1400 | .type = TAG_LOAD_STORE_4, |
| 1401 | .mask = 0xF, |
| 1402 | .dest = dest, |
| 1403 | .src = { ~0, ~0, ~0, val }, |
| 1404 | .src_types = { 0, 0, 0, type | bitsize }, |
| 1405 | .op = op |
| 1406 | }; |
| 1407 | |
| 1408 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1409 | |
| 1410 | /* cmpxchg takes an extra value in arg_2, so we don't use it for the offset */ |
| 1411 | if (op == midgard_op_atomic_cmpxchg) { |
| 1412 | unsigned addr = nir_src_index(ctx, src_offset); |
| 1413 | |
| 1414 | ins.src[1] = addr; |
| 1415 | ins.src_types[1] = nir_type_uint | nir_src_bit_size(*src_offset); |
| 1416 | |
| 1417 | unsigned xchg_val = nir_src_index(ctx, &instr->src[2]); |
| 1418 | emit_explicit_constant(ctx, xchg_val, xchg_val); |
| 1419 | |
| 1420 | ins.src[2] = val; |
| 1421 | ins.src_types[2] = type | bitsize; |
| 1422 | ins.src[3] = xchg_val; |
| 1423 | |
| 1424 | if (is_shared) |
| 1425 | ins.load_store.arg_1 |= 0x6E; |
| 1426 | } else { |
| 1427 | mir_set_offset(ctx, &ins, src_offset, is_shared); |
| 1428 | } |
| 1429 | |
| 1430 | mir_set_intr_mask(&instr->instr, &ins, true); |
| 1431 | |
| 1432 | emit_mir_instruction(ctx, ins); |
| 1433 | } |
| 1434 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1435 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1436 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1437 | compiler_context *ctx, |
| 1438 | unsigned dest, unsigned offset, |
| 1439 | unsigned nr_comp, unsigned component, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1440 | nir_src *indirect_offset, nir_alu_type type, bool flat) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1441 | { |
| 1442 | /* XXX: Half-floats? */ |
| 1443 | /* TODO: swizzle, mask */ |
| 1444 | |
| 1445 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1446 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 2d16883 | 2020-06-04 11:32:59 -0400 | [diff] [blame] | 1447 | ins.dest_type = type; |
| 1448 | |
| 1449 | if (type == nir_type_float16) { |
| 1450 | /* Ensure we are aligned so we can pack it later */ |
| 1451 | ins.mask = mask_of(ALIGN_POT(nr_comp, 2)); |
| 1452 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1453 | |
| 1454 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) |
| 1455 | ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1456 | |
| 1457 | midgard_varying_parameter p = { |
| 1458 | .is_varying = 1, |
| 1459 | .interpolation = midgard_interp_default, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1460 | .flat = flat, |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1461 | }; |
| 1462 | |
| 1463 | unsigned u; |
| 1464 | memcpy(&u, &p, sizeof(p)); |
| 1465 | ins.load_store.varying_parameters = u; |
| 1466 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1467 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1468 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1469 | ins.src_types[2] = nir_type_uint32; |
| 1470 | } else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1471 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1472 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1473 | ins.load_store.arg_1 = 0x9E; |
| 1474 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1475 | /* Use the type appropriate load */ |
| 1476 | switch (type) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1477 | case nir_type_uint32: |
| 1478 | case nir_type_bool32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1479 | ins.op = midgard_op_ld_vary_32u; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1480 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1481 | case nir_type_int32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1482 | ins.op = midgard_op_ld_vary_32i; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1483 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1484 | case nir_type_float32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1485 | ins.op = midgard_op_ld_vary_32; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1486 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1487 | case nir_type_float16: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1488 | ins.op = midgard_op_ld_vary_16; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1489 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1490 | default: |
| 1491 | unreachable("Attempted to load unknown type"); |
| 1492 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1493 | } |
| 1494 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1495 | emit_mir_instruction(ctx, ins); |
| 1496 | } |
| 1497 | |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1498 | static void |
| 1499 | emit_attr_read( |
| 1500 | compiler_context *ctx, |
| 1501 | unsigned dest, unsigned offset, |
| 1502 | unsigned nr_comp, nir_alu_type t) |
| 1503 | { |
| 1504 | midgard_instruction ins = m_ld_attr_32(dest, offset); |
| 1505 | ins.load_store.arg_1 = 0x1E; |
| 1506 | ins.load_store.arg_2 = 0x1E; |
| 1507 | ins.mask = mask_of(nr_comp); |
| 1508 | |
| 1509 | /* Use the type appropriate load */ |
| 1510 | switch (t) { |
| 1511 | case nir_type_uint: |
| 1512 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1513 | ins.op = midgard_op_ld_attr_32u; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1514 | break; |
| 1515 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1516 | ins.op = midgard_op_ld_attr_32i; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1517 | break; |
| 1518 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1519 | ins.op = midgard_op_ld_attr_32; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1520 | break; |
| 1521 | default: |
| 1522 | unreachable("Attempted to load unknown type"); |
| 1523 | break; |
| 1524 | } |
| 1525 | |
| 1526 | emit_mir_instruction(ctx, ins); |
| 1527 | } |
| 1528 | |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1529 | static void |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1530 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1531 | unsigned nr_components, unsigned offset) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1532 | { |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1533 | nir_dest nir_dest; |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1534 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1535 | /* Figure out which uniform this is */ |
Alyssa Rosenzweig | e610267 | 2020-03-10 16:06:30 -0400 | [diff] [blame] | 1536 | int sysval = panfrost_sysval_for_instr(instr, &nir_dest); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1537 | void *val = _mesa_hash_table_u64_search(ctx->sysvals.sysval_to_id, sysval); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1538 | |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1539 | unsigned dest = nir_dest_index(&nir_dest); |
| 1540 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1541 | /* Sysvals are prefix uniforms */ |
| 1542 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1543 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1544 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1545 | midgard_instruction *ins = |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1546 | emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0, 0); |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1547 | |
| 1548 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1549 | } |
| 1550 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1551 | static unsigned |
| 1552 | compute_builtin_arg(nir_op op) |
| 1553 | { |
| 1554 | switch (op) { |
| 1555 | case nir_intrinsic_load_work_group_id: |
| 1556 | return 0x14; |
| 1557 | case nir_intrinsic_load_local_invocation_id: |
| 1558 | return 0x10; |
| 1559 | default: |
| 1560 | unreachable("Invalid compute paramater loaded"); |
| 1561 | } |
| 1562 | } |
| 1563 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1564 | static void |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1565 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z, unsigned src_s, enum midgard_rt_id rt) |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1566 | { |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1567 | assert(rt < ARRAY_SIZE(ctx->writeout_branch)); |
| 1568 | |
| 1569 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 1570 | |
| 1571 | assert(!br); |
| 1572 | |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 1573 | emit_explicit_constant(ctx, src, src); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1574 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1575 | struct midgard_instruction ins = |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1576 | v_branch(false, false); |
| 1577 | |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1578 | bool depth_only = (rt == MIDGARD_ZS_RT); |
| 1579 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1580 | ins.writeout = depth_only ? 0 : PAN_WRITEOUT_C; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1581 | |
| 1582 | /* Add dependencies */ |
Alyssa Rosenzweig | 7652983 | 2019-08-30 11:01:15 -0700 | [diff] [blame] | 1583 | ins.src[0] = src; |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1584 | ins.src_types[0] = nir_type_uint32; |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1585 | ins.constants.u32[0] = depth_only ? 0xFF : (rt - MIDGARD_COLOR_RT0) * 0x100; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 1586 | for (int i = 0; i < 4; ++i) |
| 1587 | ins.swizzle[0][i] = i; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1588 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1589 | if (~src_z) { |
| 1590 | emit_explicit_constant(ctx, src_z, src_z); |
| 1591 | ins.src[2] = src_z; |
| 1592 | ins.src_types[2] = nir_type_uint32; |
| 1593 | ins.writeout |= PAN_WRITEOUT_Z; |
| 1594 | } |
| 1595 | if (~src_s) { |
| 1596 | emit_explicit_constant(ctx, src_s, src_s); |
| 1597 | ins.src[3] = src_s; |
| 1598 | ins.src_types[3] = nir_type_uint32; |
| 1599 | ins.writeout |= PAN_WRITEOUT_S; |
| 1600 | } |
| 1601 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1602 | /* Emit the branch */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1603 | br = emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 281cc6f | 2019-11-23 12:43:55 -0500 | [diff] [blame] | 1604 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 1605 | ctx->writeout_branch[rt] = br; |
| 1606 | |
| 1607 | /* Push our current location = current block count - 1 = where we'll |
| 1608 | * jump to. Maybe a bit too clever for my own good */ |
| 1609 | |
| 1610 | br->branch.target_block = ctx->block_count - 1; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1611 | } |
| 1612 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1613 | static void |
| 1614 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1615 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1616 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1617 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1618 | ins.mask = mask_of(3); |
Alyssa Rosenzweig | d3747fb | 2020-02-12 08:39:29 -0500 | [diff] [blame] | 1619 | ins.swizzle[0][3] = COMPONENT_X; /* xyzx */ |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1620 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1621 | emit_mir_instruction(ctx, ins); |
| 1622 | } |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1623 | |
| 1624 | static unsigned |
| 1625 | vertex_builtin_arg(nir_op op) |
| 1626 | { |
| 1627 | switch (op) { |
| 1628 | case nir_intrinsic_load_vertex_id: |
| 1629 | return PAN_VERTEX_ID; |
| 1630 | case nir_intrinsic_load_instance_id: |
| 1631 | return PAN_INSTANCE_ID; |
| 1632 | default: |
| 1633 | unreachable("Invalid vertex builtin"); |
| 1634 | } |
| 1635 | } |
| 1636 | |
| 1637 | static void |
| 1638 | emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1639 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1640 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1641 | emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int); |
| 1642 | } |
| 1643 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1644 | static void |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1645 | emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx) |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1646 | { |
| 1647 | unsigned reg = nir_dest_index(&instr->dest); |
| 1648 | |
| 1649 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1650 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1651 | ld.load_store.address = idx; |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1652 | ld.load_store.arg_2 = 0x1E; |
| 1653 | |
| 1654 | for (int i = 0; i < 4; ++i) |
| 1655 | ld.swizzle[0][i] = COMPONENT_X; |
| 1656 | |
| 1657 | emit_mir_instruction(ctx, ld); |
| 1658 | } |
| 1659 | |
| 1660 | static void |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1661 | emit_control_barrier(compiler_context *ctx) |
| 1662 | { |
| 1663 | midgard_instruction ins = { |
| 1664 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | fde1f2b | 2020-05-13 11:05:34 -0400 | [diff] [blame] | 1665 | .dest = ~0, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1666 | .src = { ~0, ~0, ~0, ~0 }, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 1667 | .op = TEXTURE_OP_BARRIER, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1668 | }; |
| 1669 | |
| 1670 | emit_mir_instruction(ctx, ins); |
| 1671 | } |
| 1672 | |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1673 | static unsigned |
| 1674 | mir_get_branch_cond(nir_src *src, bool *invert) |
| 1675 | { |
| 1676 | /* Wrap it. No swizzle since it's a scalar */ |
| 1677 | |
| 1678 | nir_alu_src alu = { |
| 1679 | .src = *src |
| 1680 | }; |
| 1681 | |
| 1682 | *invert = pan_has_source_mod(&alu, nir_op_inot); |
| 1683 | return nir_src_index(NULL, &alu.src); |
| 1684 | } |
| 1685 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1686 | static uint8_t |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1687 | output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr) |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1688 | { |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1689 | if (ctx->is_blend) |
| 1690 | return ctx->blend_rt; |
| 1691 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1692 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1693 | var = search_var(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr)); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1694 | assert(var); |
| 1695 | |
| 1696 | unsigned loc = var->data.location; |
| 1697 | |
| 1698 | if (loc == FRAG_RESULT_COLOR) |
| 1699 | loc = FRAG_RESULT_DATA0; |
| 1700 | |
| 1701 | if (loc >= FRAG_RESULT_DATA0) |
| 1702 | return loc - FRAG_RESULT_DATA0; |
| 1703 | |
| 1704 | if (loc == FRAG_RESULT_DEPTH) |
| 1705 | return 0x1F; |
| 1706 | if (loc == FRAG_RESULT_STENCIL) |
| 1707 | return 0x1E; |
| 1708 | |
Icecream95 | 6493d29 | 2020-07-14 15:06:09 +1200 | [diff] [blame] | 1709 | unreachable("Invalid RT to load from"); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1710 | } |
| 1711 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1712 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1713 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1714 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1715 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1716 | |
| 1717 | switch (instr->intrinsic) { |
| 1718 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1719 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1720 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1721 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1722 | discard.branch.target_type = TARGET_DISCARD; |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1723 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1724 | if (conditional) { |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1725 | discard.src[0] = mir_get_branch_cond(&instr->src[0], |
| 1726 | &discard.branch.invert_conditional); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1727 | discard.src_types[0] = nir_type_uint32; |
| 1728 | } |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1729 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1730 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1731 | schedule_barrier(ctx); |
| 1732 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1733 | break; |
| 1734 | } |
| 1735 | |
| 1736 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1737 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1738 | case nir_intrinsic_load_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1739 | case nir_intrinsic_load_shared: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1740 | case nir_intrinsic_load_input: |
| 1741 | case nir_intrinsic_load_interpolated_input: { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1742 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1743 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1744 | bool is_global = instr->intrinsic == nir_intrinsic_load_global; |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1745 | bool is_shared = instr->intrinsic == nir_intrinsic_load_shared; |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1746 | bool is_flat = instr->intrinsic == nir_intrinsic_load_input; |
| 1747 | bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1748 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1749 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1750 | /* TODO: Infer type? Does it matter? */ |
| 1751 | nir_alu_type t = |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1752 | (is_ubo || is_global || is_shared) ? nir_type_uint : |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1753 | (is_interp) ? nir_type_float : |
| 1754 | nir_intrinsic_type(instr); |
| 1755 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1756 | t = nir_alu_type_get_base_type(t); |
| 1757 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1758 | if (!(is_ubo || is_global)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1759 | offset = nir_intrinsic_base(instr); |
| 1760 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1761 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1762 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1763 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1764 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1765 | |
| 1766 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1767 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1768 | |
| 1769 | if (direct) |
| 1770 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1771 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1772 | /* We may need to apply a fractional offset */ |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1773 | int component = (is_flat || is_interp) ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1774 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1775 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1776 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1777 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1778 | emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysvals.sysval_count + offset) * 16, indirect_offset, 4, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1779 | } else if (is_ubo) { |
| 1780 | nir_src index = instr->src[0]; |
| 1781 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1782 | /* TODO: Is indirect block number possible? */ |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1783 | assert(nir_src_is_const(index)); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1784 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1785 | uint32_t uindex = nir_src_as_uint(index) + 1; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1786 | emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1787 | } else if (is_global || is_shared) { |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1788 | emit_global(ctx, &instr->instr, true, reg, src_offset, is_shared); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1789 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1790 | emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1791 | } else if (ctx->is_blend) { |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1792 | /* ctx->blend_input will be precoloured to r0/r2, where |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1793 | * the input is preloaded */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1794 | |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1795 | unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input; |
| 1796 | |
| 1797 | if (*input == ~0) |
| 1798 | *input = reg; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1799 | else |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1800 | emit_mir_instruction(ctx, v_mov(*input, reg)); |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1801 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
| 1802 | emit_attr_read(ctx, reg, offset, nr_comp, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1803 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1804 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1805 | assert(0); |
| 1806 | } |
| 1807 | |
| 1808 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1809 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1810 | |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1811 | /* Artefact of load_interpolated_input. TODO: other barycentric modes */ |
| 1812 | case nir_intrinsic_load_barycentric_pixel: |
Tomeu Vizoso | 2504206 | 2020-01-03 09:42:11 +0100 | [diff] [blame] | 1813 | case nir_intrinsic_load_barycentric_centroid: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1814 | break; |
| 1815 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1816 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1817 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1818 | case nir_intrinsic_load_raw_output_pan: { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1819 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1820 | |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1821 | /* T720 and below use different blend opcodes with slightly |
| 1822 | * different semantics than T760 and up */ |
| 1823 | |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 1824 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1825 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1826 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1827 | |
Icecream95 | c20d166 | 2020-07-16 14:16:11 +1200 | [diff] [blame] | 1828 | if (nir_src_is_const(instr->src[0])) { |
| 1829 | ld.load_store.arg_1 = nir_src_as_uint(instr->src[0]); |
| 1830 | } else { |
| 1831 | ld.load_store.varying_parameters = 2; |
| 1832 | ld.src[1] = nir_src_index(ctx, &instr->src[0]); |
| 1833 | ld.src_types[1] = nir_type_int32; |
| 1834 | } |
| 1835 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1836 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1837 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 5a175e4 | 2020-05-29 21:11:11 -0400 | [diff] [blame] | 1838 | ld.load_store.address = 16; |
| 1839 | ld.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1840 | } |
| 1841 | |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1842 | emit_mir_instruction(ctx, ld); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1843 | break; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1844 | } |
| 1845 | |
| 1846 | case nir_intrinsic_load_output: { |
| 1847 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1848 | |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1849 | unsigned bits = nir_dest_bit_size(instr->dest); |
| 1850 | |
| 1851 | midgard_instruction ld; |
| 1852 | if (bits == 16) |
| 1853 | ld = m_ld_color_buffer_as_fp16(reg, 0); |
| 1854 | else |
| 1855 | ld = m_ld_color_buffer_as_fp32(reg, 0); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1856 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1857 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1858 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1859 | for (unsigned c = 4; c < 16; ++c) |
| 1860 | ld.swizzle[0][c] = 0; |
| 1861 | |
| 1862 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1863 | if (bits == 16) |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1864 | ld.op = midgard_op_ld_color_buffer_as_fp16_old; |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1865 | else |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1866 | ld.op = midgard_op_ld_color_buffer_as_fp32_old; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1867 | ld.load_store.address = 1; |
| 1868 | ld.load_store.arg_2 = 0x1E; |
| 1869 | } |
| 1870 | |
| 1871 | emit_mir_instruction(ctx, ld); |
| 1872 | break; |
| 1873 | } |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1874 | |
| 1875 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1876 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1877 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1878 | |
| 1879 | /* Blend constants are embedded directly in the shader and |
| 1880 | * patched in, so we use some magic routing */ |
| 1881 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1882 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1883 | ins.has_constants = true; |
| 1884 | ins.has_blend_constant = true; |
| 1885 | emit_mir_instruction(ctx, ins); |
| 1886 | break; |
| 1887 | } |
| 1888 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1889 | case nir_intrinsic_store_output: |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1890 | case nir_intrinsic_store_combined_output_pan: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1891 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1892 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1893 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1894 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1895 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1896 | |
| 1897 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1898 | bool combined = instr->intrinsic == |
| 1899 | nir_intrinsic_store_combined_output_pan; |
| 1900 | |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1901 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1902 | var = search_var(ctx->nir, nir_var_shader_out, |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1903 | nir_intrinsic_base(instr)); |
| 1904 | assert(var); |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1905 | |
| 1906 | /* Dual-source blend writeout is done by leaving the |
| 1907 | * value in r2 for the blend shader to use. */ |
| 1908 | if (var->data.index) { |
| 1909 | if (instr->src[0].is_ssa) { |
| 1910 | emit_explicit_constant(ctx, reg, reg); |
| 1911 | |
| 1912 | unsigned out = make_compiler_temp(ctx); |
| 1913 | |
| 1914 | midgard_instruction ins = v_mov(reg, out); |
| 1915 | emit_mir_instruction(ctx, ins); |
| 1916 | |
| 1917 | ctx->blend_src1 = out; |
| 1918 | } else { |
| 1919 | ctx->blend_src1 = reg; |
| 1920 | } |
| 1921 | |
| 1922 | break; |
| 1923 | } |
| 1924 | |
| 1925 | enum midgard_rt_id rt; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1926 | if (var->data.location == FRAG_RESULT_COLOR) |
| 1927 | rt = MIDGARD_COLOR_RT0; |
| 1928 | else if (var->data.location >= FRAG_RESULT_DATA0) |
| 1929 | rt = MIDGARD_COLOR_RT0 + var->data.location - |
| 1930 | FRAG_RESULT_DATA0; |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1931 | else if (combined) |
| 1932 | rt = MIDGARD_ZS_RT; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1933 | else |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 1934 | unreachable("bad rt"); |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1935 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1936 | unsigned reg_z = ~0, reg_s = ~0; |
| 1937 | if (combined) { |
| 1938 | unsigned writeout = nir_intrinsic_component(instr); |
| 1939 | if (writeout & PAN_WRITEOUT_Z) |
| 1940 | reg_z = nir_src_index(ctx, &instr->src[2]); |
| 1941 | if (writeout & PAN_WRITEOUT_S) |
| 1942 | reg_s = nir_src_index(ctx, &instr->src[3]); |
| 1943 | } |
| 1944 | |
| 1945 | emit_fragment_store(ctx, reg, reg_z, reg_s, rt); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1946 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1947 | assert(instr->intrinsic == nir_intrinsic_store_output); |
| 1948 | |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1949 | /* We should have been vectorized, though we don't |
| 1950 | * currently check that st_vary is emitted only once |
| 1951 | * per slot (this is relevant, since there's not a mask |
| 1952 | * parameter available on the store [set to 0 by the |
| 1953 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1954 | * swizzle. If this is a constant source, we'll need to |
| 1955 | * emit that explicitly. */ |
| 1956 | |
| 1957 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1958 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1959 | unsigned dst_component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | 2788721 | 2019-08-15 16:53:03 -0700 | [diff] [blame] | 1960 | unsigned nr_comp = nir_src_num_components(instr->src[0]); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1961 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1962 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1963 | st.load_store.arg_1 = 0x9E; |
| 1964 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1965 | |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1966 | switch (nir_alu_type_get_base_type(nir_intrinsic_type(instr))) { |
| 1967 | case nir_type_uint: |
| 1968 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1969 | st.op = midgard_op_st_vary_32u; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1970 | break; |
| 1971 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1972 | st.op = midgard_op_st_vary_32i; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1973 | break; |
| 1974 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1975 | st.op = midgard_op_st_vary_32; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1976 | break; |
| 1977 | default: |
| 1978 | unreachable("Attempted to store unknown type"); |
| 1979 | break; |
| 1980 | } |
| 1981 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1982 | /* nir_intrinsic_component(store_intr) encodes the |
| 1983 | * destination component start. Source component offset |
| 1984 | * adjustment is taken care of in |
| 1985 | * install_registers_instr(), when offset_swizzle() is |
| 1986 | * called. |
| 1987 | */ |
| 1988 | unsigned src_component = COMPONENT_X; |
| 1989 | |
| 1990 | assert(nr_comp > 0); |
| 1991 | for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) { |
| 1992 | st.swizzle[0][i] = src_component; |
| 1993 | if (i >= dst_component && i < dst_component + nr_comp - 1) |
| 1994 | src_component++; |
| 1995 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1996 | |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1997 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1998 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1999 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2000 | assert(0); |
| 2001 | } |
| 2002 | |
| 2003 | break; |
| 2004 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 2005 | /* Special case of store_output for lowered blend shaders */ |
| 2006 | case nir_intrinsic_store_raw_output_pan: |
| 2007 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 2008 | reg = nir_src_index(ctx, &instr->src[0]); |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 2009 | emit_fragment_store(ctx, reg, ~0, ~0, ctx->blend_rt); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 2010 | break; |
| 2011 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2012 | case nir_intrinsic_store_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2013 | case nir_intrinsic_store_shared: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2014 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2015 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2016 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 2017 | emit_global(ctx, &instr->instr, false, reg, &instr->src[1], instr->intrinsic == nir_intrinsic_store_shared); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2018 | break; |
| 2019 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2020 | case nir_intrinsic_load_ssbo_address: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2021 | emit_sysval_read(ctx, &instr->instr, 1, 0); |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 2022 | break; |
| 2023 | |
| 2024 | case nir_intrinsic_get_buffer_size: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2025 | emit_sysval_read(ctx, &instr->instr, 1, 8); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2026 | break; |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 2027 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2028 | case nir_intrinsic_load_viewport_scale: |
| 2029 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 2030 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 2031 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2032 | emit_sysval_read(ctx, &instr->instr, 3, 0); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2033 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2034 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 2035 | case nir_intrinsic_load_work_group_id: |
| 2036 | case nir_intrinsic_load_local_invocation_id: |
| 2037 | emit_compute_builtin(ctx, instr); |
| 2038 | break; |
| 2039 | |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 2040 | case nir_intrinsic_load_vertex_id: |
| 2041 | case nir_intrinsic_load_instance_id: |
| 2042 | emit_vertex_builtin(ctx, instr); |
| 2043 | break; |
| 2044 | |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2045 | case nir_intrinsic_load_sample_mask_in: |
| 2046 | emit_special(ctx, instr, 96); |
| 2047 | break; |
| 2048 | |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2049 | case nir_intrinsic_load_sample_id: |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2050 | emit_special(ctx, instr, 97); |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2051 | break; |
| 2052 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 2053 | case nir_intrinsic_memory_barrier_buffer: |
| 2054 | case nir_intrinsic_memory_barrier_shared: |
| 2055 | break; |
| 2056 | |
| 2057 | case nir_intrinsic_control_barrier: |
| 2058 | schedule_barrier(ctx); |
| 2059 | emit_control_barrier(ctx); |
| 2060 | schedule_barrier(ctx); |
| 2061 | break; |
| 2062 | |
Italo Nicola | d7b6d2e | 2020-08-31 17:32:30 +0000 | [diff] [blame] | 2063 | ATOMIC_CASE(ctx, instr, add, add); |
| 2064 | ATOMIC_CASE(ctx, instr, and, and); |
| 2065 | ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg); |
| 2066 | ATOMIC_CASE(ctx, instr, exchange, xchg); |
| 2067 | ATOMIC_CASE(ctx, instr, imax, imax); |
| 2068 | ATOMIC_CASE(ctx, instr, imin, imin); |
| 2069 | ATOMIC_CASE(ctx, instr, or, or); |
| 2070 | ATOMIC_CASE(ctx, instr, umax, umax); |
| 2071 | ATOMIC_CASE(ctx, instr, umin, umin); |
| 2072 | ATOMIC_CASE(ctx, instr, xor, xor); |
| 2073 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2074 | default: |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2075 | fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2076 | assert(0); |
| 2077 | break; |
| 2078 | } |
| 2079 | } |
| 2080 | |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2081 | /* Returns dimension with 0 special casing cubemaps */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2082 | static unsigned |
| 2083 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 2084 | { |
| 2085 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2086 | case GLSL_SAMPLER_DIM_1D: |
| 2087 | case GLSL_SAMPLER_DIM_BUF: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2088 | return 1; |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2089 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2090 | case GLSL_SAMPLER_DIM_2D: |
Alyssa Rosenzweig | a2748d4 | 2020-06-30 15:31:30 -0400 | [diff] [blame] | 2091 | case GLSL_SAMPLER_DIM_MS: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2092 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 2093 | case GLSL_SAMPLER_DIM_RECT: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2094 | return 2; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2095 | |
| 2096 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2097 | return 3; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2098 | |
| 2099 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2100 | return 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2101 | |
| 2102 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2103 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2104 | assert(0); |
| 2105 | return 0; |
| 2106 | } |
| 2107 | } |
| 2108 | |
Alyssa Rosenzweig | c6c906e | 2020-05-21 18:02:38 -0400 | [diff] [blame] | 2109 | /* Tries to attach an explicit LOD or bias as a constant. Returns whether this |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2110 | * was successful */ |
| 2111 | |
| 2112 | static bool |
| 2113 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2114 | compiler_context *ctx, |
| 2115 | nir_src lod, |
| 2116 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2117 | { |
| 2118 | /* To attach as constant, it has to *be* constant */ |
| 2119 | |
| 2120 | if (!nir_src_is_const(lod)) |
| 2121 | return false; |
| 2122 | |
| 2123 | float f = nir_src_as_float(lod); |
| 2124 | |
| 2125 | /* Break into fixed-point */ |
| 2126 | signed lod_int = f; |
| 2127 | float lod_frac = f - lod_int; |
| 2128 | |
| 2129 | /* Carry over negative fractions */ |
| 2130 | if (lod_frac < 0.0) { |
| 2131 | lod_int--; |
| 2132 | lod_frac += 1.0; |
| 2133 | } |
| 2134 | |
| 2135 | /* Encode */ |
| 2136 | word->bias = float_to_ubyte(lod_frac); |
| 2137 | word->bias_int = lod_int; |
| 2138 | |
| 2139 | return true; |
| 2140 | } |
| 2141 | |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2142 | static enum mali_texture_mode |
| 2143 | mdg_texture_mode(nir_tex_instr *instr) |
| 2144 | { |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2145 | if (instr->op == nir_texop_tg4 && instr->is_shadow) |
| 2146 | return TEXTURE_GATHER_SHADOW; |
| 2147 | else if (instr->op == nir_texop_tg4) |
| 2148 | return TEXTURE_GATHER_X + instr->component; |
| 2149 | else if (instr->is_shadow) |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2150 | return TEXTURE_SHADOW; |
| 2151 | else |
| 2152 | return TEXTURE_NORMAL; |
| 2153 | } |
| 2154 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2155 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2156 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2157 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2158 | { |
| 2159 | /* TODO */ |
| 2160 | //assert (!instr->sampler); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2161 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2162 | nir_dest *dest = &instr->dest; |
| 2163 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2164 | int texture_index = instr->texture_index; |
| 2165 | int sampler_index = texture_index; |
| 2166 | |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2167 | nir_alu_type dest_base = nir_alu_type_get_base_type(instr->dest_type); |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2168 | nir_alu_type dest_type = dest_base | nir_dest_bit_size(*dest); |
| 2169 | |
| 2170 | /* texture instructions support float outmods */ |
| 2171 | unsigned outmod = midgard_outmod_none; |
| 2172 | if (dest_base == nir_type_float) { |
| 2173 | outmod = mir_determine_float_outmod(ctx, &dest, 0); |
| 2174 | } |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2175 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2176 | midgard_instruction ins = { |
| 2177 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 2178 | .mask = 0xF, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2179 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2180 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2181 | .dest_type = dest_type, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2182 | .swizzle = SWIZZLE_IDENTITY_4, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2183 | .outmod = outmod, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 2184 | .op = midgard_texop, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2185 | .texture = { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2186 | .format = midgard_tex_format(instr->sampler_dim), |
| 2187 | .texture_handle = texture_index, |
| 2188 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2189 | .mode = mdg_texture_mode(instr) |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2190 | } |
| 2191 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 2192 | |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2193 | if (instr->is_shadow && !instr->is_new_style_shadow && instr->op != nir_texop_tg4) |
Icecream95 | d1290e7 | 2020-05-12 10:16:31 +1200 | [diff] [blame] | 2194 | for (int i = 0; i < 4; ++i) |
| 2195 | ins.swizzle[0][i] = COMPONENT_X; |
| 2196 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2197 | /* We may need a temporary for the coordinate */ |
| 2198 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2199 | bool needs_temp_coord = |
| 2200 | (midgard_texop == TEXTURE_OP_TEXEL_FETCH) || |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2201 | (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) || |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2202 | (instr->is_shadow); |
| 2203 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2204 | unsigned coords = needs_temp_coord ? make_compiler_temp_reg(ctx) : 0; |
| 2205 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2206 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2207 | int index = nir_src_index(ctx, &instr->src[i].src); |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2208 | unsigned nr_components = nir_src_num_components(instr->src[i].src); |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2209 | unsigned sz = nir_src_bit_size(instr->src[i].src); |
| 2210 | nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2211 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2212 | switch (instr->src[i].src_type) { |
| 2213 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2214 | emit_explicit_constant(ctx, index, index); |
| 2215 | |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2216 | unsigned coord_mask = mask_of(instr->coord_components); |
| 2217 | |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2218 | bool flip_zw = (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) && (coord_mask & (1 << COMPONENT_Z)); |
| 2219 | |
| 2220 | if (flip_zw) |
| 2221 | coord_mask ^= ((1 << COMPONENT_Z) | (1 << COMPONENT_W)); |
| 2222 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2223 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
| 2224 | /* texelFetch is undefined on samplerCube */ |
| 2225 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 2226 | |
| 2227 | /* For cubemaps, we use a special ld/st op to |
| 2228 | * select the face and copy the xy into the |
| 2229 | * texture register */ |
| 2230 | |
| 2231 | midgard_instruction ld = m_ld_cubemap_coords(coords, 0); |
| 2232 | ld.src[1] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2233 | ld.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2234 | ld.mask = 0x3; /* xy */ |
| 2235 | ld.load_store.arg_1 = 0x20; |
| 2236 | ld.swizzle[1][3] = COMPONENT_X; |
| 2237 | emit_mir_instruction(ctx, ld); |
| 2238 | |
| 2239 | /* xyzw -> xyxx */ |
| 2240 | ins.swizzle[1][2] = instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
| 2241 | ins.swizzle[1][3] = COMPONENT_X; |
| 2242 | } else if (needs_temp_coord) { |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2243 | /* mov coord_temp, coords */ |
| 2244 | midgard_instruction mov = v_mov(index, coords); |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2245 | mov.mask = coord_mask; |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2246 | |
| 2247 | if (flip_zw) |
| 2248 | mov.swizzle[1][COMPONENT_W] = COMPONENT_Z; |
| 2249 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2250 | emit_mir_instruction(ctx, mov); |
| 2251 | } else { |
| 2252 | coords = index; |
| 2253 | } |
| 2254 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2255 | ins.src[1] = coords; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2256 | ins.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2257 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2258 | /* Texelfetch coordinates uses all four elements |
| 2259 | * (xyz/index) regardless of texture dimensionality, |
| 2260 | * which means it's necessary to zero the unused |
| 2261 | * components to keep everything happy */ |
| 2262 | |
| 2263 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2264 | /* mov index.zw, #0, or generalized */ |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2265 | midgard_instruction mov = |
| 2266 | v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), coords); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2267 | mov.has_constants = true; |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2268 | mov.mask = coord_mask ^ 0xF; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2269 | emit_mir_instruction(ctx, mov); |
| 2270 | } |
| 2271 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2272 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2273 | /* Array component in w but NIR wants it in z, |
| 2274 | * but if we have a temp coord we already fixed |
| 2275 | * that up */ |
| 2276 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2277 | if (nr_components == 3) { |
| 2278 | ins.swizzle[1][2] = COMPONENT_Z; |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2279 | ins.swizzle[1][3] = needs_temp_coord ? COMPONENT_W : COMPONENT_Z; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2280 | } else if (nr_components == 2) { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2281 | ins.swizzle[1][2] = |
| 2282 | instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2283 | ins.swizzle[1][3] = COMPONENT_X; |
| 2284 | } else |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2285 | unreachable("Invalid texture 2D components"); |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 2286 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2287 | |
Alyssa Rosenzweig | 64b2fe9 | 2019-12-20 12:38:24 -0500 | [diff] [blame] | 2288 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 2289 | /* We zeroed */ |
| 2290 | ins.swizzle[1][2] = COMPONENT_Z; |
| 2291 | ins.swizzle[1][3] = COMPONENT_W; |
| 2292 | } |
| 2293 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2294 | break; |
| 2295 | } |
| 2296 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 2297 | case nir_tex_src_bias: |
| 2298 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2299 | /* Try as a constant if we can */ |
| 2300 | |
| 2301 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 2302 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 2303 | break; |
| 2304 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2305 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2306 | ins.src[2] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2307 | ins.src_types[2] = T; |
Alyssa Rosenzweig | 72e5749 | 2019-12-20 12:34:20 -0500 | [diff] [blame] | 2308 | |
| 2309 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2310 | ins.swizzle[2][c] = COMPONENT_X; |
| 2311 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2312 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2313 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2314 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2315 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2316 | |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2317 | case nir_tex_src_offset: { |
| 2318 | ins.texture.offset_register = true; |
| 2319 | ins.src[3] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2320 | ins.src_types[3] = T; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2321 | |
| 2322 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2323 | ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c; |
| 2324 | |
| 2325 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | 4ec1f95 | 2019-12-20 12:58:10 -0500 | [diff] [blame] | 2326 | break; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2327 | }; |
| 2328 | |
Alyssa Rosenzweig | 6d9f951 | 2020-06-30 15:31:39 -0400 | [diff] [blame] | 2329 | case nir_tex_src_comparator: |
| 2330 | case nir_tex_src_ms_index: { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2331 | unsigned comp = COMPONENT_Z; |
| 2332 | |
| 2333 | /* mov coord_temp.foo, coords */ |
| 2334 | midgard_instruction mov = v_mov(index, coords); |
| 2335 | mov.mask = 1 << comp; |
| 2336 | |
| 2337 | for (unsigned i = 0; i < MIR_VEC_COMPONENTS; ++i) |
| 2338 | mov.swizzle[1][i] = COMPONENT_X; |
| 2339 | |
| 2340 | emit_mir_instruction(ctx, mov); |
| 2341 | break; |
| 2342 | } |
| 2343 | |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2344 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2345 | fprintf(stderr, "Unknown texture source type: %d\n", instr->src[i].src_type); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2346 | assert(0); |
| 2347 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2348 | } |
| 2349 | } |
| 2350 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2351 | emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2352 | } |
| 2353 | |
| 2354 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2355 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 2356 | { |
| 2357 | switch (instr->op) { |
| 2358 | case nir_texop_tex: |
| 2359 | case nir_texop_txb: |
| 2360 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 2361 | break; |
| 2362 | case nir_texop_txl: |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2363 | case nir_texop_tg4: |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2364 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 2365 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2366 | case nir_texop_txf: |
Alyssa Rosenzweig | 63a8722 | 2020-06-30 15:32:01 -0400 | [diff] [blame] | 2367 | case nir_texop_txf_ms: |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2368 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 2369 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2370 | case nir_texop_txs: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2371 | emit_sysval_read(ctx, &instr->instr, 4, 0); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2372 | break; |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2373 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2374 | fprintf(stderr, "Unhandled texture op: %d\n", instr->op); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2375 | assert(0); |
| 2376 | } |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2377 | } |
| 2378 | } |
| 2379 | |
| 2380 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2381 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 2382 | { |
| 2383 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2384 | case nir_jump_break: { |
| 2385 | /* Emit a branch out of the loop */ |
| 2386 | struct midgard_instruction br = v_branch(false, false); |
| 2387 | br.branch.target_type = TARGET_BREAK; |
| 2388 | br.branch.target_break = ctx->current_loop_depth; |
| 2389 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2390 | break; |
| 2391 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2392 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2393 | default: |
| 2394 | DBG("Unknown jump type %d\n", instr->type); |
| 2395 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2396 | } |
| 2397 | } |
| 2398 | |
| 2399 | static void |
| 2400 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 2401 | { |
| 2402 | switch (instr->type) { |
| 2403 | case nir_instr_type_load_const: |
| 2404 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 2405 | break; |
| 2406 | |
| 2407 | case nir_instr_type_intrinsic: |
| 2408 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 2409 | break; |
| 2410 | |
| 2411 | case nir_instr_type_alu: |
| 2412 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 2413 | break; |
| 2414 | |
| 2415 | case nir_instr_type_tex: |
| 2416 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 2417 | break; |
| 2418 | |
| 2419 | case nir_instr_type_jump: |
| 2420 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 2421 | break; |
| 2422 | |
| 2423 | case nir_instr_type_ssa_undef: |
| 2424 | /* Spurious */ |
| 2425 | break; |
| 2426 | |
| 2427 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2428 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2429 | break; |
| 2430 | } |
| 2431 | } |
| 2432 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2433 | |
| 2434 | /* ALU instructions can inline or embed constants, which decreases register |
| 2435 | * pressure and saves space. */ |
| 2436 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2437 | #define CONDITIONAL_ATTACH(idx) { \ |
| 2438 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2439 | \ |
| 2440 | if (entry) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2441 | attach_constants(ctx, alu, entry, alu->src[idx] + 1); \ |
| 2442 | alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2443 | } \ |
| 2444 | } |
| 2445 | |
| 2446 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2447 | inline_alu_constants(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2448 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2449 | mir_foreach_instr_in_block(block, alu) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2450 | /* Other instructions cannot inline constants */ |
| 2451 | if (alu->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 2452 | if (alu->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2453 | |
| 2454 | /* If there is already a constant here, we can do nothing */ |
| 2455 | if (alu->has_constants) continue; |
| 2456 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2457 | CONDITIONAL_ATTACH(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2458 | |
| 2459 | if (!alu->has_constants) { |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2460 | CONDITIONAL_ATTACH(1) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2461 | } else if (!alu->inline_constant) { |
| 2462 | /* Corner case: _two_ vec4 constants, for instance with a |
| 2463 | * csel. For this case, we can only use a constant |
| 2464 | * register for one, we'll have to emit a move for the |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2465 | * other. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2466 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2467 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1); |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2468 | unsigned scratch = make_compiler_temp(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2469 | |
| 2470 | if (entry) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 2471 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch); |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2472 | attach_constants(ctx, &ins, entry, alu->src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2473 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2474 | /* Set the source */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2475 | alu->src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2476 | |
| 2477 | /* Inject us -before- the last instruction which set r31 */ |
Boris Brezillon | 938c5b0 | 2019-08-28 09:17:21 +0200 | [diff] [blame] | 2478 | mir_insert_instruction_before(ctx, mir_prev_op(alu), ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2479 | } |
| 2480 | } |
| 2481 | } |
| 2482 | } |
| 2483 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2484 | unsigned |
| 2485 | max_bitsize_for_alu(midgard_instruction *ins) |
| 2486 | { |
| 2487 | unsigned max_bitsize = 0; |
| 2488 | for (int i = 0; i < MIR_SRC_COUNT; i++) { |
| 2489 | if (ins->src[i] == ~0) continue; |
| 2490 | unsigned src_bitsize = nir_alu_type_get_type_size(ins->src_types[i]); |
| 2491 | max_bitsize = MAX2(src_bitsize, max_bitsize); |
| 2492 | } |
| 2493 | unsigned dst_bitsize = nir_alu_type_get_type_size(ins->dest_type); |
| 2494 | max_bitsize = MAX2(dst_bitsize, max_bitsize); |
| 2495 | |
| 2496 | /* We don't have fp16 LUTs, so we'll want to emit code like: |
| 2497 | * |
| 2498 | * vlut.fsinr hr0, hr0 |
| 2499 | * |
| 2500 | * where both input and output are 16-bit but the operation is carried |
| 2501 | * out in 32-bit |
| 2502 | */ |
| 2503 | |
| 2504 | switch (ins->op) { |
| 2505 | case midgard_alu_op_fsqrt: |
| 2506 | case midgard_alu_op_frcp: |
| 2507 | case midgard_alu_op_frsqrt: |
| 2508 | case midgard_alu_op_fsin: |
| 2509 | case midgard_alu_op_fcos: |
| 2510 | case midgard_alu_op_fexp2: |
| 2511 | case midgard_alu_op_flog2: |
| 2512 | max_bitsize = MAX2(max_bitsize, 32); |
| 2513 | break; |
| 2514 | |
| 2515 | default: |
| 2516 | break; |
| 2517 | } |
| 2518 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 2519 | /* High implies computing at a higher bitsize, e.g umul_high of 32-bit |
| 2520 | * requires computing at 64-bit */ |
| 2521 | if (midgard_is_integer_out_op(ins->op) && ins->outmod == midgard_outmod_int_high) { |
| 2522 | max_bitsize *= 2; |
| 2523 | assert(max_bitsize <= 64); |
| 2524 | } |
| 2525 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2526 | return max_bitsize; |
| 2527 | } |
| 2528 | |
| 2529 | midgard_reg_mode |
| 2530 | reg_mode_for_bitsize(unsigned bitsize) |
| 2531 | { |
| 2532 | switch (bitsize) { |
| 2533 | /* use 16 pipe for 8 since we don't support vec16 yet */ |
| 2534 | case 8: |
| 2535 | case 16: |
| 2536 | return midgard_reg_mode_16; |
| 2537 | case 32: |
| 2538 | return midgard_reg_mode_32; |
| 2539 | case 64: |
| 2540 | return midgard_reg_mode_64; |
| 2541 | default: |
| 2542 | unreachable("invalid bit size"); |
| 2543 | } |
| 2544 | } |
| 2545 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2546 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2547 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2548 | * constants can be demoted to inline constants, for space savings and |
| 2549 | * sometimes a performance boost */ |
| 2550 | |
| 2551 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2552 | embedded_to_inline_constant(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2553 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2554 | mir_foreach_instr_in_block(block, ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2555 | if (!ins->has_constants) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2556 | if (ins->has_inline_constant) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2557 | |
| 2558 | /* Blend constants must not be inlined by definition */ |
| 2559 | if (ins->has_blend_constant) continue; |
| 2560 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2561 | unsigned max_bitsize = max_bitsize_for_alu(ins); |
| 2562 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2563 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2564 | bool is_16 = max_bitsize == 16; |
| 2565 | bool is_32 = max_bitsize == 32; |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2566 | |
| 2567 | if (!(is_16 || is_32)) |
| 2568 | continue; |
| 2569 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2570 | /* src1 cannot be an inline constant due to encoding |
| 2571 | * restrictions. So, if possible we try to flip the arguments |
| 2572 | * in that case */ |
| 2573 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2574 | int op = ins->op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2575 | |
Alyssa Rosenzweig | ba9f3d1 | 2020-04-30 13:11:52 -0400 | [diff] [blame] | 2576 | if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT) && |
| 2577 | alu_opcode_props[op].props & OP_COMMUTES) { |
| 2578 | mir_flip(ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2579 | } |
| 2580 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2581 | if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2582 | /* Component is from the swizzle. Take a nonzero component */ |
| 2583 | assert(ins->mask); |
| 2584 | unsigned first_comp = ffs(ins->mask) - 1; |
| 2585 | unsigned component = ins->swizzle[1][first_comp]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2586 | |
| 2587 | /* Scale constant appropriately, if we can legally */ |
Icecream95 | d97aaad | 2020-06-05 20:17:27 +1200 | [diff] [blame] | 2588 | int16_t scaled_constant = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2589 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2590 | if (is_16) { |
| 2591 | scaled_constant = ins->constants.u16[component]; |
| 2592 | } else if (midgard_is_integer_op(op)) { |
| 2593 | scaled_constant = ins->constants.u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2594 | |
| 2595 | /* Constant overflow after resize */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2596 | if (scaled_constant != ins->constants.u32[component]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2597 | continue; |
| 2598 | } else { |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2599 | float original = ins->constants.f32[component]; |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2600 | scaled_constant = _mesa_float_to_half(original); |
| 2601 | |
| 2602 | /* Check for loss of precision. If this is |
| 2603 | * mediump, we don't care, but for a highp |
| 2604 | * shader, we need to pay attention. NIR |
| 2605 | * doesn't yet tell us which mode we're in! |
| 2606 | * Practically this prevents most constants |
| 2607 | * from being inlined, sadly. */ |
| 2608 | |
| 2609 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2610 | |
| 2611 | if (fp32 != original) |
| 2612 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2613 | } |
| 2614 | |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 2615 | /* Should've been const folded */ |
| 2616 | if (ins->src_abs[1] || ins->src_neg[1]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2617 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2618 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2619 | /* Make sure that the constant is not itself a vector |
| 2620 | * by checking if all accessed values are the same. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2621 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2622 | const midgard_constants *cons = &ins->constants; |
| 2623 | uint32_t value = is_16 ? cons->u16[component] : cons->u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2624 | |
| 2625 | bool is_vector = false; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2626 | unsigned mask = effective_writemask(ins->op, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2627 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2628 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2629 | /* We only care if this component is actually used */ |
| 2630 | if (!(mask & (1 << c))) |
| 2631 | continue; |
| 2632 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2633 | uint32_t test = is_16 ? |
| 2634 | cons->u16[ins->swizzle[1][c]] : |
| 2635 | cons->u32[ins->swizzle[1][c]]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2636 | |
| 2637 | if (test != value) { |
| 2638 | is_vector = true; |
| 2639 | break; |
| 2640 | } |
| 2641 | } |
| 2642 | |
| 2643 | if (is_vector) |
| 2644 | continue; |
| 2645 | |
| 2646 | /* Get rid of the embedded constant */ |
| 2647 | ins->has_constants = false; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2648 | ins->src[1] = ~0; |
| 2649 | ins->has_inline_constant = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2650 | ins->inline_constant = scaled_constant; |
| 2651 | } |
| 2652 | } |
| 2653 | } |
| 2654 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2655 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2656 | * per block is legal semantically */ |
| 2657 | |
| 2658 | static void |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 2659 | midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2660 | { |
| 2661 | bool branched = false; |
| 2662 | |
| 2663 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2664 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2665 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2666 | if (branched) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2667 | mir_remove_instruction(ins); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2668 | |
| 2669 | branched = true; |
| 2670 | } |
| 2671 | } |
| 2672 | |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2673 | /* We want to force the invert on AND/OR to the second slot to legalize into |
| 2674 | * iandnot/iornot. The relevant patterns are for AND (and OR respectively) |
| 2675 | * |
| 2676 | * ~a & #b = ~a & ~(#~b) |
| 2677 | * ~a & b = b & ~a |
| 2678 | */ |
| 2679 | |
| 2680 | static void |
| 2681 | midgard_legalize_invert(compiler_context *ctx, midgard_block *block) |
| 2682 | { |
| 2683 | mir_foreach_instr_in_block(block, ins) { |
| 2684 | if (ins->type != TAG_ALU_4) continue; |
| 2685 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2686 | if (ins->op != midgard_alu_op_iand && |
| 2687 | ins->op != midgard_alu_op_ior) continue; |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2688 | |
| 2689 | if (ins->src_invert[1] || !ins->src_invert[0]) continue; |
| 2690 | |
| 2691 | if (ins->has_inline_constant) { |
| 2692 | /* ~(#~a) = ~(~#a) = a, so valid, and forces both |
| 2693 | * inverts on */ |
| 2694 | ins->inline_constant = ~ins->inline_constant; |
| 2695 | ins->src_invert[1] = true; |
| 2696 | } else { |
| 2697 | /* Flip to the right invert order. Note |
| 2698 | * has_inline_constant false by assumption on the |
| 2699 | * branch, so flipping makes sense. */ |
| 2700 | mir_flip(ins); |
| 2701 | } |
| 2702 | } |
| 2703 | } |
| 2704 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2705 | static unsigned |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2706 | emit_fragment_epilogue(compiler_context *ctx, unsigned rt) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2707 | { |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2708 | /* Loop to ourselves */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2709 | midgard_instruction *br = ctx->writeout_branch[rt]; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2710 | struct midgard_instruction ins = v_branch(false, false); |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 2711 | ins.writeout = br->writeout; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2712 | ins.branch.target_block = ctx->block_count - 1; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2713 | ins.constants.u32[0] = br->constants.u32[0]; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 2714 | memcpy(&ins.src_types, &br->src_types, sizeof(ins.src_types)); |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2715 | emit_mir_instruction(ctx, ins); |
| 2716 | |
Alyssa Rosenzweig | 3448b26 | 2019-12-03 10:37:01 -0500 | [diff] [blame] | 2717 | ctx->current_block->epilogue = true; |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2718 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2719 | return ins.branch.target_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2720 | } |
| 2721 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2722 | static midgard_block * |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2723 | emit_block_init(compiler_context *ctx) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2724 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2725 | midgard_block *this_block = ctx->after_block; |
| 2726 | ctx->after_block = NULL; |
| 2727 | |
| 2728 | if (!this_block) |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2729 | this_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2730 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2731 | list_addtail(&this_block->base.link, &ctx->blocks); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2732 | |
Alyssa Rosenzweig | c5dd1d5 | 2020-03-11 08:22:08 -0400 | [diff] [blame] | 2733 | this_block->scheduled = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2734 | ++ctx->block_count; |
| 2735 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2736 | /* Set up current block */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2737 | list_inithead(&this_block->base.instructions); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2738 | ctx->current_block = this_block; |
| 2739 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2740 | return this_block; |
| 2741 | } |
| 2742 | |
| 2743 | static midgard_block * |
| 2744 | emit_block(compiler_context *ctx, nir_block *block) |
| 2745 | { |
| 2746 | midgard_block *this_block = emit_block_init(ctx); |
| 2747 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2748 | nir_foreach_instr(instr, block) { |
| 2749 | emit_instr(ctx, instr); |
| 2750 | ++ctx->instruction_count; |
| 2751 | } |
| 2752 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2753 | return this_block; |
| 2754 | } |
| 2755 | |
| 2756 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2757 | |
| 2758 | static void |
| 2759 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2760 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2761 | midgard_block *before_block = ctx->current_block; |
| 2762 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2763 | /* Speculatively emit the branch, but we can't fill it in until later */ |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2764 | bool inv = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2765 | EMIT(branch, true, true); |
| 2766 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2767 | then_branch->src[0] = mir_get_branch_cond(&nif->condition, &inv); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 2768 | then_branch->src_types[0] = nir_type_uint32; |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2769 | then_branch->branch.invert_conditional = !inv; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2770 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2771 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2772 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2773 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2774 | |
| 2775 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2776 | EMIT(branch, false, false); |
| 2777 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2778 | |
| 2779 | /* Emit second block, and check if it's empty */ |
| 2780 | |
| 2781 | int else_idx = ctx->block_count; |
| 2782 | int count_in = ctx->instruction_count; |
| 2783 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2784 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2785 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2786 | |
| 2787 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2788 | |
| 2789 | assert(then_block); |
| 2790 | assert(else_block); |
| 2791 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2792 | if (ctx->instruction_count == count_in) { |
| 2793 | /* The else block is empty, so don't emit an exit jump */ |
| 2794 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2795 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2796 | } else { |
| 2797 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2798 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2799 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2800 | |
| 2801 | /* Wire up the successors */ |
| 2802 | |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2803 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2804 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2805 | pan_block_add_successor(&before_block->base, &then_block->base); |
| 2806 | pan_block_add_successor(&before_block->base, &else_block->base); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2807 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2808 | pan_block_add_successor(&end_then_block->base, &ctx->after_block->base); |
| 2809 | pan_block_add_successor(&end_else_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2810 | } |
| 2811 | |
| 2812 | static void |
| 2813 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2814 | { |
| 2815 | /* Remember where we are */ |
| 2816 | midgard_block *start_block = ctx->current_block; |
| 2817 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2818 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2819 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2820 | |
| 2821 | /* Get index from before the body so we can loop back later */ |
| 2822 | int start_idx = ctx->block_count; |
| 2823 | |
| 2824 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2825 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2826 | |
| 2827 | /* Branch back to loop back */ |
| 2828 | struct midgard_instruction br_back = v_branch(false, false); |
| 2829 | br_back.branch.target_block = start_idx; |
| 2830 | emit_mir_instruction(ctx, br_back); |
| 2831 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2832 | /* Mark down that branch in the graph. */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2833 | pan_block_add_successor(&start_block->base, &loop_block->base); |
| 2834 | pan_block_add_successor(&ctx->current_block->base, &loop_block->base); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2835 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2836 | /* Find the index of the block about to follow us (note: we don't add |
| 2837 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2838 | int break_block_idx = ctx->block_count; |
| 2839 | |
| 2840 | /* Fix up the break statements we emitted to point to the right place, |
| 2841 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2842 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2843 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2844 | mir_foreach_block_from(ctx, start_block, _block) { |
| 2845 | mir_foreach_instr_in_block(((midgard_block *) _block), ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2846 | if (ins->type != TAG_ALU_4) continue; |
| 2847 | if (!ins->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2848 | |
| 2849 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2850 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2851 | |
| 2852 | /* It's a break! Check if it's our break */ |
| 2853 | if (ins->branch.target_break != loop_idx) continue; |
| 2854 | |
| 2855 | /* Okay, cool, we're breaking out of this loop. |
| 2856 | * Rewrite from a break to a goto */ |
| 2857 | |
| 2858 | ins->branch.target_type = TARGET_GOTO; |
| 2859 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2860 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2861 | pan_block_add_successor(_block, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2862 | } |
| 2863 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2864 | |
| 2865 | /* Now that we've finished emitting the loop, free up the depth again |
| 2866 | * so we play nice with recursion amid nested loops */ |
| 2867 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2868 | |
| 2869 | /* Dump loop stats */ |
| 2870 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2871 | } |
| 2872 | |
| 2873 | static midgard_block * |
| 2874 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2875 | { |
| 2876 | midgard_block *start_block = NULL; |
| 2877 | |
| 2878 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2879 | switch (node->type) { |
| 2880 | case nir_cf_node_block: { |
| 2881 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2882 | |
| 2883 | if (!start_block) |
| 2884 | start_block = block; |
| 2885 | |
| 2886 | break; |
| 2887 | } |
| 2888 | |
| 2889 | case nir_cf_node_if: |
| 2890 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2891 | break; |
| 2892 | |
| 2893 | case nir_cf_node_loop: |
| 2894 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2895 | break; |
| 2896 | |
| 2897 | case nir_cf_node_function: |
| 2898 | assert(0); |
| 2899 | break; |
| 2900 | } |
| 2901 | } |
| 2902 | |
| 2903 | return start_block; |
| 2904 | } |
| 2905 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2906 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2907 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2908 | * until we find one that 'works' */ |
| 2909 | |
Italo Nicola | 8150c1d | 2020-07-29 20:14:55 +0000 | [diff] [blame] | 2910 | unsigned |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2911 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2912 | { |
| 2913 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2914 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2915 | mir_foreach_block_from(ctx, initial_block, _v) { |
| 2916 | midgard_block *v = (midgard_block *) _v; |
Alyssa Rosenzweig | 45ac8ea | 2019-11-04 10:32:49 -0500 | [diff] [blame] | 2917 | if (v->quadword_count) { |
| 2918 | midgard_bundle *initial_bundle = |
| 2919 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2920 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2921 | return initial_bundle->tag; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2922 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2923 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2924 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2925 | /* Default to a tag 1 which will break from the shader, in case we jump |
| 2926 | * to the exit block (i.e. `return` in a compute shader) */ |
| 2927 | |
| 2928 | return 1; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2929 | } |
| 2930 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2931 | /* For each fragment writeout instruction, generate a writeout loop to |
| 2932 | * associate with it */ |
| 2933 | |
| 2934 | static void |
| 2935 | mir_add_writeout_loops(compiler_context *ctx) |
| 2936 | { |
| 2937 | for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) { |
| 2938 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 2939 | if (!br) continue; |
| 2940 | |
| 2941 | unsigned popped = br->branch.target_block; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2942 | pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base), &ctx->current_block->base); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2943 | br->branch.target_block = emit_fragment_epilogue(ctx, rt); |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2944 | br->branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2945 | |
| 2946 | /* If we have more RTs, we'll need to restore back after our |
| 2947 | * loop terminates */ |
| 2948 | |
| 2949 | if ((rt + 1) < ARRAY_SIZE(ctx->writeout_branch) && ctx->writeout_branch[rt + 1]) { |
| 2950 | midgard_instruction uncond = v_branch(false, false); |
| 2951 | uncond.branch.target_block = popped; |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2952 | uncond.branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2953 | emit_mir_instruction(ctx, uncond); |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2954 | pan_block_add_successor(&ctx->current_block->base, &(mir_get_block(ctx, popped)->base)); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2955 | schedule_barrier(ctx); |
| 2956 | } else { |
| 2957 | /* We're last, so we can terminate here */ |
| 2958 | br->last_writeout = true; |
| 2959 | } |
| 2960 | } |
| 2961 | } |
| 2962 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2963 | int |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 2964 | midgard_compile_shader_nir(nir_shader *nir, panfrost_program *program, bool is_blend, unsigned blend_rt, unsigned gpu_id, bool shaderdb, bool silent) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2965 | { |
| 2966 | struct util_dynarray *compiled = &program->compiled; |
| 2967 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2968 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2969 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2970 | /* TODO: Bound against what? */ |
| 2971 | compiler_context *ctx = rzalloc(NULL, compiler_context); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2972 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2973 | ctx->nir = nir; |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2974 | ctx->stage = nir->info.stage; |
| 2975 | ctx->is_blend = is_blend; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2976 | ctx->blend_rt = MIDGARD_COLOR_RT0 + blend_rt; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 2977 | ctx->blend_input = ~0; |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 2978 | ctx->blend_src1 = ~0; |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 2979 | ctx->quirks = midgard_get_quirks(gpu_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2980 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2981 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2982 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2983 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2984 | ctx->uniform_cutoff = 8; |
| 2985 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2986 | /* Initialize at a global (not block) level hash tables */ |
| 2987 | |
| 2988 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2989 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2990 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2991 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2992 | * messed with our I/O quite a bit already) */ |
| 2993 | |
| 2994 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2995 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2996 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2997 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | 2023716 | 2019-08-26 12:14:11 -0700 | [diff] [blame] | 2998 | NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2999 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3000 | |
| 3001 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3002 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 3003 | NIR_PASS_V(nir, nir_split_var_copies); |
| 3004 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3005 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 3006 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 3007 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 3008 | |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3009 | unsigned pan_quirks = panfrost_get_quirks(gpu_id); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3010 | NIR_PASS_V(nir, pan_lower_framebuffer, |
Icecream95 | 2fa60b7 | 2020-07-06 19:34:49 +1200 | [diff] [blame] | 3011 | program->rt_formats, is_blend, pan_quirks); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3012 | |
Jason Ekstrand | b019b22 | 2020-06-10 17:54:25 -0500 | [diff] [blame] | 3013 | NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out, |
| 3014 | glsl_type_size, 0); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 3015 | NIR_PASS_V(nir, nir_lower_ssbo); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 3016 | NIR_PASS_V(nir, midgard_nir_lower_zs_store); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3017 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3018 | /* Optimisation passes */ |
| 3019 | |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 3020 | optimise_nir(nir, ctx->quirks, is_blend); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3021 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 3022 | NIR_PASS_V(nir, midgard_nir_reorder_writeout); |
| 3023 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3024 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !silent) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3025 | nir_print_shader(nir, stdout); |
| 3026 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3027 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 3028 | /* Assign sysvals and counts, now that we're sure |
| 3029 | * (post-optimisation) */ |
| 3030 | |
Alyssa Rosenzweig | 680fb05 | 2020-08-18 08:31:42 -0400 | [diff] [blame] | 3031 | panfrost_nir_assign_sysvals(&ctx->sysvals, ctx, nir); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 3032 | program->sysval_count = ctx->sysvals.sysval_count; |
| 3033 | memcpy(program->sysvals, ctx->sysvals.sysvals, sizeof(ctx->sysvals.sysvals[0]) * ctx->sysvals.sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3034 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3035 | nir_foreach_function(func, nir) { |
| 3036 | if (!func->impl) |
| 3037 | continue; |
| 3038 | |
| 3039 | list_inithead(&ctx->blocks); |
| 3040 | ctx->block_count = 0; |
| 3041 | ctx->func = func; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3042 | ctx->already_emitted = calloc(BITSET_WORDS(func->impl->ssa_alloc), sizeof(BITSET_WORD)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3043 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 3044 | if (nir->info.outputs_read && !is_blend) { |
| 3045 | emit_block_init(ctx); |
| 3046 | |
| 3047 | struct midgard_instruction wait = v_branch(false, false); |
| 3048 | wait.branch.target_type = TARGET_TILEBUF_WAIT; |
| 3049 | |
| 3050 | emit_mir_instruction(ctx, wait); |
| 3051 | |
| 3052 | ++ctx->instruction_count; |
| 3053 | } |
| 3054 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3055 | emit_cf_list(ctx, &func->impl->body); |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3056 | free(ctx->already_emitted); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3057 | break; /* TODO: Multi-function shaders */ |
| 3058 | } |
| 3059 | |
| 3060 | util_dynarray_init(compiled, NULL); |
| 3061 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3062 | /* Per-block lowering before opts */ |
| 3063 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3064 | mir_foreach_block(ctx, _block) { |
| 3065 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3066 | inline_alu_constants(ctx, block); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3067 | embedded_to_inline_constant(ctx, block); |
| 3068 | } |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3069 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3070 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3071 | bool progress = false; |
| 3072 | |
| 3073 | do { |
| 3074 | progress = false; |
Alyssa Rosenzweig | fc06b8b | 2020-05-06 17:34:09 -0400 | [diff] [blame] | 3075 | progress |= midgard_opt_dead_code_eliminate(ctx); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3076 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3077 | mir_foreach_block(ctx, _block) { |
| 3078 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3079 | progress |= midgard_opt_copy_prop(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 3080 | progress |= midgard_opt_combine_projection(ctx, block); |
| 3081 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3082 | } |
| 3083 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3084 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3085 | mir_foreach_block(ctx, _block) { |
| 3086 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 3087 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 3088 | midgard_legalize_invert(ctx, block); |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 3089 | midgard_cull_dead_branch(ctx, block); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 3090 | } |
| 3091 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3092 | if (ctx->stage == MESA_SHADER_FRAGMENT) |
| 3093 | mir_add_writeout_loops(ctx); |
| 3094 | |
Alyssa Rosenzweig | 9a7f0e2 | 2020-05-12 13:26:32 -0400 | [diff] [blame] | 3095 | /* Analyze now that the code is known but before scheduling creates |
| 3096 | * pipeline registers which are harder to track */ |
| 3097 | mir_analyze_helper_terminate(ctx); |
| 3098 | mir_analyze_helper_requirements(ctx); |
| 3099 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3100 | /* Schedule! */ |
Robert Foss | 62adb65 | 2020-01-15 01:14:16 +0100 | [diff] [blame] | 3101 | midgard_schedule_program(ctx); |
Alyssa Rosenzweig | 9dc3b18 | 2019-12-06 09:32:38 -0500 | [diff] [blame] | 3102 | mir_ra(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3103 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3104 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 3105 | * sequence. Save instruction boundaries such that lookahead tags can |
| 3106 | * be assigned easily */ |
| 3107 | |
| 3108 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 3109 | |
| 3110 | int bundle_count = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3111 | mir_foreach_block(ctx, _block) { |
| 3112 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3113 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 3114 | } |
| 3115 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 3116 | int bundle_idx = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3117 | mir_foreach_block(ctx, _block) { |
| 3118 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3119 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 3120 | source_order_bundles[bundle_idx++] = bundle; |
| 3121 | } |
| 3122 | } |
| 3123 | |
| 3124 | int current_bundle = 0; |
| 3125 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3126 | /* Midgard prefetches instruction types, so during emission we |
| 3127 | * need to lookahead. Unless this is the last instruction, in |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3128 | * which we return 1. */ |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3129 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3130 | mir_foreach_block(ctx, _block) { |
| 3131 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 3132 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3133 | int lookahead = 1; |
| 3134 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3135 | if (!bundle->last_writeout && (current_bundle + 1 < bundle_count)) |
| 3136 | lookahead = source_order_bundles[current_bundle + 1]->tag; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3137 | |
Alyssa Rosenzweig | 30a393f | 2020-05-21 19:14:23 -0400 | [diff] [blame] | 3138 | emit_binary_bundle(ctx, block, bundle, compiled, lookahead); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3139 | ++current_bundle; |
| 3140 | } |
| 3141 | |
| 3142 | /* TODO: Free deeper */ |
| 3143 | //util_dynarray_fini(&block->instructions); |
| 3144 | } |
| 3145 | |
| 3146 | free(source_order_bundles); |
| 3147 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 3148 | /* Report the very first tag executed */ |
| 3149 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3150 | |
| 3151 | /* Deal with off-by-one related to the fencepost problem */ |
| 3152 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3153 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 3154 | |
| 3155 | program->blend_patch_offset = ctx->blend_constant_offset; |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 3156 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3157 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3158 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !silent) |
Icecream95 | 968f36d | 2020-01-23 09:42:12 +1300 | [diff] [blame] | 3159 | disassemble_midgard(stdout, program->compiled.data, program->compiled.size, gpu_id, ctx->stage); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3160 | |
Icecream95 | 4a8ad1e | 2020-07-17 22:34:04 +1200 | [diff] [blame] | 3161 | if ((midgard_debug & MIDGARD_DBG_SHADERDB || shaderdb) && !silent) { |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3162 | unsigned nr_bundles = 0, nr_ins = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3163 | |
| 3164 | /* Count instructions and bundles */ |
| 3165 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3166 | mir_foreach_block(ctx, _block) { |
| 3167 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3168 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3169 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 3170 | |
Alyssa Rosenzweig | 67909c8 | 2019-08-30 13:08:16 -0700 | [diff] [blame] | 3171 | mir_foreach_bundle_in_block(block, bun) |
| 3172 | nr_ins += bun->instruction_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3173 | } |
| 3174 | |
| 3175 | /* Calculate thread count. There are certain cutoffs by |
| 3176 | * register count for thread count */ |
| 3177 | |
| 3178 | unsigned nr_registers = program->work_register_count; |
| 3179 | |
| 3180 | unsigned nr_threads = |
| 3181 | (nr_registers <= 4) ? 4 : |
| 3182 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3183 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3184 | |
| 3185 | /* Dump stats */ |
| 3186 | |
| 3187 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3188 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3189 | "%u registers, %u threads, %u loops, " |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 3190 | "%u:%u spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3191 | SHADER_DB_COUNT++, |
Alyssa Rosenzweig | 014d2e4 | 2020-05-25 13:19:43 -0400 | [diff] [blame] | 3192 | ctx->is_blend ? "PAN_SHADER_BLEND" : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3193 | gl_shader_stage_name(ctx->stage), |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3194 | nr_ins, nr_bundles, ctx->quadword_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3195 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3196 | ctx->loop_count, |
| 3197 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3198 | } |
| 3199 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 3200 | ralloc_free(ctx); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3201 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3202 | return 0; |
| 3203 | } |