Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 36 | #include "compiler/nir/nir_builder.h" |
| 37 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 38 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 39 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 40 | #include "util/u_dynarray.h" |
| 41 | #include "util/list.h" |
| 42 | #include "main/mtypes.h" |
| 43 | |
| 44 | #include "midgard.h" |
| 45 | #include "midgard_nir.h" |
| 46 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 47 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 48 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 49 | #include "compiler.h" |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 50 | #include "midgard_quirks.h" |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 51 | #include "panfrost-quirks.h" |
| 52 | #include "panfrost/util/pan_lower_framebuffer.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 53 | |
| 54 | #include "disassemble.h" |
| 55 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 56 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 57 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 58 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 59 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 60 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 61 | }; |
| 62 | |
| 63 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 64 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 65 | unsigned SHADER_DB_COUNT = 0; |
| 66 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 67 | int midgard_debug = 0; |
| 68 | |
| 69 | #define DBG(fmt, ...) \ |
| 70 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 71 | fprintf(stderr, "%s:%d: "fmt, \ |
| 72 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 73 | static midgard_block * |
| 74 | create_empty_block(compiler_context *ctx) |
| 75 | { |
| 76 | midgard_block *blk = rzalloc(ctx, midgard_block); |
| 77 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 78 | blk->base.predecessors = _mesa_set_create(blk, |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 79 | _mesa_hash_pointer, |
| 80 | _mesa_key_pointer_equal); |
| 81 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 82 | blk->base.name = ctx->block_source_count++; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 83 | |
| 84 | return blk; |
| 85 | } |
| 86 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 87 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 88 | schedule_barrier(compiler_context *ctx) |
| 89 | { |
| 90 | midgard_block *temp = ctx->after_block; |
| 91 | ctx->after_block = create_empty_block(ctx); |
| 92 | ctx->block_count++; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 93 | list_addtail(&ctx->after_block->base.link, &ctx->blocks); |
| 94 | list_inithead(&ctx->after_block->base.instructions); |
| 95 | pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 96 | ctx->current_block = ctx->after_block; |
| 97 | ctx->after_block = temp; |
| 98 | } |
| 99 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 100 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 101 | * driver seems to do it that way */ |
| 102 | |
| 103 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 104 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 105 | #define M_LOAD_STORE(name, store, T) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 106 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 107 | midgard_instruction i = { \ |
| 108 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 109 | .mask = 0xF, \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 110 | .dest = ~0, \ |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 111 | .src = { ~0, ~0, ~0, ~0 }, \ |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 112 | .swizzle = SWIZZLE_IDENTITY_4, \ |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 113 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 114 | .load_store = { \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 115 | .address = address \ |
| 116 | } \ |
| 117 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 118 | \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 119 | if (store) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 120 | i.src[0] = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 121 | i.src_types[0] = T; \ |
Alyssa Rosenzweig | 9915bb2 | 2020-05-07 10:12:38 -0400 | [diff] [blame] | 122 | i.dest_type = T; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 123 | } else { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 124 | i.dest = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 125 | i.dest_type = T; \ |
| 126 | } \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 127 | return i; \ |
| 128 | } |
| 129 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 130 | #define M_LOAD(name, T) M_LOAD_STORE(name, false, T) |
| 131 | #define M_STORE(name, T) M_LOAD_STORE(name, true, T) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 132 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 133 | M_LOAD(ld_attr_32, nir_type_uint32); |
| 134 | M_LOAD(ld_vary_32, nir_type_uint32); |
| 135 | M_LOAD(ld_ubo_int4, nir_type_uint32); |
| 136 | M_LOAD(ld_int4, nir_type_uint32); |
| 137 | M_STORE(st_int4, nir_type_uint32); |
| 138 | M_LOAD(ld_color_buffer_32u, nir_type_uint32); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 139 | M_LOAD(ld_color_buffer_as_fp16, nir_type_float16); |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 140 | M_LOAD(ld_color_buffer_as_fp32, nir_type_float32); |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 141 | M_STORE(st_vary_32, nir_type_uint32); |
| 142 | M_LOAD(ld_cubemap_coords, nir_type_uint32); |
| 143 | M_LOAD(ld_compute_id, nir_type_uint32); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 144 | |
| 145 | static midgard_instruction |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 146 | v_branch(bool conditional, bool invert) |
| 147 | { |
| 148 | midgard_instruction ins = { |
| 149 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 150 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 151 | .compact_branch = true, |
| 152 | .branch = { |
| 153 | .conditional = conditional, |
| 154 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 155 | }, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 156 | .dest = ~0, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 157 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 158 | }; |
| 159 | |
| 160 | return ins; |
| 161 | } |
| 162 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 163 | static void |
| 164 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 165 | { |
| 166 | ins->has_constants = true; |
| 167 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 171 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | { |
| 173 | return glsl_count_attribute_slots(type, false); |
| 174 | } |
| 175 | |
| 176 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 177 | static bool |
| 178 | midgard_nir_lower_fdot2_instr(nir_builder *b, nir_instr *instr, void *data) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 179 | { |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 180 | if (instr->type != nir_instr_type_alu) |
| 181 | return false; |
| 182 | |
| 183 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 184 | if (alu->op != nir_op_fdot2) |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 185 | return false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 186 | |
| 187 | b->cursor = nir_before_instr(&alu->instr); |
| 188 | |
| 189 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 190 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 191 | |
| 192 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 193 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 194 | nir_ssa_def *sum = nir_fadd(b, |
| 195 | nir_channel(b, product, 0), |
| 196 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 197 | |
| 198 | /* Replace the fdot2 with this sum */ |
| 199 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 200 | |
| 201 | return true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 202 | } |
| 203 | |
| 204 | static bool |
| 205 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 206 | { |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 207 | return nir_shader_instructions_pass(shader, |
| 208 | midgard_nir_lower_fdot2_instr, |
| 209 | nir_metadata_block_index | nir_metadata_dominance, |
| 210 | NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 211 | } |
| 212 | |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 213 | static const nir_variable * |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 214 | search_var(nir_shader *nir, nir_variable_mode mode, unsigned driver_loc) |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 215 | { |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 216 | nir_foreach_variable_with_modes(var, nir, mode) { |
Icecream95 | 7534a31 | 2020-06-06 15:39:22 +1200 | [diff] [blame] | 217 | if (var->data.driver_location == driver_loc) |
| 218 | return var; |
| 219 | } |
| 220 | |
| 221 | return NULL; |
| 222 | } |
| 223 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 224 | /* Midgard can write all of color, depth and stencil in a single writeout |
| 225 | * operation, so we merge depth/stencil stores with color stores. |
| 226 | * If there are no color stores, we add a write to the "depth RT". |
| 227 | */ |
| 228 | static bool |
| 229 | midgard_nir_lower_zs_store(nir_shader *nir) |
| 230 | { |
| 231 | if (nir->info.stage != MESA_SHADER_FRAGMENT) |
| 232 | return false; |
| 233 | |
| 234 | nir_variable *z_var = NULL, *s_var = NULL; |
| 235 | |
Jason Ekstrand | 2956d53 | 2020-07-18 18:24:25 -0500 | [diff] [blame] | 236 | nir_foreach_shader_out_variable(var, nir) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 237 | if (var->data.location == FRAG_RESULT_DEPTH) |
| 238 | z_var = var; |
| 239 | else if (var->data.location == FRAG_RESULT_STENCIL) |
| 240 | s_var = var; |
| 241 | } |
| 242 | |
| 243 | if (!z_var && !s_var) |
| 244 | return false; |
| 245 | |
| 246 | bool progress = false; |
| 247 | |
| 248 | nir_foreach_function(function, nir) { |
| 249 | if (!function->impl) continue; |
| 250 | |
| 251 | nir_intrinsic_instr *z_store = NULL, *s_store = NULL; |
| 252 | |
| 253 | nir_foreach_block(block, function->impl) { |
| 254 | nir_foreach_instr_safe(instr, block) { |
| 255 | if (instr->type != nir_instr_type_intrinsic) |
| 256 | continue; |
| 257 | |
| 258 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 259 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 260 | continue; |
| 261 | |
| 262 | if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) { |
| 263 | assert(!z_store); |
| 264 | z_store = intr; |
| 265 | } |
| 266 | |
| 267 | if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) { |
| 268 | assert(!s_store); |
| 269 | s_store = intr; |
| 270 | } |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | if (!z_store && !s_store) continue; |
| 275 | |
| 276 | bool replaced = false; |
| 277 | |
| 278 | nir_foreach_block(block, function->impl) { |
| 279 | nir_foreach_instr_safe(instr, block) { |
| 280 | if (instr->type != nir_instr_type_intrinsic) |
| 281 | continue; |
| 282 | |
| 283 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 284 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 285 | continue; |
| 286 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 287 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 288 | assert(var); |
| 289 | |
| 290 | if (var->data.location != FRAG_RESULT_COLOR && |
| 291 | var->data.location < FRAG_RESULT_DATA0) |
| 292 | continue; |
| 293 | |
Icecream95 | 334dab0 | 2020-07-10 23:28:21 +1200 | [diff] [blame] | 294 | if (var->data.index) |
| 295 | continue; |
| 296 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 297 | assert(nir_src_is_const(intr->src[1]) && "no indirect outputs"); |
| 298 | |
| 299 | nir_builder b; |
| 300 | nir_builder_init(&b, function->impl); |
| 301 | |
| 302 | assert(!z_store || z_store->instr.block == instr->block); |
| 303 | assert(!s_store || s_store->instr.block == instr->block); |
| 304 | b.cursor = nir_after_block_before_jump(instr->block); |
| 305 | |
| 306 | nir_intrinsic_instr *combined_store; |
| 307 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 308 | |
| 309 | combined_store->num_components = intr->src[0].ssa->num_components; |
| 310 | |
| 311 | nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr)); |
| 312 | |
| 313 | unsigned writeout = PAN_WRITEOUT_C; |
| 314 | if (z_store) |
| 315 | writeout |= PAN_WRITEOUT_Z; |
| 316 | if (s_store) |
| 317 | writeout |= PAN_WRITEOUT_S; |
| 318 | |
| 319 | nir_intrinsic_set_component(combined_store, writeout); |
| 320 | |
| 321 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 322 | |
| 323 | struct nir_ssa_def *src[4] = { |
| 324 | intr->src[0].ssa, |
| 325 | intr->src[1].ssa, |
| 326 | z_store ? z_store->src[0].ssa : zero, |
| 327 | s_store ? s_store->src[0].ssa : zero, |
| 328 | }; |
| 329 | |
| 330 | for (int i = 0; i < 4; ++i) |
| 331 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 332 | |
| 333 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 334 | |
| 335 | nir_instr_remove(instr); |
| 336 | |
| 337 | replaced = true; |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | /* Insert a store to the depth RT (0xff) if needed */ |
| 342 | if (!replaced) { |
| 343 | nir_builder b; |
| 344 | nir_builder_init(&b, function->impl); |
| 345 | |
| 346 | nir_block *block = NULL; |
| 347 | if (z_store && s_store) |
| 348 | assert(z_store->instr.block == s_store->instr.block); |
| 349 | |
| 350 | if (z_store) |
| 351 | block = z_store->instr.block; |
| 352 | else |
| 353 | block = s_store->instr.block; |
| 354 | |
| 355 | b.cursor = nir_after_block_before_jump(block); |
| 356 | |
| 357 | nir_intrinsic_instr *combined_store; |
| 358 | combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); |
| 359 | |
| 360 | combined_store->num_components = 4; |
| 361 | |
Icecream95 | 18059f4 | 2020-07-08 16:00:51 +1200 | [diff] [blame] | 362 | unsigned base; |
| 363 | if (z_store) |
| 364 | base = nir_intrinsic_base(z_store); |
| 365 | else |
| 366 | base = nir_intrinsic_base(s_store); |
| 367 | nir_intrinsic_set_base(combined_store, base); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 368 | |
| 369 | unsigned writeout = 0; |
| 370 | if (z_store) |
| 371 | writeout |= PAN_WRITEOUT_Z; |
| 372 | if (s_store) |
| 373 | writeout |= PAN_WRITEOUT_S; |
| 374 | |
| 375 | nir_intrinsic_set_component(combined_store, writeout); |
| 376 | |
| 377 | struct nir_ssa_def *zero = nir_imm_int(&b, 0); |
| 378 | |
| 379 | struct nir_ssa_def *src[4] = { |
| 380 | nir_imm_vec4(&b, 0, 0, 0, 0), |
| 381 | zero, |
| 382 | z_store ? z_store->src[0].ssa : zero, |
| 383 | s_store ? s_store->src[0].ssa : zero, |
| 384 | }; |
| 385 | |
| 386 | for (int i = 0; i < 4; ++i) |
| 387 | combined_store->src[i] = nir_src_for_ssa(src[i]); |
| 388 | |
| 389 | nir_builder_instr_insert(&b, &combined_store->instr); |
| 390 | } |
| 391 | |
| 392 | if (z_store) |
| 393 | nir_instr_remove(&z_store->instr); |
| 394 | |
| 395 | if (s_store) |
| 396 | nir_instr_remove(&s_store->instr); |
| 397 | |
| 398 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 399 | progress = true; |
| 400 | } |
| 401 | |
| 402 | return progress; |
| 403 | } |
| 404 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 405 | /* Real writeout stores, which break execution, need to be moved to after |
| 406 | * dual-source stores, which are just standard register writes. */ |
| 407 | static bool |
| 408 | midgard_nir_reorder_writeout(nir_shader *nir) |
| 409 | { |
| 410 | bool progress = false; |
| 411 | |
| 412 | nir_foreach_function(function, nir) { |
| 413 | if (!function->impl) continue; |
| 414 | |
| 415 | nir_foreach_block(block, function->impl) { |
| 416 | nir_instr *last_writeout = NULL; |
| 417 | |
| 418 | nir_foreach_instr_reverse_safe(instr, block) { |
| 419 | if (instr->type != nir_instr_type_intrinsic) |
| 420 | continue; |
| 421 | |
| 422 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 423 | if (intr->intrinsic != nir_intrinsic_store_output) |
| 424 | continue; |
| 425 | |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 426 | const nir_variable *var = search_var(nir, nir_var_shader_out, nir_intrinsic_base(intr)); |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 427 | |
| 428 | if (var->data.index) { |
| 429 | if (!last_writeout) |
| 430 | last_writeout = instr; |
| 431 | continue; |
| 432 | } |
| 433 | |
| 434 | if (!last_writeout) |
| 435 | continue; |
| 436 | |
| 437 | /* This is a real store, so move it to after dual-source stores */ |
| 438 | exec_node_remove(&instr->node); |
| 439 | exec_node_insert_after(&last_writeout->node, &instr->node); |
| 440 | |
| 441 | progress = true; |
| 442 | } |
| 443 | } |
| 444 | } |
| 445 | |
| 446 | return progress; |
| 447 | } |
| 448 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 449 | static bool |
| 450 | mdg_is_64(const nir_instr *instr, const void *_unused) |
| 451 | { |
| 452 | const nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 453 | |
| 454 | if (nir_dest_bit_size(alu->dest.dest) == 64) |
| 455 | return true; |
| 456 | |
| 457 | switch (alu->op) { |
| 458 | case nir_op_umul_high: |
| 459 | case nir_op_imul_high: |
| 460 | return true; |
| 461 | default: |
| 462 | return false; |
| 463 | } |
| 464 | } |
| 465 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 466 | /* Flushes undefined values to zero */ |
| 467 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 468 | static void |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 469 | optimise_nir(nir_shader *nir, unsigned quirks, bool is_blend) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 470 | { |
| 471 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 472 | unsigned lower_flrp = |
| 473 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 474 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 475 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 476 | |
| 477 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
Rhys Perry | 8b98d09 | 2019-02-05 15:56:24 +0000 | [diff] [blame] | 478 | NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 479 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 480 | nir_lower_tex_options lower_tex_options = { |
| 481 | .lower_txs_lod = true, |
Alyssa Rosenzweig | 4c43b35 | 2019-11-21 13:40:00 -0500 | [diff] [blame] | 482 | .lower_txp = ~0, |
| 483 | .lower_tex_without_implicit_lod = |
| 484 | (quirks & MIDGARD_EXPLICIT_LOD), |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 485 | .lower_tg4_broadcom_swizzle = true, |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 486 | |
| 487 | /* TODO: we have native gradient.. */ |
| 488 | .lower_txd = true, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 489 | }; |
| 490 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 491 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 492 | |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 493 | /* Must lower fdot2 after tex is lowered */ |
| 494 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
| 495 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 496 | /* T720 is broken. */ |
| 497 | |
| 498 | if (quirks & MIDGARD_BROKEN_LOD) |
| 499 | NIR_PASS_V(nir, midgard_nir_lod_errata); |
| 500 | |
Alyssa Rosenzweig | c495c6c | 2020-05-12 19:07:48 -0400 | [diff] [blame] | 501 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_early); |
| 502 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 503 | do { |
| 504 | progress = false; |
| 505 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 506 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 507 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 508 | |
| 509 | NIR_PASS(progress, nir, nir_copy_prop); |
Boris Brezillon | 440b0d6 | 2020-01-06 14:31:38 +0100 | [diff] [blame] | 510 | NIR_PASS(progress, nir, nir_opt_remove_phis); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 511 | NIR_PASS(progress, nir, nir_opt_dce); |
| 512 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 513 | NIR_PASS(progress, nir, nir_opt_cse); |
| 514 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 515 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 516 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 517 | |
| 518 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 519 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 520 | NIR_PASS(lower_flrp_progress, |
| 521 | nir, |
| 522 | nir_lower_flrp, |
| 523 | lower_flrp, |
Marek Olšák | ac55b1a | 2020-07-22 22:13:16 -0400 | [diff] [blame] | 524 | false /* always_precise */); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 525 | if (lower_flrp_progress) { |
| 526 | NIR_PASS(progress, nir, |
| 527 | nir_opt_constant_folding); |
| 528 | progress = true; |
| 529 | } |
| 530 | |
| 531 | /* Nothing should rematerialize any flrps, so we only |
| 532 | * need to do this lowering once. |
| 533 | */ |
| 534 | lower_flrp = 0; |
| 535 | } |
| 536 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 537 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 538 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 539 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 540 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 541 | nir_var_shader_in | |
| 542 | nir_var_shader_out | |
| 543 | nir_var_function_temp); |
| 544 | |
Eric Anholt | f25e169 | 2020-08-27 12:49:13 -0700 | [diff] [blame] | 545 | NIR_PASS(progress, nir, nir_opt_vectorize, NULL, NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 546 | } while (progress); |
| 547 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 548 | NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_is_64, NULL); |
| 549 | |
Alyssa Rosenzweig | d838cb9 | 2020-06-16 13:07:02 -0400 | [diff] [blame] | 550 | /* Run after opts so it can hit more */ |
| 551 | if (!is_blend) |
| 552 | NIR_PASS(progress, nir, nir_fuse_io_16); |
| 553 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 554 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 555 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 556 | |
| 557 | do { |
| 558 | progress = false; |
| 559 | |
| 560 | NIR_PASS(progress, nir, nir_opt_dce); |
| 561 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 562 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 563 | NIR_PASS(progress, nir, nir_copy_prop); |
| 564 | } while (progress); |
| 565 | |
| 566 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 211dee4 | 2020-04-29 20:27:16 -0400 | [diff] [blame] | 567 | NIR_PASS(progress, nir, nir_opt_algebraic_distribute_src_mods); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 568 | |
| 569 | /* We implement booleans as 32-bit 0/~0 */ |
| 570 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 571 | |
| 572 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 573 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 574 | NIR_PASS(progress, nir, midgard_nir_cancel_inot); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 575 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 576 | NIR_PASS(progress, nir, nir_copy_prop); |
| 577 | NIR_PASS(progress, nir, nir_opt_dce); |
| 578 | |
| 579 | /* Take us out of SSA */ |
| 580 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 581 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 582 | |
| 583 | /* We are a vector architecture; write combine where possible */ |
| 584 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 585 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 586 | |
| 587 | NIR_PASS(progress, nir, nir_opt_dce); |
| 588 | } |
| 589 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 590 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 591 | |
| 592 | static void |
| 593 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 594 | { |
| 595 | nir_ssa_def def = instr->def; |
| 596 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 597 | midgard_constants *consts = rzalloc(NULL, midgard_constants); |
| 598 | |
| 599 | assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8); |
| 600 | |
| 601 | #define RAW_CONST_COPY(bits) \ |
| 602 | nir_const_value_to_array(consts->u##bits, instr->value, \ |
| 603 | instr->def.num_components, u##bits) |
| 604 | |
| 605 | switch (instr->def.bit_size) { |
| 606 | case 64: |
| 607 | RAW_CONST_COPY(64); |
| 608 | break; |
| 609 | case 32: |
| 610 | RAW_CONST_COPY(32); |
| 611 | break; |
| 612 | case 16: |
| 613 | RAW_CONST_COPY(16); |
| 614 | break; |
| 615 | case 8: |
| 616 | RAW_CONST_COPY(8); |
| 617 | break; |
| 618 | default: |
| 619 | unreachable("Invalid bit_size for load_const instruction\n"); |
| 620 | } |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 621 | |
| 622 | /* Shifted for SSA, +1 for off-by-one */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 623 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 624 | } |
| 625 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 626 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 627 | * explicitly emit a move with the constant source */ |
| 628 | |
| 629 | static void |
| 630 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 631 | { |
| 632 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 633 | |
| 634 | if (constant_value) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 635 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), to); |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 636 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 637 | emit_mir_instruction(ctx, ins); |
| 638 | } |
| 639 | } |
| 640 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 641 | static bool |
| 642 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 643 | { |
| 644 | unsigned comp = src->swizzle[0]; |
| 645 | |
| 646 | for (unsigned c = 1; c < nr_components; ++c) { |
| 647 | if (src->swizzle[c] != comp) |
| 648 | return true; |
| 649 | } |
| 650 | |
| 651 | return false; |
| 652 | } |
| 653 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 654 | #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \ |
| 655 | case nir_intrinsic_##nir: \ |
| 656 | emit_atomic(ctx, instr, is_shared, midgard_op_##op); \ |
| 657 | break; |
| 658 | |
| 659 | #define ATOMIC_CASE(ctx, instr, nir, op) \ |
| 660 | ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \ |
| 661 | ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false); |
| 662 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 663 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 664 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 665 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 666 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 667 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 668 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 669 | #define ALU_CASE_RTZ(nir, _op) \ |
| 670 | case nir_op_##nir: \ |
| 671 | op = midgard_alu_op_##_op; \ |
| 672 | roundmode = MIDGARD_RTZ; \ |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 673 | break; |
| 674 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 675 | #define ALU_CHECK_CMP() \ |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 676 | assert(src_bitsize == 16 || src_bitsize == 32); \ |
| 677 | assert(dst_bitsize == 16 || dst_bitsize == 32); \ |
| 678 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 679 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 680 | case nir_op_##nir: \ |
| 681 | op = midgard_alu_op_##_op; \ |
| 682 | broadcast_swizzle = count; \ |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 683 | ALU_CHECK_CMP(); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 684 | break; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 685 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 686 | #define ALU_CASE_CMP(nir, _op) \ |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 687 | case nir_op_##nir: \ |
| 688 | op = midgard_alu_op_##_op; \ |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 689 | ALU_CHECK_CMP(); \ |
| 690 | break; |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 691 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 692 | /* Compare mir_lower_invert */ |
| 693 | static bool |
| 694 | nir_accepts_inot(nir_op op, unsigned src) |
| 695 | { |
| 696 | switch (op) { |
| 697 | case nir_op_ior: |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 698 | case nir_op_iand: /* TODO: b2f16 */ |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 699 | case nir_op_ixor: |
| 700 | return true; |
| 701 | case nir_op_b32csel: |
| 702 | /* Only the condition */ |
| 703 | return (src == 0); |
| 704 | default: |
| 705 | return false; |
| 706 | } |
| 707 | } |
| 708 | |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 709 | static bool |
| 710 | mir_accept_dest_mod(compiler_context *ctx, nir_dest **dest, nir_op op) |
| 711 | { |
| 712 | if (pan_has_dest_mod(dest, op)) { |
| 713 | assert((*dest)->is_ssa); |
| 714 | BITSET_SET(ctx->already_emitted, (*dest)->ssa.index); |
| 715 | return true; |
| 716 | } |
| 717 | |
| 718 | return false; |
| 719 | } |
| 720 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 721 | /* Look for floating point mods. We have the mods fsat, fsat_signed, |
| 722 | * and fpos. We also have the relations (note 3 * 2 = 6 cases): |
| 723 | * |
| 724 | * fsat_signed(fpos(x)) = fsat(x) |
| 725 | * fsat_signed(fsat(x)) = fsat(x) |
| 726 | * fpos(fsat_signed(x)) = fsat(x) |
| 727 | * fpos(fsat(x)) = fsat(x) |
| 728 | * fsat(fsat_signed(x)) = fsat(x) |
| 729 | * fsat(fpos(x)) = fsat(x) |
| 730 | * |
| 731 | * So by cases any composition of output modifiers is equivalent to |
| 732 | * fsat alone. |
| 733 | */ |
| 734 | static unsigned |
| 735 | mir_determine_float_outmod(compiler_context *ctx, nir_dest **dest, unsigned prior_outmod) |
| 736 | { |
| 737 | bool fpos = mir_accept_dest_mod(ctx, dest, nir_op_fclamp_pos); |
| 738 | bool fsat = mir_accept_dest_mod(ctx, dest, nir_op_fsat); |
| 739 | bool ssat = mir_accept_dest_mod(ctx, dest, nir_op_fsat_signed); |
| 740 | bool prior = (prior_outmod != midgard_outmod_none); |
| 741 | int count = (int) prior + (int) fpos + (int) ssat + (int) fsat; |
| 742 | |
| 743 | return ((count > 1) || fsat) ? midgard_outmod_sat : |
| 744 | fpos ? midgard_outmod_pos : |
| 745 | ssat ? midgard_outmod_sat_signed : |
| 746 | prior_outmod; |
| 747 | } |
| 748 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 749 | static void |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 750 | mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i, unsigned to, bool *abs, bool *neg, bool *not, enum midgard_roundmode *roundmode, bool is_int, unsigned bcast_count) |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 751 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 752 | nir_alu_src src = instr->src[i]; |
Alyssa Rosenzweig | b124f53 | 2020-04-29 18:10:43 -0400 | [diff] [blame] | 753 | |
| 754 | if (!is_int) { |
| 755 | if (pan_has_source_mod(&src, nir_op_fneg)) |
| 756 | *neg = !(*neg); |
| 757 | |
| 758 | if (pan_has_source_mod(&src, nir_op_fabs)) |
| 759 | *abs = true; |
| 760 | } |
| 761 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 762 | if (nir_accepts_inot(instr->op, i) && pan_has_source_mod(&src, nir_op_inot)) |
| 763 | *not = true; |
| 764 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 765 | if (roundmode) { |
| 766 | if (pan_has_source_mod(&src, nir_op_fround_even)) |
| 767 | *roundmode = MIDGARD_RTE; |
| 768 | |
| 769 | if (pan_has_source_mod(&src, nir_op_ftrunc)) |
| 770 | *roundmode = MIDGARD_RTZ; |
| 771 | |
| 772 | if (pan_has_source_mod(&src, nir_op_ffloor)) |
| 773 | *roundmode = MIDGARD_RTN; |
| 774 | |
| 775 | if (pan_has_source_mod(&src, nir_op_fceil)) |
| 776 | *roundmode = MIDGARD_RTP; |
| 777 | } |
| 778 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 779 | unsigned bits = nir_src_bit_size(src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 780 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 781 | ins->src[to] = nir_src_index(NULL, &src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 782 | ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 783 | |
| 784 | for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 785 | ins->swizzle[to][c] = src.swizzle[ |
| 786 | (!bcast_count || c < bcast_count) ? c : |
| 787 | (bcast_count - 1)]; |
| 788 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 789 | } |
| 790 | |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 791 | /* Midgard features both fcsel and icsel, depending on whether you want int or |
| 792 | * float modifiers. NIR's csel is typeless, so we want a heuristic to guess if |
| 793 | * we should emit an int or float csel depending on what modifiers could be |
| 794 | * placed. In the absense of modifiers, this is probably arbitrary. */ |
| 795 | |
| 796 | static bool |
| 797 | mir_is_bcsel_float(nir_alu_instr *instr) |
| 798 | { |
| 799 | nir_op intmods[] = { |
| 800 | nir_op_i2i8, nir_op_i2i16, |
| 801 | nir_op_i2i32, nir_op_i2i64 |
| 802 | }; |
| 803 | |
| 804 | nir_op floatmods[] = { |
| 805 | nir_op_fabs, nir_op_fneg, |
| 806 | nir_op_f2f16, nir_op_f2f32, |
| 807 | nir_op_f2f64 |
| 808 | }; |
| 809 | |
| 810 | nir_op floatdestmods[] = { |
| 811 | nir_op_fsat, nir_op_fsat_signed, nir_op_fclamp_pos, |
| 812 | nir_op_f2f16, nir_op_f2f32 |
| 813 | }; |
| 814 | |
| 815 | signed score = 0; |
| 816 | |
| 817 | for (unsigned i = 1; i < 3; ++i) { |
| 818 | nir_alu_src s = instr->src[i]; |
| 819 | for (unsigned q = 0; q < ARRAY_SIZE(intmods); ++q) { |
| 820 | if (pan_has_source_mod(&s, intmods[q])) |
| 821 | score--; |
| 822 | } |
| 823 | } |
| 824 | |
| 825 | for (unsigned i = 1; i < 3; ++i) { |
| 826 | nir_alu_src s = instr->src[i]; |
| 827 | for (unsigned q = 0; q < ARRAY_SIZE(floatmods); ++q) { |
| 828 | if (pan_has_source_mod(&s, floatmods[q])) |
| 829 | score++; |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | for (unsigned q = 0; q < ARRAY_SIZE(floatdestmods); ++q) { |
| 834 | nir_dest *dest = &instr->dest.dest; |
| 835 | if (pan_has_dest_mod(&dest, floatdestmods[q])) |
| 836 | score++; |
| 837 | } |
| 838 | |
| 839 | return (score > 0); |
| 840 | } |
| 841 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 842 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 843 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 844 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 845 | nir_dest *dest = &instr->dest.dest; |
| 846 | |
| 847 | if (dest->is_ssa && BITSET_TEST(ctx->already_emitted, dest->ssa.index)) |
| 848 | return; |
| 849 | |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 850 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 851 | * is handled elsewhere */ |
| 852 | |
| 853 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 854 | midgard_emit_derivatives(ctx, instr); |
| 855 | return; |
| 856 | } |
| 857 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 858 | bool is_ssa = dest->is_ssa; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 859 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 860 | unsigned nr_components = nir_dest_num_components(*dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 861 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | 04f76ad | 2020-04-27 18:58:21 -0400 | [diff] [blame] | 862 | unsigned op = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 863 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 864 | /* Number of components valid to check for the instruction (the rest |
| 865 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 866 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 867 | * in Midgard */ |
| 868 | |
| 869 | unsigned broadcast_swizzle = 0; |
| 870 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 871 | /* Should we swap arguments? */ |
| 872 | bool flip_src12 = false; |
| 873 | |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 874 | ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 875 | ASSERTED unsigned dst_bitsize = nir_dest_bit_size(*dest); |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 876 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 877 | enum midgard_roundmode roundmode = MIDGARD_RTE; |
| 878 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 879 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 880 | ALU_CASE(fadd, fadd); |
| 881 | ALU_CASE(fmul, fmul); |
| 882 | ALU_CASE(fmin, fmin); |
| 883 | ALU_CASE(fmax, fmax); |
| 884 | ALU_CASE(imin, imin); |
| 885 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 886 | ALU_CASE(umin, umin); |
| 887 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 888 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 889 | ALU_CASE(fround_even, froundeven); |
| 890 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 891 | ALU_CASE(fceil, fceil); |
| 892 | ALU_CASE(fdot3, fdot3); |
| 893 | ALU_CASE(fdot4, fdot4); |
| 894 | ALU_CASE(iadd, iadd); |
| 895 | ALU_CASE(isub, isub); |
| 896 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 897 | ALU_CASE(imul_high, imul); |
| 898 | ALU_CASE(umul_high, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 899 | |
| 900 | /* Zero shoved as second-arg */ |
| 901 | ALU_CASE(iabs, iabsdiff); |
| 902 | |
Italo Nicola | c9192d1 | 2020-09-19 10:36:08 +0000 | [diff] [blame] | 903 | ALU_CASE(uabs_isub, iabsdiff); |
| 904 | ALU_CASE(uabs_usub, uabsdiff); |
| 905 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 906 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 907 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 908 | ALU_CASE_CMP(feq32, feq); |
| 909 | ALU_CASE_CMP(fneu32, fne); |
| 910 | ALU_CASE_CMP(flt32, flt); |
| 911 | ALU_CASE_CMP(ieq32, ieq); |
| 912 | ALU_CASE_CMP(ine32, ine); |
| 913 | ALU_CASE_CMP(ilt32, ilt); |
| 914 | ALU_CASE_CMP(ult32, ult); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 915 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 916 | /* We don't have a native b2f32 instruction. Instead, like many |
| 917 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 918 | * correspondingly AND |
| 919 | * by 1.0 to do the type conversion. For the moment, prime us |
| 920 | * to emit: |
| 921 | * |
| 922 | * iand [whatever], #0 |
| 923 | * |
| 924 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 925 | */ |
| 926 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 927 | ALU_CASE_CMP(b2f32, iand); |
| 928 | ALU_CASE_CMP(b2f16, iand); |
| 929 | ALU_CASE_CMP(b2i32, iand); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 930 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 931 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 932 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 933 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 934 | ALU_CASE_CMP(f2b32, fne); |
| 935 | ALU_CASE_CMP(i2b32, ine); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 936 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 937 | ALU_CASE(frcp, frcp); |
| 938 | ALU_CASE(frsq, frsqrt); |
| 939 | ALU_CASE(fsqrt, fsqrt); |
| 940 | ALU_CASE(fexp2, fexp2); |
| 941 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 942 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 943 | ALU_CASE_RTZ(f2i64, f2i_rte); |
| 944 | ALU_CASE_RTZ(f2u64, f2u_rte); |
| 945 | ALU_CASE_RTZ(i2f64, i2f_rte); |
| 946 | ALU_CASE_RTZ(u2f64, u2f_rte); |
Boris Brezillon | fcceeaf | 2020-01-20 22:05:14 +0100 | [diff] [blame] | 947 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 948 | ALU_CASE_RTZ(f2i32, f2i_rte); |
| 949 | ALU_CASE_RTZ(f2u32, f2u_rte); |
| 950 | ALU_CASE_RTZ(i2f32, i2f_rte); |
| 951 | ALU_CASE_RTZ(u2f32, u2f_rte); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 952 | |
Alyssa Rosenzweig | 0ae0141 | 2020-05-25 14:46:40 -0400 | [diff] [blame] | 953 | ALU_CASE_RTZ(f2i8, f2i_rte); |
| 954 | ALU_CASE_RTZ(f2u8, f2u_rte); |
| 955 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 956 | ALU_CASE_RTZ(f2i16, f2i_rte); |
| 957 | ALU_CASE_RTZ(f2u16, f2u_rte); |
| 958 | ALU_CASE_RTZ(i2f16, i2f_rte); |
| 959 | ALU_CASE_RTZ(u2f16, u2f_rte); |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 960 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 961 | ALU_CASE(fsin, fsin); |
| 962 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 963 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 964 | /* We'll get 0 in the second arg, so: |
| 965 | * ~a = ~(a | 0) = nor(a, 0) */ |
| 966 | ALU_CASE(inot, inor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 967 | ALU_CASE(iand, iand); |
| 968 | ALU_CASE(ior, ior); |
| 969 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 970 | ALU_CASE(ishl, ishl); |
| 971 | ALU_CASE(ishr, iasr); |
| 972 | ALU_CASE(ushr, ilsr); |
| 973 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 974 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 975 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 976 | ALU_CASE_CMP(b32all_fequal4, fball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 977 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 978 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 979 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 980 | ALU_CASE_CMP(b32any_fnequal4, fbany_neq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 981 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 982 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 983 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 984 | ALU_CASE_CMP(b32all_iequal4, iball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 985 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 986 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 987 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 988 | ALU_CASE_CMP(b32any_inequal4, ibany_neq); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 989 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 990 | /* Source mods will be shoved in later */ |
| 991 | ALU_CASE(fabs, fmov); |
| 992 | ALU_CASE(fneg, fmov); |
| 993 | ALU_CASE(fsat, fmov); |
Alyssa Rosenzweig | 24e2e24 | 2020-05-04 16:12:41 -0400 | [diff] [blame] | 994 | ALU_CASE(fsat_signed, fmov); |
| 995 | ALU_CASE(fclamp_pos, fmov); |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 996 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 997 | /* For size conversion, we use a move. Ideally though we would squash |
| 998 | * these ops together; maybe that has to happen after in NIR as part of |
| 999 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1000 | * only / exactly one size. If stepping down, we use a dest override to |
| 1001 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 1002 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1003 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1004 | case nir_op_i2i8: |
| 1005 | case nir_op_i2i16: |
| 1006 | case nir_op_i2i32: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1007 | case nir_op_i2i64: |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1008 | case nir_op_u2u8: |
| 1009 | case nir_op_u2u16: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 1010 | case nir_op_u2u32: |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1011 | case nir_op_u2u64: |
| 1012 | case nir_op_f2f16: |
Boris Brezillon | e1f9e8d | 2020-01-20 16:05:31 +0100 | [diff] [blame] | 1013 | case nir_op_f2f32: |
| 1014 | case nir_op_f2f64: { |
| 1015 | if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 || |
| 1016 | instr->op == nir_op_f2f64) |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 1017 | op = midgard_alu_op_fmov; |
| 1018 | else |
| 1019 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1020 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1021 | break; |
| 1022 | } |
| 1023 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1024 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1025 | * arguments */ |
| 1026 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 1027 | case nir_op_fge: |
| 1028 | case nir_op_fge32: |
| 1029 | case nir_op_ige32: |
| 1030 | case nir_op_uge32: { |
| 1031 | op = |
| 1032 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 1033 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 1034 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 1035 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 1036 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1037 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1038 | flip_src12 = true; |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 1039 | ALU_CHECK_CMP(); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1040 | break; |
| 1041 | } |
| 1042 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 1043 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 1044 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 1045 | bool is_float = mir_is_bcsel_float(instr); |
| 1046 | op = is_float ? |
| 1047 | (mixed ? midgard_alu_op_fcsel_v : midgard_alu_op_fcsel) : |
| 1048 | (mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1049 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1050 | break; |
| 1051 | } |
| 1052 | |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1053 | case nir_op_unpack_32_2x16: |
| 1054 | case nir_op_unpack_32_4x8: |
| 1055 | case nir_op_pack_32_2x16: |
| 1056 | case nir_op_pack_32_4x8: { |
| 1057 | op = midgard_alu_op_imov; |
| 1058 | break; |
| 1059 | } |
| 1060 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1061 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1062 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1063 | assert(0); |
| 1064 | return; |
| 1065 | } |
| 1066 | |
Alyssa Rosenzweig | 72c1e3a | 2020-05-21 12:31:40 -0400 | [diff] [blame] | 1067 | /* Promote imov to fmov if it might help inline a constant */ |
| 1068 | if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src) |
| 1069 | && nir_src_bit_size(instr->src[0].src) == 32 |
| 1070 | && nir_is_same_comp_swizzle(instr->src[0].swizzle, |
| 1071 | nir_src_num_components(instr->src[0].src))) { |
| 1072 | op = midgard_alu_op_fmov; |
| 1073 | } |
| 1074 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 1075 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1076 | |
| 1077 | unsigned outmod = 0; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1078 | bool is_int = midgard_is_integer_op(op); |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 1079 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 1080 | if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) { |
| 1081 | outmod = midgard_outmod_int_high; |
| 1082 | } else if (midgard_is_integer_out_op(op)) { |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1083 | outmod = midgard_outmod_int_wrap; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1084 | } else if (instr->op == nir_op_fsat) { |
| 1085 | outmod = midgard_outmod_sat; |
| 1086 | } else if (instr->op == nir_op_fsat_signed) { |
| 1087 | outmod = midgard_outmod_sat_signed; |
| 1088 | } else if (instr->op == nir_op_fclamp_pos) { |
| 1089 | outmod = midgard_outmod_pos; |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 1090 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1091 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1092 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 1093 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1094 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1095 | |
Italo Nicola | 2096903 | 2020-07-13 16:19:08 +0000 | [diff] [blame] | 1096 | if (!midgard_is_integer_out_op(op)) { |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1097 | outmod = mir_determine_float_outmod(ctx, &dest, outmod); |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 1098 | } |
| 1099 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1100 | midgard_instruction ins = { |
| 1101 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1102 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ecf9466 | 2020-04-27 18:57:34 -0400 | [diff] [blame] | 1103 | .dest_type = nir_op_infos[instr->op].output_type |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1104 | | nir_dest_bit_size(*dest), |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 1105 | .roundmode = roundmode, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1106 | }; |
| 1107 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1108 | enum midgard_roundmode *roundptr = (opcode_props & MIDGARD_ROUNDS) ? |
| 1109 | &ins.roundmode : NULL; |
| 1110 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1111 | for (unsigned i = nr_inputs; i < ARRAY_SIZE(ins.src); ++i) |
| 1112 | ins.src[i] = ~0; |
| 1113 | |
| 1114 | if (quirk_flipped_r24) { |
| 1115 | ins.src[0] = ~0; |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1116 | mir_copy_src(&ins, instr, 0, 1, &ins.src_abs[1], &ins.src_neg[1], &ins.src_invert[1], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1117 | } else { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1118 | for (unsigned i = 0; i < nr_inputs; ++i) { |
| 1119 | unsigned to = i; |
| 1120 | |
| 1121 | if (instr->op == nir_op_b32csel) { |
| 1122 | /* The condition is the first argument; move |
| 1123 | * the other arguments up one to be a binary |
| 1124 | * instruction for Midgard with the condition |
| 1125 | * last */ |
| 1126 | |
| 1127 | if (i == 0) |
| 1128 | to = 2; |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1129 | else if (flip_src12) |
| 1130 | to = 2 - i; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1131 | else |
| 1132 | to = i - 1; |
| 1133 | } else if (flip_src12) { |
| 1134 | to = 1 - to; |
| 1135 | } |
| 1136 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 1137 | mir_copy_src(&ins, instr, i, to, &ins.src_abs[to], &ins.src_neg[to], &ins.src_invert[to], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 1138 | |
| 1139 | /* (!c) ? a : b = c ? b : a */ |
| 1140 | if (instr->op == nir_op_b32csel && ins.src_invert[2]) { |
| 1141 | ins.src_invert[2] = false; |
| 1142 | flip_src12 ^= true; |
| 1143 | } |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1144 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 1145 | } |
| 1146 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1147 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1148 | /* Lowered to move */ |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1149 | if (instr->op == nir_op_fneg) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1150 | ins.src_neg[1] ^= true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1151 | |
| 1152 | if (instr->op == nir_op_fabs) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 1153 | ins.src_abs[1] = true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1154 | } |
| 1155 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1156 | ins.mask = mask_of(nr_components); |
| 1157 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1158 | /* Apply writemask if non-SSA, keeping in mind that we can't write to |
| 1159 | * components that don't exist. Note modifier => SSA => !reg => no |
| 1160 | * writemask, so we don't have to worry about writemasks here.*/ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1161 | |
| 1162 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1163 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1164 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1165 | ins.op = op; |
Italo Nicola | 5011373 | 2020-07-15 18:43:18 +0000 | [diff] [blame] | 1166 | ins.outmod = outmod; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 1167 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1168 | /* Late fixup for emulated instructions */ |
| 1169 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1170 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1171 | /* Presently, our second argument is an inline #0 constant. |
| 1172 | * Switch over to an embedded 1.0 constant (that can't fit |
| 1173 | * inline, since we're 32-bit, not 16-bit like the inline |
| 1174 | * constants) */ |
| 1175 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1176 | ins.has_inline_constant = false; |
| 1177 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1178 | ins.src_types[1] = nir_type_float32; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1179 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1180 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1181 | if (instr->op == nir_op_b2f32) |
| 1182 | ins.constants.f32[0] = 1.0f; |
| 1183 | else |
| 1184 | ins.constants.i32[0] = 1; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1185 | |
| 1186 | for (unsigned c = 0; c < 16; ++c) |
| 1187 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 1188 | } else if (instr->op == nir_op_b2f16) { |
| 1189 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
| 1190 | ins.src_types[1] = nir_type_float16; |
| 1191 | ins.has_constants = true; |
| 1192 | ins.constants.i16[0] = _mesa_float_to_half(1.0); |
| 1193 | |
| 1194 | for (unsigned c = 0; c < 16; ++c) |
| 1195 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 1196 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 1197 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1198 | ins.has_inline_constant = false; |
| 1199 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Italo Nicola | b1b0ce0 | 2020-07-10 14:51:52 +0000 | [diff] [blame] | 1200 | ins.src_types[1] = ins.src_types[0]; |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1201 | ins.has_constants = true; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1202 | ins.constants.u32[0] = 0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1203 | |
| 1204 | for (unsigned c = 0; c < 16; ++c) |
| 1205 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1206 | } else if (instr->op == nir_op_pack_32_2x16) { |
| 1207 | ins.dest_type = nir_type_uint16; |
| 1208 | ins.mask = mask_of(nr_components * 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1209 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1210 | } else if (instr->op == nir_op_pack_32_4x8) { |
| 1211 | ins.dest_type = nir_type_uint8; |
| 1212 | ins.mask = mask_of(nr_components * 4); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1213 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1214 | } else if (instr->op == nir_op_unpack_32_2x16) { |
| 1215 | ins.dest_type = nir_type_uint32; |
| 1216 | ins.mask = mask_of(nr_components >> 1); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1217 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 1218 | } else if (instr->op == nir_op_unpack_32_4x8) { |
| 1219 | ins.dest_type = nir_type_uint32; |
| 1220 | ins.mask = mask_of(nr_components >> 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 1221 | ins.is_pack = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1222 | } |
| 1223 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1224 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 1225 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 1226 | * instructions can only operate as if they were scalars. Lower |
| 1227 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1228 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1229 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1230 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1231 | unsigned swizzle_back[MIR_VEC_COMPONENTS]; |
| 1232 | memcpy(&swizzle_back, ins.swizzle[0], sizeof(swizzle_back)); |
| 1233 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1234 | midgard_instruction ins_split[MIR_VEC_COMPONENTS]; |
| 1235 | unsigned ins_count = 0; |
| 1236 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1237 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1238 | /* Mask the associated component, dropping the |
| 1239 | * instruction if needed */ |
| 1240 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1241 | ins.mask = 1 << i; |
| 1242 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1243 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1244 | for (unsigned j = 0; j < ins_count; ++j) { |
| 1245 | if (swizzle_back[i] == ins_split[j].swizzle[0][0]) { |
| 1246 | ins_split[j].mask |= ins.mask; |
| 1247 | ins.mask = 0; |
| 1248 | break; |
| 1249 | } |
| 1250 | } |
| 1251 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1252 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1253 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1254 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1255 | for (unsigned j = 0; j < MIR_VEC_COMPONENTS; ++j) |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1256 | ins.swizzle[0][j] = swizzle_back[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1257 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1258 | ins_split[ins_count] = ins; |
| 1259 | |
| 1260 | ++ins_count; |
| 1261 | } |
| 1262 | |
| 1263 | for (unsigned i = 0; i < ins_count; ++i) { |
| 1264 | emit_mir_instruction(ctx, ins_split[i]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1265 | } |
| 1266 | } else { |
| 1267 | emit_mir_instruction(ctx, ins); |
| 1268 | } |
| 1269 | } |
| 1270 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1271 | #undef ALU_CASE |
| 1272 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1273 | static void |
| 1274 | mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read) |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1275 | { |
| 1276 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1277 | unsigned nir_mask = 0; |
| 1278 | unsigned dsize = 0; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1279 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1280 | if (is_read) { |
| 1281 | nir_mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1282 | dsize = nir_dest_bit_size(intr->dest); |
| 1283 | } else { |
| 1284 | nir_mask = nir_intrinsic_write_mask(intr); |
| 1285 | dsize = 32; |
| 1286 | } |
| 1287 | |
| 1288 | /* Once we have the NIR mask, we need to normalize to work in 32-bit space */ |
Alyssa Rosenzweig | 9b8cb9f | 2020-03-09 20:19:29 -0400 | [diff] [blame] | 1289 | unsigned bytemask = pan_to_bytemask(dsize, nir_mask); |
Alyssa Rosenzweig | b91d715 | 2020-05-11 15:06:53 -0400 | [diff] [blame] | 1290 | ins->dest_type = nir_type_uint | dsize; |
Italo Nicola | 1101261 | 2020-08-26 14:56:13 +0000 | [diff] [blame] | 1291 | mir_set_bytemask(ins, bytemask); |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1292 | } |
| 1293 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1294 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1295 | * optimized) versions of UBO #0 */ |
| 1296 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1297 | static midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1298 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1299 | compiler_context *ctx, |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1300 | nir_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1301 | unsigned dest, |
| 1302 | unsigned offset, |
| 1303 | nir_src *indirect_offset, |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1304 | unsigned indirect_shift, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1305 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1306 | { |
| 1307 | /* TODO: half-floats */ |
| 1308 | |
Alyssa Rosenzweig | bc9a7d0 | 2019-11-15 14:19:34 -0500 | [diff] [blame] | 1309 | midgard_instruction ins = m_ld_ubo_int4(dest, 0); |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1310 | ins.constants.u32[0] = offset; |
Alyssa Rosenzweig | da73651 | 2019-12-19 11:12:25 -0500 | [diff] [blame] | 1311 | |
| 1312 | if (instr->type == nir_instr_type_intrinsic) |
| 1313 | mir_set_intr_mask(instr, &ins, true); |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1314 | |
| 1315 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1316 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1317 | ins.src_types[2] = nir_type_uint32; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1318 | ins.load_store.arg_2 = (indirect_shift << 5); |
Alyssa Rosenzweig | 797fa87 | 2020-07-06 10:57:04 -0400 | [diff] [blame] | 1319 | |
| 1320 | /* X component for the whole swizzle to prevent register |
| 1321 | * pressure from ballooning from the extra components */ |
| 1322 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[2]); ++i) |
| 1323 | ins.swizzle[2][i] = 0; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1324 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1325 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1326 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1327 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1328 | ins.load_store.arg_1 = index; |
| 1329 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1330 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1331 | } |
| 1332 | |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1333 | /* Globals are like UBOs if you squint. And shared memory is like globals if |
| 1334 | * you squint even harder */ |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1335 | |
| 1336 | static void |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1337 | emit_global( |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1338 | compiler_context *ctx, |
| 1339 | nir_instr *instr, |
| 1340 | bool is_read, |
| 1341 | unsigned srcdest, |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1342 | nir_src *offset, |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1343 | bool is_shared) |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1344 | { |
| 1345 | /* TODO: types */ |
| 1346 | |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 1347 | midgard_instruction ins; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1348 | |
| 1349 | if (is_read) |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1350 | ins = m_ld_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1351 | else |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1352 | ins = m_st_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1353 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1354 | mir_set_offset(ctx, &ins, offset, is_shared); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1355 | mir_set_intr_mask(instr, &ins, is_read); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1356 | |
Alyssa Rosenzweig | 41184f8 | 2020-08-27 15:13:19 -0400 | [diff] [blame] | 1357 | /* Set a valid swizzle for masked out components */ |
| 1358 | assert(ins.mask); |
| 1359 | unsigned first_component = __builtin_ffs(ins.mask) - 1; |
| 1360 | |
| 1361 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) { |
| 1362 | if (!(ins.mask & (1 << i))) |
| 1363 | ins.swizzle[0][i] = first_component; |
| 1364 | } |
| 1365 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1366 | emit_mir_instruction(ctx, ins); |
| 1367 | } |
| 1368 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 1369 | /* If is_shared is off, the only other possible value are globals, since |
| 1370 | * SSBO's are being lowered to globals through a NIR pass. */ |
| 1371 | static void |
| 1372 | emit_atomic( |
| 1373 | compiler_context *ctx, |
| 1374 | nir_intrinsic_instr *instr, |
| 1375 | bool is_shared, |
| 1376 | midgard_load_store_op op) |
| 1377 | { |
| 1378 | unsigned bitsize = nir_src_bit_size(instr->src[1]); |
| 1379 | nir_alu_type type = |
| 1380 | (op == midgard_op_atomic_imin || op == midgard_op_atomic_imax) ? |
| 1381 | nir_type_int : nir_type_uint; |
| 1382 | |
| 1383 | unsigned dest = nir_dest_index(&instr->dest); |
| 1384 | unsigned val = nir_src_index(ctx, &instr->src[1]); |
| 1385 | emit_explicit_constant(ctx, val, val); |
| 1386 | |
| 1387 | midgard_instruction ins = { |
| 1388 | .type = TAG_LOAD_STORE_4, |
| 1389 | .mask = 0xF, |
| 1390 | .dest = dest, |
| 1391 | .src = { ~0, ~0, ~0, val }, |
| 1392 | .src_types = { 0, 0, 0, type | bitsize }, |
| 1393 | .op = op |
| 1394 | }; |
| 1395 | |
| 1396 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1397 | |
| 1398 | /* cmpxchg takes an extra value in arg_2, so we don't use it for the offset */ |
| 1399 | if (op == midgard_op_atomic_cmpxchg) { |
| 1400 | unsigned addr = nir_src_index(ctx, src_offset); |
| 1401 | |
| 1402 | ins.src[1] = addr; |
| 1403 | ins.src_types[1] = nir_type_uint | nir_src_bit_size(*src_offset); |
| 1404 | |
| 1405 | unsigned xchg_val = nir_src_index(ctx, &instr->src[2]); |
| 1406 | emit_explicit_constant(ctx, xchg_val, xchg_val); |
| 1407 | |
| 1408 | ins.src[2] = val; |
| 1409 | ins.src_types[2] = type | bitsize; |
| 1410 | ins.src[3] = xchg_val; |
| 1411 | |
| 1412 | if (is_shared) |
| 1413 | ins.load_store.arg_1 |= 0x6E; |
| 1414 | } else { |
| 1415 | mir_set_offset(ctx, &ins, src_offset, is_shared); |
| 1416 | } |
| 1417 | |
| 1418 | mir_set_intr_mask(&instr->instr, &ins, true); |
| 1419 | |
| 1420 | emit_mir_instruction(ctx, ins); |
| 1421 | } |
| 1422 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1423 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1424 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1425 | compiler_context *ctx, |
| 1426 | unsigned dest, unsigned offset, |
| 1427 | unsigned nr_comp, unsigned component, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1428 | nir_src *indirect_offset, nir_alu_type type, bool flat) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1429 | { |
| 1430 | /* XXX: Half-floats? */ |
| 1431 | /* TODO: swizzle, mask */ |
| 1432 | |
| 1433 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1434 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 2d16883 | 2020-06-04 11:32:59 -0400 | [diff] [blame] | 1435 | ins.dest_type = type; |
| 1436 | |
| 1437 | if (type == nir_type_float16) { |
| 1438 | /* Ensure we are aligned so we can pack it later */ |
| 1439 | ins.mask = mask_of(ALIGN_POT(nr_comp, 2)); |
| 1440 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1441 | |
| 1442 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) |
| 1443 | ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1444 | |
| 1445 | midgard_varying_parameter p = { |
| 1446 | .is_varying = 1, |
| 1447 | .interpolation = midgard_interp_default, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1448 | .flat = flat, |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1449 | }; |
| 1450 | |
| 1451 | unsigned u; |
| 1452 | memcpy(&u, &p, sizeof(p)); |
| 1453 | ins.load_store.varying_parameters = u; |
| 1454 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1455 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1456 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1457 | ins.src_types[2] = nir_type_uint32; |
| 1458 | } else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1459 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1460 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1461 | ins.load_store.arg_1 = 0x9E; |
| 1462 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1463 | /* Use the type appropriate load */ |
| 1464 | switch (type) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1465 | case nir_type_uint32: |
| 1466 | case nir_type_bool32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1467 | ins.op = midgard_op_ld_vary_32u; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1468 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1469 | case nir_type_int32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1470 | ins.op = midgard_op_ld_vary_32i; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1471 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1472 | case nir_type_float32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1473 | ins.op = midgard_op_ld_vary_32; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1474 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1475 | case nir_type_float16: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1476 | ins.op = midgard_op_ld_vary_16; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1477 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1478 | default: |
| 1479 | unreachable("Attempted to load unknown type"); |
| 1480 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1481 | } |
| 1482 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1483 | emit_mir_instruction(ctx, ins); |
| 1484 | } |
| 1485 | |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1486 | static void |
| 1487 | emit_attr_read( |
| 1488 | compiler_context *ctx, |
| 1489 | unsigned dest, unsigned offset, |
| 1490 | unsigned nr_comp, nir_alu_type t) |
| 1491 | { |
| 1492 | midgard_instruction ins = m_ld_attr_32(dest, offset); |
| 1493 | ins.load_store.arg_1 = 0x1E; |
| 1494 | ins.load_store.arg_2 = 0x1E; |
| 1495 | ins.mask = mask_of(nr_comp); |
| 1496 | |
| 1497 | /* Use the type appropriate load */ |
| 1498 | switch (t) { |
| 1499 | case nir_type_uint: |
| 1500 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1501 | ins.op = midgard_op_ld_attr_32u; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1502 | break; |
| 1503 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1504 | ins.op = midgard_op_ld_attr_32i; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1505 | break; |
| 1506 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1507 | ins.op = midgard_op_ld_attr_32; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1508 | break; |
| 1509 | default: |
| 1510 | unreachable("Attempted to load unknown type"); |
| 1511 | break; |
| 1512 | } |
| 1513 | |
| 1514 | emit_mir_instruction(ctx, ins); |
| 1515 | } |
| 1516 | |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1517 | static void |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1518 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1519 | unsigned nr_components, unsigned offset) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1520 | { |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1521 | nir_dest nir_dest; |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1522 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1523 | /* Figure out which uniform this is */ |
Alyssa Rosenzweig | e610267 | 2020-03-10 16:06:30 -0400 | [diff] [blame] | 1524 | int sysval = panfrost_sysval_for_instr(instr, &nir_dest); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1525 | void *val = _mesa_hash_table_u64_search(ctx->sysvals.sysval_to_id, sysval); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1526 | |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1527 | unsigned dest = nir_dest_index(&nir_dest); |
| 1528 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1529 | /* Sysvals are prefix uniforms */ |
| 1530 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1531 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1532 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1533 | midgard_instruction *ins = |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1534 | emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0, 0); |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1535 | |
| 1536 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1537 | } |
| 1538 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1539 | static unsigned |
| 1540 | compute_builtin_arg(nir_op op) |
| 1541 | { |
| 1542 | switch (op) { |
| 1543 | case nir_intrinsic_load_work_group_id: |
| 1544 | return 0x14; |
| 1545 | case nir_intrinsic_load_local_invocation_id: |
| 1546 | return 0x10; |
| 1547 | default: |
| 1548 | unreachable("Invalid compute paramater loaded"); |
| 1549 | } |
| 1550 | } |
| 1551 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1552 | static void |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1553 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z, unsigned src_s, enum midgard_rt_id rt) |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1554 | { |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1555 | assert(rt < ARRAY_SIZE(ctx->writeout_branch)); |
| 1556 | |
| 1557 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 1558 | |
| 1559 | assert(!br); |
| 1560 | |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 1561 | emit_explicit_constant(ctx, src, src); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1562 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1563 | struct midgard_instruction ins = |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1564 | v_branch(false, false); |
| 1565 | |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1566 | bool depth_only = (rt == MIDGARD_ZS_RT); |
| 1567 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1568 | ins.writeout = depth_only ? 0 : PAN_WRITEOUT_C; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1569 | |
| 1570 | /* Add dependencies */ |
Alyssa Rosenzweig | 7652983 | 2019-08-30 11:01:15 -0700 | [diff] [blame] | 1571 | ins.src[0] = src; |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1572 | ins.src_types[0] = nir_type_uint32; |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1573 | ins.constants.u32[0] = depth_only ? 0xFF : (rt - MIDGARD_COLOR_RT0) * 0x100; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 1574 | for (int i = 0; i < 4; ++i) |
| 1575 | ins.swizzle[0][i] = i; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1576 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1577 | if (~src_z) { |
| 1578 | emit_explicit_constant(ctx, src_z, src_z); |
| 1579 | ins.src[2] = src_z; |
| 1580 | ins.src_types[2] = nir_type_uint32; |
| 1581 | ins.writeout |= PAN_WRITEOUT_Z; |
| 1582 | } |
| 1583 | if (~src_s) { |
| 1584 | emit_explicit_constant(ctx, src_s, src_s); |
| 1585 | ins.src[3] = src_s; |
| 1586 | ins.src_types[3] = nir_type_uint32; |
| 1587 | ins.writeout |= PAN_WRITEOUT_S; |
| 1588 | } |
| 1589 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1590 | /* Emit the branch */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1591 | br = emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 281cc6f | 2019-11-23 12:43:55 -0500 | [diff] [blame] | 1592 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 1593 | ctx->writeout_branch[rt] = br; |
| 1594 | |
| 1595 | /* Push our current location = current block count - 1 = where we'll |
| 1596 | * jump to. Maybe a bit too clever for my own good */ |
| 1597 | |
| 1598 | br->branch.target_block = ctx->block_count - 1; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1599 | } |
| 1600 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1601 | static void |
| 1602 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1603 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1604 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1605 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1606 | ins.mask = mask_of(3); |
Alyssa Rosenzweig | d3747fb | 2020-02-12 08:39:29 -0500 | [diff] [blame] | 1607 | ins.swizzle[0][3] = COMPONENT_X; /* xyzx */ |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1608 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1609 | emit_mir_instruction(ctx, ins); |
| 1610 | } |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1611 | |
| 1612 | static unsigned |
| 1613 | vertex_builtin_arg(nir_op op) |
| 1614 | { |
| 1615 | switch (op) { |
| 1616 | case nir_intrinsic_load_vertex_id: |
| 1617 | return PAN_VERTEX_ID; |
| 1618 | case nir_intrinsic_load_instance_id: |
| 1619 | return PAN_INSTANCE_ID; |
| 1620 | default: |
| 1621 | unreachable("Invalid vertex builtin"); |
| 1622 | } |
| 1623 | } |
| 1624 | |
| 1625 | static void |
| 1626 | emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1627 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1628 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1629 | emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int); |
| 1630 | } |
| 1631 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1632 | static void |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1633 | emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx) |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1634 | { |
| 1635 | unsigned reg = nir_dest_index(&instr->dest); |
| 1636 | |
| 1637 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1638 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1639 | ld.load_store.address = idx; |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1640 | ld.load_store.arg_2 = 0x1E; |
| 1641 | |
| 1642 | for (int i = 0; i < 4; ++i) |
| 1643 | ld.swizzle[0][i] = COMPONENT_X; |
| 1644 | |
| 1645 | emit_mir_instruction(ctx, ld); |
| 1646 | } |
| 1647 | |
| 1648 | static void |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1649 | emit_control_barrier(compiler_context *ctx) |
| 1650 | { |
| 1651 | midgard_instruction ins = { |
| 1652 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | fde1f2b | 2020-05-13 11:05:34 -0400 | [diff] [blame] | 1653 | .dest = ~0, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1654 | .src = { ~0, ~0, ~0, ~0 }, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 1655 | .op = TEXTURE_OP_BARRIER, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1656 | }; |
| 1657 | |
| 1658 | emit_mir_instruction(ctx, ins); |
| 1659 | } |
| 1660 | |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1661 | static unsigned |
| 1662 | mir_get_branch_cond(nir_src *src, bool *invert) |
| 1663 | { |
| 1664 | /* Wrap it. No swizzle since it's a scalar */ |
| 1665 | |
| 1666 | nir_alu_src alu = { |
| 1667 | .src = *src |
| 1668 | }; |
| 1669 | |
| 1670 | *invert = pan_has_source_mod(&alu, nir_op_inot); |
| 1671 | return nir_src_index(NULL, &alu.src); |
| 1672 | } |
| 1673 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1674 | static uint8_t |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1675 | output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr) |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1676 | { |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1677 | if (ctx->is_blend) |
| 1678 | return ctx->blend_rt; |
| 1679 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1680 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1681 | var = search_var(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr)); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1682 | assert(var); |
| 1683 | |
| 1684 | unsigned loc = var->data.location; |
| 1685 | |
| 1686 | if (loc == FRAG_RESULT_COLOR) |
| 1687 | loc = FRAG_RESULT_DATA0; |
| 1688 | |
| 1689 | if (loc >= FRAG_RESULT_DATA0) |
| 1690 | return loc - FRAG_RESULT_DATA0; |
| 1691 | |
| 1692 | if (loc == FRAG_RESULT_DEPTH) |
| 1693 | return 0x1F; |
| 1694 | if (loc == FRAG_RESULT_STENCIL) |
| 1695 | return 0x1E; |
| 1696 | |
Icecream95 | 6493d29 | 2020-07-14 15:06:09 +1200 | [diff] [blame] | 1697 | unreachable("Invalid RT to load from"); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1698 | } |
| 1699 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1700 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1701 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1702 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1703 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1704 | |
| 1705 | switch (instr->intrinsic) { |
| 1706 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1707 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1708 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1709 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1710 | discard.branch.target_type = TARGET_DISCARD; |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1711 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1712 | if (conditional) { |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1713 | discard.src[0] = mir_get_branch_cond(&instr->src[0], |
| 1714 | &discard.branch.invert_conditional); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1715 | discard.src_types[0] = nir_type_uint32; |
| 1716 | } |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1717 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1718 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1719 | schedule_barrier(ctx); |
| 1720 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1721 | break; |
| 1722 | } |
| 1723 | |
| 1724 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1725 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1726 | case nir_intrinsic_load_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1727 | case nir_intrinsic_load_shared: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1728 | case nir_intrinsic_load_input: |
| 1729 | case nir_intrinsic_load_interpolated_input: { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1730 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1731 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1732 | bool is_global = instr->intrinsic == nir_intrinsic_load_global; |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1733 | bool is_shared = instr->intrinsic == nir_intrinsic_load_shared; |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1734 | bool is_flat = instr->intrinsic == nir_intrinsic_load_input; |
| 1735 | bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1736 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1737 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1738 | /* TODO: Infer type? Does it matter? */ |
| 1739 | nir_alu_type t = |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1740 | (is_ubo || is_global || is_shared) ? nir_type_uint : |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1741 | (is_interp) ? nir_type_float : |
Jason Ekstrand | 0aa08ae | 2020-09-30 21:20:53 -0500 | [diff] [blame] | 1742 | nir_intrinsic_dest_type(instr); |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1743 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1744 | t = nir_alu_type_get_base_type(t); |
| 1745 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1746 | if (!(is_ubo || is_global)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1747 | offset = nir_intrinsic_base(instr); |
| 1748 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1749 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1750 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1751 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1752 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1753 | |
| 1754 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1755 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1756 | |
| 1757 | if (direct) |
| 1758 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1759 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1760 | /* We may need to apply a fractional offset */ |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1761 | int component = (is_flat || is_interp) ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1762 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1763 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1764 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1765 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1766 | emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysvals.sysval_count + offset) * 16, indirect_offset, 4, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1767 | } else if (is_ubo) { |
| 1768 | nir_src index = instr->src[0]; |
| 1769 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1770 | /* TODO: Is indirect block number possible? */ |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1771 | assert(nir_src_is_const(index)); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1772 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1773 | uint32_t uindex = nir_src_as_uint(index) + 1; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1774 | emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1775 | } else if (is_global || is_shared) { |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1776 | emit_global(ctx, &instr->instr, true, reg, src_offset, is_shared); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1777 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1778 | emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1779 | } else if (ctx->is_blend) { |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1780 | /* ctx->blend_input will be precoloured to r0/r2, where |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1781 | * the input is preloaded */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1782 | |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1783 | unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input; |
| 1784 | |
| 1785 | if (*input == ~0) |
| 1786 | *input = reg; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1787 | else |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1788 | emit_mir_instruction(ctx, v_mov(*input, reg)); |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1789 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
| 1790 | emit_attr_read(ctx, reg, offset, nr_comp, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1791 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1792 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1793 | assert(0); |
| 1794 | } |
| 1795 | |
| 1796 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1797 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1798 | |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1799 | /* Artefact of load_interpolated_input. TODO: other barycentric modes */ |
| 1800 | case nir_intrinsic_load_barycentric_pixel: |
Tomeu Vizoso | 2504206 | 2020-01-03 09:42:11 +0100 | [diff] [blame] | 1801 | case nir_intrinsic_load_barycentric_centroid: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1802 | break; |
| 1803 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1804 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1805 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1806 | case nir_intrinsic_load_raw_output_pan: { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1807 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1808 | |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1809 | /* T720 and below use different blend opcodes with slightly |
| 1810 | * different semantics than T760 and up */ |
| 1811 | |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 1812 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1813 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1814 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1815 | |
Icecream95 | c20d166 | 2020-07-16 14:16:11 +1200 | [diff] [blame] | 1816 | if (nir_src_is_const(instr->src[0])) { |
| 1817 | ld.load_store.arg_1 = nir_src_as_uint(instr->src[0]); |
| 1818 | } else { |
| 1819 | ld.load_store.varying_parameters = 2; |
| 1820 | ld.src[1] = nir_src_index(ctx, &instr->src[0]); |
| 1821 | ld.src_types[1] = nir_type_int32; |
| 1822 | } |
| 1823 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1824 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1825 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 5a175e4 | 2020-05-29 21:11:11 -0400 | [diff] [blame] | 1826 | ld.load_store.address = 16; |
| 1827 | ld.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1828 | } |
| 1829 | |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1830 | emit_mir_instruction(ctx, ld); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1831 | break; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1832 | } |
| 1833 | |
| 1834 | case nir_intrinsic_load_output: { |
| 1835 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1836 | |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1837 | unsigned bits = nir_dest_bit_size(instr->dest); |
| 1838 | |
| 1839 | midgard_instruction ld; |
| 1840 | if (bits == 16) |
| 1841 | ld = m_ld_color_buffer_as_fp16(reg, 0); |
| 1842 | else |
| 1843 | ld = m_ld_color_buffer_as_fp32(reg, 0); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1844 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1845 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1846 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1847 | for (unsigned c = 4; c < 16; ++c) |
| 1848 | ld.swizzle[0][c] = 0; |
| 1849 | |
| 1850 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1851 | if (bits == 16) |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1852 | ld.op = midgard_op_ld_color_buffer_as_fp16_old; |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1853 | else |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1854 | ld.op = midgard_op_ld_color_buffer_as_fp32_old; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1855 | ld.load_store.address = 1; |
| 1856 | ld.load_store.arg_2 = 0x1E; |
| 1857 | } |
| 1858 | |
| 1859 | emit_mir_instruction(ctx, ld); |
| 1860 | break; |
| 1861 | } |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1862 | |
| 1863 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1864 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1865 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1866 | |
| 1867 | /* Blend constants are embedded directly in the shader and |
| 1868 | * patched in, so we use some magic routing */ |
| 1869 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1870 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1871 | ins.has_constants = true; |
| 1872 | ins.has_blend_constant = true; |
| 1873 | emit_mir_instruction(ctx, ins); |
| 1874 | break; |
| 1875 | } |
| 1876 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1877 | case nir_intrinsic_store_output: |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1878 | case nir_intrinsic_store_combined_output_pan: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1879 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1880 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1881 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1882 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1883 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1884 | |
| 1885 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1886 | bool combined = instr->intrinsic == |
| 1887 | nir_intrinsic_store_combined_output_pan; |
| 1888 | |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1889 | const nir_variable *var; |
Jason Ekstrand | 94f0bae | 2020-07-20 16:07:11 -0500 | [diff] [blame] | 1890 | var = search_var(ctx->nir, nir_var_shader_out, |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1891 | nir_intrinsic_base(instr)); |
| 1892 | assert(var); |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1893 | |
| 1894 | /* Dual-source blend writeout is done by leaving the |
| 1895 | * value in r2 for the blend shader to use. */ |
| 1896 | if (var->data.index) { |
| 1897 | if (instr->src[0].is_ssa) { |
| 1898 | emit_explicit_constant(ctx, reg, reg); |
| 1899 | |
| 1900 | unsigned out = make_compiler_temp(ctx); |
| 1901 | |
| 1902 | midgard_instruction ins = v_mov(reg, out); |
| 1903 | emit_mir_instruction(ctx, ins); |
| 1904 | |
| 1905 | ctx->blend_src1 = out; |
| 1906 | } else { |
| 1907 | ctx->blend_src1 = reg; |
| 1908 | } |
| 1909 | |
| 1910 | break; |
| 1911 | } |
| 1912 | |
| 1913 | enum midgard_rt_id rt; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1914 | if (var->data.location == FRAG_RESULT_COLOR) |
| 1915 | rt = MIDGARD_COLOR_RT0; |
| 1916 | else if (var->data.location >= FRAG_RESULT_DATA0) |
| 1917 | rt = MIDGARD_COLOR_RT0 + var->data.location - |
| 1918 | FRAG_RESULT_DATA0; |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1919 | else if (combined) |
| 1920 | rt = MIDGARD_ZS_RT; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1921 | else |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 1922 | unreachable("bad rt"); |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1923 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1924 | unsigned reg_z = ~0, reg_s = ~0; |
| 1925 | if (combined) { |
| 1926 | unsigned writeout = nir_intrinsic_component(instr); |
| 1927 | if (writeout & PAN_WRITEOUT_Z) |
| 1928 | reg_z = nir_src_index(ctx, &instr->src[2]); |
| 1929 | if (writeout & PAN_WRITEOUT_S) |
| 1930 | reg_s = nir_src_index(ctx, &instr->src[3]); |
| 1931 | } |
| 1932 | |
| 1933 | emit_fragment_store(ctx, reg, reg_z, reg_s, rt); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1934 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1935 | assert(instr->intrinsic == nir_intrinsic_store_output); |
| 1936 | |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1937 | /* We should have been vectorized, though we don't |
| 1938 | * currently check that st_vary is emitted only once |
| 1939 | * per slot (this is relevant, since there's not a mask |
| 1940 | * parameter available on the store [set to 0 by the |
| 1941 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1942 | * swizzle. If this is a constant source, we'll need to |
| 1943 | * emit that explicitly. */ |
| 1944 | |
| 1945 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1946 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1947 | unsigned dst_component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | 2788721 | 2019-08-15 16:53:03 -0700 | [diff] [blame] | 1948 | unsigned nr_comp = nir_src_num_components(instr->src[0]); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1949 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1950 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1951 | st.load_store.arg_1 = 0x9E; |
| 1952 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1953 | |
Jason Ekstrand | 0aa08ae | 2020-09-30 21:20:53 -0500 | [diff] [blame] | 1954 | switch (nir_alu_type_get_base_type(nir_intrinsic_src_type(instr))) { |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1955 | case nir_type_uint: |
| 1956 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1957 | st.op = midgard_op_st_vary_32u; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1958 | break; |
| 1959 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1960 | st.op = midgard_op_st_vary_32i; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1961 | break; |
| 1962 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1963 | st.op = midgard_op_st_vary_32; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1964 | break; |
| 1965 | default: |
| 1966 | unreachable("Attempted to store unknown type"); |
| 1967 | break; |
| 1968 | } |
| 1969 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1970 | /* nir_intrinsic_component(store_intr) encodes the |
| 1971 | * destination component start. Source component offset |
| 1972 | * adjustment is taken care of in |
| 1973 | * install_registers_instr(), when offset_swizzle() is |
| 1974 | * called. |
| 1975 | */ |
| 1976 | unsigned src_component = COMPONENT_X; |
| 1977 | |
| 1978 | assert(nr_comp > 0); |
| 1979 | for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) { |
| 1980 | st.swizzle[0][i] = src_component; |
| 1981 | if (i >= dst_component && i < dst_component + nr_comp - 1) |
| 1982 | src_component++; |
| 1983 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1984 | |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1985 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1986 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1987 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1988 | assert(0); |
| 1989 | } |
| 1990 | |
| 1991 | break; |
| 1992 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1993 | /* Special case of store_output for lowered blend shaders */ |
| 1994 | case nir_intrinsic_store_raw_output_pan: |
| 1995 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 1996 | reg = nir_src_index(ctx, &instr->src[0]); |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1997 | emit_fragment_store(ctx, reg, ~0, ~0, ctx->blend_rt); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1998 | break; |
| 1999 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2000 | case nir_intrinsic_store_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2001 | case nir_intrinsic_store_shared: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2002 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2003 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 2004 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 2005 | emit_global(ctx, &instr->instr, false, reg, &instr->src[1], instr->intrinsic == nir_intrinsic_store_shared); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 2006 | break; |
| 2007 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2008 | case nir_intrinsic_load_ssbo_address: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2009 | emit_sysval_read(ctx, &instr->instr, 1, 0); |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 2010 | break; |
| 2011 | |
Jason Ekstrand | 9750164 | 2020-09-22 03:24:45 -0500 | [diff] [blame] | 2012 | case nir_intrinsic_get_ssbo_size: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2013 | emit_sysval_read(ctx, &instr->instr, 1, 8); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2014 | break; |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 2015 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2016 | case nir_intrinsic_load_viewport_scale: |
| 2017 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 2018 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 2019 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2020 | emit_sysval_read(ctx, &instr->instr, 3, 0); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2021 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2022 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 2023 | case nir_intrinsic_load_work_group_id: |
| 2024 | case nir_intrinsic_load_local_invocation_id: |
| 2025 | emit_compute_builtin(ctx, instr); |
| 2026 | break; |
| 2027 | |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 2028 | case nir_intrinsic_load_vertex_id: |
| 2029 | case nir_intrinsic_load_instance_id: |
| 2030 | emit_vertex_builtin(ctx, instr); |
| 2031 | break; |
| 2032 | |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2033 | case nir_intrinsic_load_sample_mask_in: |
| 2034 | emit_special(ctx, instr, 96); |
| 2035 | break; |
| 2036 | |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2037 | case nir_intrinsic_load_sample_id: |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 2038 | emit_special(ctx, instr, 97); |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 2039 | break; |
| 2040 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 2041 | case nir_intrinsic_memory_barrier_buffer: |
| 2042 | case nir_intrinsic_memory_barrier_shared: |
| 2043 | break; |
| 2044 | |
| 2045 | case nir_intrinsic_control_barrier: |
| 2046 | schedule_barrier(ctx); |
| 2047 | emit_control_barrier(ctx); |
| 2048 | schedule_barrier(ctx); |
| 2049 | break; |
| 2050 | |
Italo Nicola | d7b6d2e | 2020-08-31 17:32:30 +0000 | [diff] [blame] | 2051 | ATOMIC_CASE(ctx, instr, add, add); |
| 2052 | ATOMIC_CASE(ctx, instr, and, and); |
| 2053 | ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg); |
| 2054 | ATOMIC_CASE(ctx, instr, exchange, xchg); |
| 2055 | ATOMIC_CASE(ctx, instr, imax, imax); |
| 2056 | ATOMIC_CASE(ctx, instr, imin, imin); |
| 2057 | ATOMIC_CASE(ctx, instr, or, or); |
| 2058 | ATOMIC_CASE(ctx, instr, umax, umax); |
| 2059 | ATOMIC_CASE(ctx, instr, umin, umin); |
| 2060 | ATOMIC_CASE(ctx, instr, xor, xor); |
| 2061 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2062 | default: |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2063 | fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2064 | assert(0); |
| 2065 | break; |
| 2066 | } |
| 2067 | } |
| 2068 | |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2069 | /* Returns dimension with 0 special casing cubemaps */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2070 | static unsigned |
| 2071 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 2072 | { |
| 2073 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2074 | case GLSL_SAMPLER_DIM_1D: |
| 2075 | case GLSL_SAMPLER_DIM_BUF: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2076 | return 1; |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 2077 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2078 | case GLSL_SAMPLER_DIM_2D: |
Alyssa Rosenzweig | a2748d4 | 2020-06-30 15:31:30 -0400 | [diff] [blame] | 2079 | case GLSL_SAMPLER_DIM_MS: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2080 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 2081 | case GLSL_SAMPLER_DIM_RECT: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2082 | return 2; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2083 | |
| 2084 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2085 | return 3; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2086 | |
| 2087 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 2088 | return 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2089 | |
| 2090 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2091 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2092 | assert(0); |
| 2093 | return 0; |
| 2094 | } |
| 2095 | } |
| 2096 | |
Alyssa Rosenzweig | c6c906e | 2020-05-21 18:02:38 -0400 | [diff] [blame] | 2097 | /* Tries to attach an explicit LOD or bias as a constant. Returns whether this |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2098 | * was successful */ |
| 2099 | |
| 2100 | static bool |
| 2101 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2102 | compiler_context *ctx, |
| 2103 | nir_src lod, |
| 2104 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 2105 | { |
| 2106 | /* To attach as constant, it has to *be* constant */ |
| 2107 | |
| 2108 | if (!nir_src_is_const(lod)) |
| 2109 | return false; |
| 2110 | |
| 2111 | float f = nir_src_as_float(lod); |
| 2112 | |
| 2113 | /* Break into fixed-point */ |
| 2114 | signed lod_int = f; |
| 2115 | float lod_frac = f - lod_int; |
| 2116 | |
| 2117 | /* Carry over negative fractions */ |
| 2118 | if (lod_frac < 0.0) { |
| 2119 | lod_int--; |
| 2120 | lod_frac += 1.0; |
| 2121 | } |
| 2122 | |
| 2123 | /* Encode */ |
| 2124 | word->bias = float_to_ubyte(lod_frac); |
| 2125 | word->bias_int = lod_int; |
| 2126 | |
| 2127 | return true; |
| 2128 | } |
| 2129 | |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2130 | static enum mali_texture_mode |
| 2131 | mdg_texture_mode(nir_tex_instr *instr) |
| 2132 | { |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2133 | if (instr->op == nir_texop_tg4 && instr->is_shadow) |
| 2134 | return TEXTURE_GATHER_SHADOW; |
| 2135 | else if (instr->op == nir_texop_tg4) |
| 2136 | return TEXTURE_GATHER_X + instr->component; |
| 2137 | else if (instr->is_shadow) |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2138 | return TEXTURE_SHADOW; |
| 2139 | else |
| 2140 | return TEXTURE_NORMAL; |
| 2141 | } |
| 2142 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2143 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2144 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2145 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2146 | { |
| 2147 | /* TODO */ |
| 2148 | //assert (!instr->sampler); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2149 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2150 | nir_dest *dest = &instr->dest; |
| 2151 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2152 | int texture_index = instr->texture_index; |
| 2153 | int sampler_index = texture_index; |
| 2154 | |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2155 | nir_alu_type dest_base = nir_alu_type_get_base_type(instr->dest_type); |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2156 | nir_alu_type dest_type = dest_base | nir_dest_bit_size(*dest); |
| 2157 | |
| 2158 | /* texture instructions support float outmods */ |
| 2159 | unsigned outmod = midgard_outmod_none; |
| 2160 | if (dest_base == nir_type_float) { |
| 2161 | outmod = mir_determine_float_outmod(ctx, &dest, 0); |
| 2162 | } |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2163 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2164 | midgard_instruction ins = { |
| 2165 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 2166 | .mask = 0xF, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2167 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2168 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2169 | .dest_type = dest_type, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2170 | .swizzle = SWIZZLE_IDENTITY_4, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 2171 | .outmod = outmod, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 2172 | .op = midgard_texop, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2173 | .texture = { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2174 | .format = midgard_tex_format(instr->sampler_dim), |
| 2175 | .texture_handle = texture_index, |
| 2176 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 2177 | .mode = mdg_texture_mode(instr) |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2178 | } |
| 2179 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 2180 | |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2181 | if (instr->is_shadow && !instr->is_new_style_shadow && instr->op != nir_texop_tg4) |
Icecream95 | d1290e7 | 2020-05-12 10:16:31 +1200 | [diff] [blame] | 2182 | for (int i = 0; i < 4; ++i) |
| 2183 | ins.swizzle[0][i] = COMPONENT_X; |
| 2184 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2185 | /* We may need a temporary for the coordinate */ |
| 2186 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2187 | bool needs_temp_coord = |
| 2188 | (midgard_texop == TEXTURE_OP_TEXEL_FETCH) || |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2189 | (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) || |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2190 | (instr->is_shadow); |
| 2191 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2192 | unsigned coords = needs_temp_coord ? make_compiler_temp_reg(ctx) : 0; |
| 2193 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2194 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2195 | int index = nir_src_index(ctx, &instr->src[i].src); |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2196 | unsigned nr_components = nir_src_num_components(instr->src[i].src); |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2197 | unsigned sz = nir_src_bit_size(instr->src[i].src); |
| 2198 | nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2199 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2200 | switch (instr->src[i].src_type) { |
| 2201 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2202 | emit_explicit_constant(ctx, index, index); |
| 2203 | |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2204 | unsigned coord_mask = mask_of(instr->coord_components); |
| 2205 | |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2206 | bool flip_zw = (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) && (coord_mask & (1 << COMPONENT_Z)); |
| 2207 | |
| 2208 | if (flip_zw) |
| 2209 | coord_mask ^= ((1 << COMPONENT_Z) | (1 << COMPONENT_W)); |
| 2210 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2211 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
| 2212 | /* texelFetch is undefined on samplerCube */ |
| 2213 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 2214 | |
| 2215 | /* For cubemaps, we use a special ld/st op to |
| 2216 | * select the face and copy the xy into the |
| 2217 | * texture register */ |
| 2218 | |
| 2219 | midgard_instruction ld = m_ld_cubemap_coords(coords, 0); |
| 2220 | ld.src[1] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2221 | ld.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2222 | ld.mask = 0x3; /* xy */ |
| 2223 | ld.load_store.arg_1 = 0x20; |
| 2224 | ld.swizzle[1][3] = COMPONENT_X; |
| 2225 | emit_mir_instruction(ctx, ld); |
| 2226 | |
| 2227 | /* xyzw -> xyxx */ |
| 2228 | ins.swizzle[1][2] = instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
| 2229 | ins.swizzle[1][3] = COMPONENT_X; |
| 2230 | } else if (needs_temp_coord) { |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2231 | /* mov coord_temp, coords */ |
| 2232 | midgard_instruction mov = v_mov(index, coords); |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2233 | mov.mask = coord_mask; |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 2234 | |
| 2235 | if (flip_zw) |
| 2236 | mov.swizzle[1][COMPONENT_W] = COMPONENT_Z; |
| 2237 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2238 | emit_mir_instruction(ctx, mov); |
| 2239 | } else { |
| 2240 | coords = index; |
| 2241 | } |
| 2242 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2243 | ins.src[1] = coords; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2244 | ins.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2245 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2246 | /* Texelfetch coordinates uses all four elements |
| 2247 | * (xyz/index) regardless of texture dimensionality, |
| 2248 | * which means it's necessary to zero the unused |
| 2249 | * components to keep everything happy */ |
| 2250 | |
| 2251 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2252 | /* mov index.zw, #0, or generalized */ |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2253 | midgard_instruction mov = |
| 2254 | v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), coords); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2255 | mov.has_constants = true; |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2256 | mov.mask = coord_mask ^ 0xF; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2257 | emit_mir_instruction(ctx, mov); |
| 2258 | } |
| 2259 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2260 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2261 | /* Array component in w but NIR wants it in z, |
| 2262 | * but if we have a temp coord we already fixed |
| 2263 | * that up */ |
| 2264 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2265 | if (nr_components == 3) { |
| 2266 | ins.swizzle[1][2] = COMPONENT_Z; |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2267 | ins.swizzle[1][3] = needs_temp_coord ? COMPONENT_W : COMPONENT_Z; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2268 | } else if (nr_components == 2) { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2269 | ins.swizzle[1][2] = |
| 2270 | instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2271 | ins.swizzle[1][3] = COMPONENT_X; |
| 2272 | } else |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2273 | unreachable("Invalid texture 2D components"); |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 2274 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2275 | |
Alyssa Rosenzweig | 64b2fe9 | 2019-12-20 12:38:24 -0500 | [diff] [blame] | 2276 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 2277 | /* We zeroed */ |
| 2278 | ins.swizzle[1][2] = COMPONENT_Z; |
| 2279 | ins.swizzle[1][3] = COMPONENT_W; |
| 2280 | } |
| 2281 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2282 | break; |
| 2283 | } |
| 2284 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 2285 | case nir_tex_src_bias: |
| 2286 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2287 | /* Try as a constant if we can */ |
| 2288 | |
| 2289 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 2290 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 2291 | break; |
| 2292 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2293 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2294 | ins.src[2] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2295 | ins.src_types[2] = T; |
Alyssa Rosenzweig | 72e5749 | 2019-12-20 12:34:20 -0500 | [diff] [blame] | 2296 | |
| 2297 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2298 | ins.swizzle[2][c] = COMPONENT_X; |
| 2299 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2300 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2301 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2302 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2303 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2304 | |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2305 | case nir_tex_src_offset: { |
| 2306 | ins.texture.offset_register = true; |
| 2307 | ins.src[3] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2308 | ins.src_types[3] = T; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2309 | |
| 2310 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2311 | ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c; |
| 2312 | |
| 2313 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | 4ec1f95 | 2019-12-20 12:58:10 -0500 | [diff] [blame] | 2314 | break; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2315 | }; |
| 2316 | |
Alyssa Rosenzweig | 6d9f951 | 2020-06-30 15:31:39 -0400 | [diff] [blame] | 2317 | case nir_tex_src_comparator: |
| 2318 | case nir_tex_src_ms_index: { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2319 | unsigned comp = COMPONENT_Z; |
| 2320 | |
| 2321 | /* mov coord_temp.foo, coords */ |
| 2322 | midgard_instruction mov = v_mov(index, coords); |
| 2323 | mov.mask = 1 << comp; |
| 2324 | |
| 2325 | for (unsigned i = 0; i < MIR_VEC_COMPONENTS; ++i) |
| 2326 | mov.swizzle[1][i] = COMPONENT_X; |
| 2327 | |
| 2328 | emit_mir_instruction(ctx, mov); |
| 2329 | break; |
| 2330 | } |
| 2331 | |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2332 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2333 | fprintf(stderr, "Unknown texture source type: %d\n", instr->src[i].src_type); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2334 | assert(0); |
| 2335 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2336 | } |
| 2337 | } |
| 2338 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2339 | emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2340 | } |
| 2341 | |
| 2342 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2343 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 2344 | { |
| 2345 | switch (instr->op) { |
| 2346 | case nir_texop_tex: |
| 2347 | case nir_texop_txb: |
| 2348 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 2349 | break; |
| 2350 | case nir_texop_txl: |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2351 | case nir_texop_tg4: |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2352 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 2353 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2354 | case nir_texop_txf: |
Alyssa Rosenzweig | 63a8722 | 2020-06-30 15:32:01 -0400 | [diff] [blame] | 2355 | case nir_texop_txf_ms: |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2356 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 2357 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2358 | case nir_texop_txs: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2359 | emit_sysval_read(ctx, &instr->instr, 4, 0); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2360 | break; |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2361 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2362 | fprintf(stderr, "Unhandled texture op: %d\n", instr->op); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2363 | assert(0); |
| 2364 | } |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2365 | } |
| 2366 | } |
| 2367 | |
| 2368 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2369 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 2370 | { |
| 2371 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2372 | case nir_jump_break: { |
| 2373 | /* Emit a branch out of the loop */ |
| 2374 | struct midgard_instruction br = v_branch(false, false); |
| 2375 | br.branch.target_type = TARGET_BREAK; |
| 2376 | br.branch.target_break = ctx->current_loop_depth; |
| 2377 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2378 | break; |
| 2379 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2380 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2381 | default: |
| 2382 | DBG("Unknown jump type %d\n", instr->type); |
| 2383 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2384 | } |
| 2385 | } |
| 2386 | |
| 2387 | static void |
| 2388 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 2389 | { |
| 2390 | switch (instr->type) { |
| 2391 | case nir_instr_type_load_const: |
| 2392 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 2393 | break; |
| 2394 | |
| 2395 | case nir_instr_type_intrinsic: |
| 2396 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 2397 | break; |
| 2398 | |
| 2399 | case nir_instr_type_alu: |
| 2400 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 2401 | break; |
| 2402 | |
| 2403 | case nir_instr_type_tex: |
| 2404 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 2405 | break; |
| 2406 | |
| 2407 | case nir_instr_type_jump: |
| 2408 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 2409 | break; |
| 2410 | |
| 2411 | case nir_instr_type_ssa_undef: |
| 2412 | /* Spurious */ |
| 2413 | break; |
| 2414 | |
| 2415 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2416 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2417 | break; |
| 2418 | } |
| 2419 | } |
| 2420 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2421 | |
| 2422 | /* ALU instructions can inline or embed constants, which decreases register |
| 2423 | * pressure and saves space. */ |
| 2424 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2425 | #define CONDITIONAL_ATTACH(idx) { \ |
| 2426 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2427 | \ |
| 2428 | if (entry) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2429 | attach_constants(ctx, alu, entry, alu->src[idx] + 1); \ |
| 2430 | alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2431 | } \ |
| 2432 | } |
| 2433 | |
| 2434 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2435 | inline_alu_constants(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2436 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2437 | mir_foreach_instr_in_block(block, alu) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2438 | /* Other instructions cannot inline constants */ |
| 2439 | if (alu->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 2440 | if (alu->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2441 | |
| 2442 | /* If there is already a constant here, we can do nothing */ |
| 2443 | if (alu->has_constants) continue; |
| 2444 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2445 | CONDITIONAL_ATTACH(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2446 | |
| 2447 | if (!alu->has_constants) { |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2448 | CONDITIONAL_ATTACH(1) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2449 | } else if (!alu->inline_constant) { |
| 2450 | /* Corner case: _two_ vec4 constants, for instance with a |
| 2451 | * csel. For this case, we can only use a constant |
| 2452 | * register for one, we'll have to emit a move for the |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2453 | * other. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2454 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2455 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1); |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2456 | unsigned scratch = make_compiler_temp(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2457 | |
| 2458 | if (entry) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 2459 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch); |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2460 | attach_constants(ctx, &ins, entry, alu->src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2461 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2462 | /* Set the source */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2463 | alu->src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2464 | |
| 2465 | /* Inject us -before- the last instruction which set r31 */ |
Boris Brezillon | 938c5b0 | 2019-08-28 09:17:21 +0200 | [diff] [blame] | 2466 | mir_insert_instruction_before(ctx, mir_prev_op(alu), ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2467 | } |
| 2468 | } |
| 2469 | } |
| 2470 | } |
| 2471 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2472 | unsigned |
| 2473 | max_bitsize_for_alu(midgard_instruction *ins) |
| 2474 | { |
| 2475 | unsigned max_bitsize = 0; |
| 2476 | for (int i = 0; i < MIR_SRC_COUNT; i++) { |
| 2477 | if (ins->src[i] == ~0) continue; |
| 2478 | unsigned src_bitsize = nir_alu_type_get_type_size(ins->src_types[i]); |
| 2479 | max_bitsize = MAX2(src_bitsize, max_bitsize); |
| 2480 | } |
| 2481 | unsigned dst_bitsize = nir_alu_type_get_type_size(ins->dest_type); |
| 2482 | max_bitsize = MAX2(dst_bitsize, max_bitsize); |
| 2483 | |
| 2484 | /* We don't have fp16 LUTs, so we'll want to emit code like: |
| 2485 | * |
| 2486 | * vlut.fsinr hr0, hr0 |
| 2487 | * |
| 2488 | * where both input and output are 16-bit but the operation is carried |
| 2489 | * out in 32-bit |
| 2490 | */ |
| 2491 | |
| 2492 | switch (ins->op) { |
| 2493 | case midgard_alu_op_fsqrt: |
| 2494 | case midgard_alu_op_frcp: |
| 2495 | case midgard_alu_op_frsqrt: |
| 2496 | case midgard_alu_op_fsin: |
| 2497 | case midgard_alu_op_fcos: |
| 2498 | case midgard_alu_op_fexp2: |
| 2499 | case midgard_alu_op_flog2: |
| 2500 | max_bitsize = MAX2(max_bitsize, 32); |
| 2501 | break; |
| 2502 | |
| 2503 | default: |
| 2504 | break; |
| 2505 | } |
| 2506 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 2507 | /* High implies computing at a higher bitsize, e.g umul_high of 32-bit |
| 2508 | * requires computing at 64-bit */ |
| 2509 | if (midgard_is_integer_out_op(ins->op) && ins->outmod == midgard_outmod_int_high) { |
| 2510 | max_bitsize *= 2; |
| 2511 | assert(max_bitsize <= 64); |
| 2512 | } |
| 2513 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2514 | return max_bitsize; |
| 2515 | } |
| 2516 | |
| 2517 | midgard_reg_mode |
| 2518 | reg_mode_for_bitsize(unsigned bitsize) |
| 2519 | { |
| 2520 | switch (bitsize) { |
| 2521 | /* use 16 pipe for 8 since we don't support vec16 yet */ |
| 2522 | case 8: |
| 2523 | case 16: |
| 2524 | return midgard_reg_mode_16; |
| 2525 | case 32: |
| 2526 | return midgard_reg_mode_32; |
| 2527 | case 64: |
| 2528 | return midgard_reg_mode_64; |
| 2529 | default: |
| 2530 | unreachable("invalid bit size"); |
| 2531 | } |
| 2532 | } |
| 2533 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2534 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2535 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2536 | * constants can be demoted to inline constants, for space savings and |
| 2537 | * sometimes a performance boost */ |
| 2538 | |
| 2539 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2540 | embedded_to_inline_constant(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2541 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2542 | mir_foreach_instr_in_block(block, ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2543 | if (!ins->has_constants) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2544 | if (ins->has_inline_constant) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2545 | |
| 2546 | /* Blend constants must not be inlined by definition */ |
| 2547 | if (ins->has_blend_constant) continue; |
| 2548 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2549 | unsigned max_bitsize = max_bitsize_for_alu(ins); |
| 2550 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2551 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2552 | bool is_16 = max_bitsize == 16; |
| 2553 | bool is_32 = max_bitsize == 32; |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2554 | |
| 2555 | if (!(is_16 || is_32)) |
| 2556 | continue; |
| 2557 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2558 | /* src1 cannot be an inline constant due to encoding |
| 2559 | * restrictions. So, if possible we try to flip the arguments |
| 2560 | * in that case */ |
| 2561 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2562 | int op = ins->op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2563 | |
Alyssa Rosenzweig | ba9f3d1 | 2020-04-30 13:11:52 -0400 | [diff] [blame] | 2564 | if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT) && |
| 2565 | alu_opcode_props[op].props & OP_COMMUTES) { |
| 2566 | mir_flip(ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2567 | } |
| 2568 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2569 | if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2570 | /* Component is from the swizzle. Take a nonzero component */ |
| 2571 | assert(ins->mask); |
| 2572 | unsigned first_comp = ffs(ins->mask) - 1; |
| 2573 | unsigned component = ins->swizzle[1][first_comp]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2574 | |
| 2575 | /* Scale constant appropriately, if we can legally */ |
Icecream95 | d97aaad | 2020-06-05 20:17:27 +1200 | [diff] [blame] | 2576 | int16_t scaled_constant = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2577 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2578 | if (is_16) { |
| 2579 | scaled_constant = ins->constants.u16[component]; |
| 2580 | } else if (midgard_is_integer_op(op)) { |
| 2581 | scaled_constant = ins->constants.u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2582 | |
| 2583 | /* Constant overflow after resize */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2584 | if (scaled_constant != ins->constants.u32[component]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2585 | continue; |
| 2586 | } else { |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2587 | float original = ins->constants.f32[component]; |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2588 | scaled_constant = _mesa_float_to_half(original); |
| 2589 | |
| 2590 | /* Check for loss of precision. If this is |
| 2591 | * mediump, we don't care, but for a highp |
| 2592 | * shader, we need to pay attention. NIR |
| 2593 | * doesn't yet tell us which mode we're in! |
| 2594 | * Practically this prevents most constants |
| 2595 | * from being inlined, sadly. */ |
| 2596 | |
| 2597 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2598 | |
| 2599 | if (fp32 != original) |
| 2600 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2601 | } |
| 2602 | |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 2603 | /* Should've been const folded */ |
| 2604 | if (ins->src_abs[1] || ins->src_neg[1]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2605 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2606 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2607 | /* Make sure that the constant is not itself a vector |
| 2608 | * by checking if all accessed values are the same. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2609 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2610 | const midgard_constants *cons = &ins->constants; |
| 2611 | uint32_t value = is_16 ? cons->u16[component] : cons->u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2612 | |
| 2613 | bool is_vector = false; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2614 | unsigned mask = effective_writemask(ins->op, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2615 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2616 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2617 | /* We only care if this component is actually used */ |
| 2618 | if (!(mask & (1 << c))) |
| 2619 | continue; |
| 2620 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2621 | uint32_t test = is_16 ? |
| 2622 | cons->u16[ins->swizzle[1][c]] : |
| 2623 | cons->u32[ins->swizzle[1][c]]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2624 | |
| 2625 | if (test != value) { |
| 2626 | is_vector = true; |
| 2627 | break; |
| 2628 | } |
| 2629 | } |
| 2630 | |
| 2631 | if (is_vector) |
| 2632 | continue; |
| 2633 | |
| 2634 | /* Get rid of the embedded constant */ |
| 2635 | ins->has_constants = false; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2636 | ins->src[1] = ~0; |
| 2637 | ins->has_inline_constant = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2638 | ins->inline_constant = scaled_constant; |
| 2639 | } |
| 2640 | } |
| 2641 | } |
| 2642 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2643 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2644 | * per block is legal semantically */ |
| 2645 | |
| 2646 | static void |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 2647 | midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2648 | { |
| 2649 | bool branched = false; |
| 2650 | |
| 2651 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2652 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2653 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2654 | if (branched) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2655 | mir_remove_instruction(ins); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2656 | |
| 2657 | branched = true; |
| 2658 | } |
| 2659 | } |
| 2660 | |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2661 | /* We want to force the invert on AND/OR to the second slot to legalize into |
| 2662 | * iandnot/iornot. The relevant patterns are for AND (and OR respectively) |
| 2663 | * |
| 2664 | * ~a & #b = ~a & ~(#~b) |
| 2665 | * ~a & b = b & ~a |
| 2666 | */ |
| 2667 | |
| 2668 | static void |
| 2669 | midgard_legalize_invert(compiler_context *ctx, midgard_block *block) |
| 2670 | { |
| 2671 | mir_foreach_instr_in_block(block, ins) { |
| 2672 | if (ins->type != TAG_ALU_4) continue; |
| 2673 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2674 | if (ins->op != midgard_alu_op_iand && |
| 2675 | ins->op != midgard_alu_op_ior) continue; |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2676 | |
| 2677 | if (ins->src_invert[1] || !ins->src_invert[0]) continue; |
| 2678 | |
| 2679 | if (ins->has_inline_constant) { |
| 2680 | /* ~(#~a) = ~(~#a) = a, so valid, and forces both |
| 2681 | * inverts on */ |
| 2682 | ins->inline_constant = ~ins->inline_constant; |
| 2683 | ins->src_invert[1] = true; |
| 2684 | } else { |
| 2685 | /* Flip to the right invert order. Note |
| 2686 | * has_inline_constant false by assumption on the |
| 2687 | * branch, so flipping makes sense. */ |
| 2688 | mir_flip(ins); |
| 2689 | } |
| 2690 | } |
| 2691 | } |
| 2692 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2693 | static unsigned |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2694 | emit_fragment_epilogue(compiler_context *ctx, unsigned rt) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2695 | { |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2696 | /* Loop to ourselves */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2697 | midgard_instruction *br = ctx->writeout_branch[rt]; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2698 | struct midgard_instruction ins = v_branch(false, false); |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 2699 | ins.writeout = br->writeout; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2700 | ins.branch.target_block = ctx->block_count - 1; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2701 | ins.constants.u32[0] = br->constants.u32[0]; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 2702 | memcpy(&ins.src_types, &br->src_types, sizeof(ins.src_types)); |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2703 | emit_mir_instruction(ctx, ins); |
| 2704 | |
Alyssa Rosenzweig | 3448b26 | 2019-12-03 10:37:01 -0500 | [diff] [blame] | 2705 | ctx->current_block->epilogue = true; |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2706 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2707 | return ins.branch.target_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2708 | } |
| 2709 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2710 | static midgard_block * |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2711 | emit_block_init(compiler_context *ctx) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2712 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2713 | midgard_block *this_block = ctx->after_block; |
| 2714 | ctx->after_block = NULL; |
| 2715 | |
| 2716 | if (!this_block) |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2717 | this_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2718 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2719 | list_addtail(&this_block->base.link, &ctx->blocks); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2720 | |
Alyssa Rosenzweig | c5dd1d5 | 2020-03-11 08:22:08 -0400 | [diff] [blame] | 2721 | this_block->scheduled = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2722 | ++ctx->block_count; |
| 2723 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2724 | /* Set up current block */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2725 | list_inithead(&this_block->base.instructions); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2726 | ctx->current_block = this_block; |
| 2727 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2728 | return this_block; |
| 2729 | } |
| 2730 | |
| 2731 | static midgard_block * |
| 2732 | emit_block(compiler_context *ctx, nir_block *block) |
| 2733 | { |
| 2734 | midgard_block *this_block = emit_block_init(ctx); |
| 2735 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2736 | nir_foreach_instr(instr, block) { |
| 2737 | emit_instr(ctx, instr); |
| 2738 | ++ctx->instruction_count; |
| 2739 | } |
| 2740 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2741 | return this_block; |
| 2742 | } |
| 2743 | |
| 2744 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2745 | |
| 2746 | static void |
| 2747 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2748 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2749 | midgard_block *before_block = ctx->current_block; |
| 2750 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2751 | /* Speculatively emit the branch, but we can't fill it in until later */ |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2752 | bool inv = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2753 | EMIT(branch, true, true); |
| 2754 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2755 | then_branch->src[0] = mir_get_branch_cond(&nif->condition, &inv); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 2756 | then_branch->src_types[0] = nir_type_uint32; |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2757 | then_branch->branch.invert_conditional = !inv; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2758 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2759 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2760 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2761 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2762 | |
| 2763 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2764 | EMIT(branch, false, false); |
| 2765 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2766 | |
| 2767 | /* Emit second block, and check if it's empty */ |
| 2768 | |
| 2769 | int else_idx = ctx->block_count; |
| 2770 | int count_in = ctx->instruction_count; |
| 2771 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2772 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2773 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2774 | |
| 2775 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2776 | |
| 2777 | assert(then_block); |
| 2778 | assert(else_block); |
| 2779 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2780 | if (ctx->instruction_count == count_in) { |
| 2781 | /* The else block is empty, so don't emit an exit jump */ |
| 2782 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2783 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2784 | } else { |
| 2785 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2786 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2787 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2788 | |
| 2789 | /* Wire up the successors */ |
| 2790 | |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2791 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2792 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2793 | pan_block_add_successor(&before_block->base, &then_block->base); |
| 2794 | pan_block_add_successor(&before_block->base, &else_block->base); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2795 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2796 | pan_block_add_successor(&end_then_block->base, &ctx->after_block->base); |
| 2797 | pan_block_add_successor(&end_else_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2798 | } |
| 2799 | |
| 2800 | static void |
| 2801 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2802 | { |
| 2803 | /* Remember where we are */ |
| 2804 | midgard_block *start_block = ctx->current_block; |
| 2805 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2806 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2807 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2808 | |
| 2809 | /* Get index from before the body so we can loop back later */ |
| 2810 | int start_idx = ctx->block_count; |
| 2811 | |
| 2812 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2813 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2814 | |
| 2815 | /* Branch back to loop back */ |
| 2816 | struct midgard_instruction br_back = v_branch(false, false); |
| 2817 | br_back.branch.target_block = start_idx; |
| 2818 | emit_mir_instruction(ctx, br_back); |
| 2819 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2820 | /* Mark down that branch in the graph. */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2821 | pan_block_add_successor(&start_block->base, &loop_block->base); |
| 2822 | pan_block_add_successor(&ctx->current_block->base, &loop_block->base); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2823 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2824 | /* Find the index of the block about to follow us (note: we don't add |
| 2825 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2826 | int break_block_idx = ctx->block_count; |
| 2827 | |
| 2828 | /* Fix up the break statements we emitted to point to the right place, |
| 2829 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2830 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2831 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2832 | mir_foreach_block_from(ctx, start_block, _block) { |
| 2833 | mir_foreach_instr_in_block(((midgard_block *) _block), ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2834 | if (ins->type != TAG_ALU_4) continue; |
| 2835 | if (!ins->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2836 | |
| 2837 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2838 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2839 | |
| 2840 | /* It's a break! Check if it's our break */ |
| 2841 | if (ins->branch.target_break != loop_idx) continue; |
| 2842 | |
| 2843 | /* Okay, cool, we're breaking out of this loop. |
| 2844 | * Rewrite from a break to a goto */ |
| 2845 | |
| 2846 | ins->branch.target_type = TARGET_GOTO; |
| 2847 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2848 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2849 | pan_block_add_successor(_block, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2850 | } |
| 2851 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2852 | |
| 2853 | /* Now that we've finished emitting the loop, free up the depth again |
| 2854 | * so we play nice with recursion amid nested loops */ |
| 2855 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2856 | |
| 2857 | /* Dump loop stats */ |
| 2858 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2859 | } |
| 2860 | |
| 2861 | static midgard_block * |
| 2862 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2863 | { |
| 2864 | midgard_block *start_block = NULL; |
| 2865 | |
| 2866 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2867 | switch (node->type) { |
| 2868 | case nir_cf_node_block: { |
| 2869 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2870 | |
| 2871 | if (!start_block) |
| 2872 | start_block = block; |
| 2873 | |
| 2874 | break; |
| 2875 | } |
| 2876 | |
| 2877 | case nir_cf_node_if: |
| 2878 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2879 | break; |
| 2880 | |
| 2881 | case nir_cf_node_loop: |
| 2882 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2883 | break; |
| 2884 | |
| 2885 | case nir_cf_node_function: |
| 2886 | assert(0); |
| 2887 | break; |
| 2888 | } |
| 2889 | } |
| 2890 | |
| 2891 | return start_block; |
| 2892 | } |
| 2893 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2894 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2895 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2896 | * until we find one that 'works' */ |
| 2897 | |
Italo Nicola | 8150c1d | 2020-07-29 20:14:55 +0000 | [diff] [blame] | 2898 | unsigned |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2899 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2900 | { |
| 2901 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2902 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2903 | mir_foreach_block_from(ctx, initial_block, _v) { |
| 2904 | midgard_block *v = (midgard_block *) _v; |
Alyssa Rosenzweig | 45ac8ea | 2019-11-04 10:32:49 -0500 | [diff] [blame] | 2905 | if (v->quadword_count) { |
| 2906 | midgard_bundle *initial_bundle = |
| 2907 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2908 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2909 | return initial_bundle->tag; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2910 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2911 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2912 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2913 | /* Default to a tag 1 which will break from the shader, in case we jump |
| 2914 | * to the exit block (i.e. `return` in a compute shader) */ |
| 2915 | |
| 2916 | return 1; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2917 | } |
| 2918 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2919 | /* For each fragment writeout instruction, generate a writeout loop to |
| 2920 | * associate with it */ |
| 2921 | |
| 2922 | static void |
| 2923 | mir_add_writeout_loops(compiler_context *ctx) |
| 2924 | { |
| 2925 | for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) { |
| 2926 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 2927 | if (!br) continue; |
| 2928 | |
| 2929 | unsigned popped = br->branch.target_block; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2930 | pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base), &ctx->current_block->base); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2931 | br->branch.target_block = emit_fragment_epilogue(ctx, rt); |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2932 | br->branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2933 | |
| 2934 | /* If we have more RTs, we'll need to restore back after our |
| 2935 | * loop terminates */ |
| 2936 | |
| 2937 | if ((rt + 1) < ARRAY_SIZE(ctx->writeout_branch) && ctx->writeout_branch[rt + 1]) { |
| 2938 | midgard_instruction uncond = v_branch(false, false); |
| 2939 | uncond.branch.target_block = popped; |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2940 | uncond.branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2941 | emit_mir_instruction(ctx, uncond); |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2942 | pan_block_add_successor(&ctx->current_block->base, &(mir_get_block(ctx, popped)->base)); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2943 | schedule_barrier(ctx); |
| 2944 | } else { |
| 2945 | /* We're last, so we can terminate here */ |
| 2946 | br->last_writeout = true; |
| 2947 | } |
| 2948 | } |
| 2949 | } |
| 2950 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2951 | int |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 2952 | midgard_compile_shader_nir(nir_shader *nir, panfrost_program *program, |
| 2953 | const struct panfrost_compile_inputs *inputs) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2954 | { |
| 2955 | struct util_dynarray *compiled = &program->compiled; |
| 2956 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2957 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2958 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2959 | /* TODO: Bound against what? */ |
| 2960 | compiler_context *ctx = rzalloc(NULL, compiler_context); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2961 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2962 | ctx->nir = nir; |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2963 | ctx->stage = nir->info.stage; |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 2964 | ctx->is_blend = inputs->is_blend; |
| 2965 | ctx->blend_rt = MIDGARD_COLOR_RT0 + inputs->blend.rt; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 2966 | ctx->blend_input = ~0; |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 2967 | ctx->blend_src1 = ~0; |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 2968 | ctx->quirks = midgard_get_quirks(inputs->gpu_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2969 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2970 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2971 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2972 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2973 | ctx->uniform_cutoff = 8; |
| 2974 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2975 | /* Initialize at a global (not block) level hash tables */ |
| 2976 | |
| 2977 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2978 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2979 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2980 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2981 | * messed with our I/O quite a bit already) */ |
| 2982 | |
| 2983 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2984 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2985 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2986 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | 2023716 | 2019-08-26 12:14:11 -0700 | [diff] [blame] | 2987 | NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2988 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2989 | |
| 2990 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2991 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 2992 | NIR_PASS_V(nir, nir_split_var_copies); |
| 2993 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2994 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 2995 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2996 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2997 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 2998 | unsigned pan_quirks = panfrost_get_quirks(inputs->gpu_id); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 2999 | NIR_PASS_V(nir, pan_lower_framebuffer, |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 3000 | inputs->rt_formats, inputs->is_blend, pan_quirks); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 3001 | |
Jason Ekstrand | b019b22 | 2020-06-10 17:54:25 -0500 | [diff] [blame] | 3002 | NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out, |
| 3003 | glsl_type_size, 0); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 3004 | NIR_PASS_V(nir, nir_lower_ssbo); |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 3005 | NIR_PASS_V(nir, midgard_nir_lower_zs_store); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3006 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3007 | /* Optimisation passes */ |
| 3008 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 3009 | optimise_nir(nir, ctx->quirks, inputs->is_blend); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3010 | |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 3011 | NIR_PASS_V(nir, midgard_nir_reorder_writeout); |
| 3012 | |
Icecream95 | 756441b | 2020-09-26 12:19:14 +1200 | [diff] [blame] | 3013 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !nir->info.internal) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3014 | nir_print_shader(nir, stdout); |
| 3015 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3016 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 3017 | /* Assign sysvals and counts, now that we're sure |
| 3018 | * (post-optimisation) */ |
| 3019 | |
Alyssa Rosenzweig | 680fb05 | 2020-08-18 08:31:42 -0400 | [diff] [blame] | 3020 | panfrost_nir_assign_sysvals(&ctx->sysvals, ctx, nir); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 3021 | program->sysval_count = ctx->sysvals.sysval_count; |
| 3022 | memcpy(program->sysvals, ctx->sysvals.sysvals, sizeof(ctx->sysvals.sysvals[0]) * ctx->sysvals.sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3023 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3024 | nir_foreach_function(func, nir) { |
| 3025 | if (!func->impl) |
| 3026 | continue; |
| 3027 | |
| 3028 | list_inithead(&ctx->blocks); |
| 3029 | ctx->block_count = 0; |
| 3030 | ctx->func = func; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3031 | ctx->already_emitted = calloc(BITSET_WORDS(func->impl->ssa_alloc), sizeof(BITSET_WORD)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3032 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 3033 | if (nir->info.outputs_read && !inputs->is_blend) { |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 3034 | emit_block_init(ctx); |
| 3035 | |
| 3036 | struct midgard_instruction wait = v_branch(false, false); |
| 3037 | wait.branch.target_type = TARGET_TILEBUF_WAIT; |
| 3038 | |
| 3039 | emit_mir_instruction(ctx, wait); |
| 3040 | |
| 3041 | ++ctx->instruction_count; |
| 3042 | } |
| 3043 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3044 | emit_cf_list(ctx, &func->impl->body); |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 3045 | free(ctx->already_emitted); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3046 | break; /* TODO: Multi-function shaders */ |
| 3047 | } |
| 3048 | |
| 3049 | util_dynarray_init(compiled, NULL); |
| 3050 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3051 | /* Per-block lowering before opts */ |
| 3052 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3053 | mir_foreach_block(ctx, _block) { |
| 3054 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3055 | inline_alu_constants(ctx, block); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 3056 | embedded_to_inline_constant(ctx, block); |
| 3057 | } |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3058 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3059 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3060 | bool progress = false; |
| 3061 | |
| 3062 | do { |
| 3063 | progress = false; |
Alyssa Rosenzweig | fc06b8b | 2020-05-06 17:34:09 -0400 | [diff] [blame] | 3064 | progress |= midgard_opt_dead_code_eliminate(ctx); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3065 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3066 | mir_foreach_block(ctx, _block) { |
| 3067 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3068 | progress |= midgard_opt_copy_prop(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 3069 | progress |= midgard_opt_combine_projection(ctx, block); |
| 3070 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 3071 | } |
| 3072 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 3073 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3074 | mir_foreach_block(ctx, _block) { |
| 3075 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 3076 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 3077 | midgard_legalize_invert(ctx, block); |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 3078 | midgard_cull_dead_branch(ctx, block); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 3079 | } |
| 3080 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3081 | if (ctx->stage == MESA_SHADER_FRAGMENT) |
| 3082 | mir_add_writeout_loops(ctx); |
| 3083 | |
Alyssa Rosenzweig | 9a7f0e2 | 2020-05-12 13:26:32 -0400 | [diff] [blame] | 3084 | /* Analyze now that the code is known but before scheduling creates |
| 3085 | * pipeline registers which are harder to track */ |
| 3086 | mir_analyze_helper_terminate(ctx); |
| 3087 | mir_analyze_helper_requirements(ctx); |
| 3088 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3089 | /* Schedule! */ |
Robert Foss | 62adb65 | 2020-01-15 01:14:16 +0100 | [diff] [blame] | 3090 | midgard_schedule_program(ctx); |
Alyssa Rosenzweig | 9dc3b18 | 2019-12-06 09:32:38 -0500 | [diff] [blame] | 3091 | mir_ra(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3092 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3093 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 3094 | * sequence. Save instruction boundaries such that lookahead tags can |
| 3095 | * be assigned easily */ |
| 3096 | |
| 3097 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 3098 | |
| 3099 | int bundle_count = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3100 | mir_foreach_block(ctx, _block) { |
| 3101 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3102 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 3103 | } |
| 3104 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 3105 | int bundle_idx = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3106 | mir_foreach_block(ctx, _block) { |
| 3107 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3108 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 3109 | source_order_bundles[bundle_idx++] = bundle; |
| 3110 | } |
| 3111 | } |
| 3112 | |
| 3113 | int current_bundle = 0; |
| 3114 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3115 | /* Midgard prefetches instruction types, so during emission we |
| 3116 | * need to lookahead. Unless this is the last instruction, in |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3117 | * which we return 1. */ |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 3118 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3119 | mir_foreach_block(ctx, _block) { |
| 3120 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 3121 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3122 | int lookahead = 1; |
| 3123 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 3124 | if (!bundle->last_writeout && (current_bundle + 1 < bundle_count)) |
| 3125 | lookahead = source_order_bundles[current_bundle + 1]->tag; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3126 | |
Alyssa Rosenzweig | 30a393f | 2020-05-21 19:14:23 -0400 | [diff] [blame] | 3127 | emit_binary_bundle(ctx, block, bundle, compiled, lookahead); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3128 | ++current_bundle; |
| 3129 | } |
| 3130 | |
| 3131 | /* TODO: Free deeper */ |
| 3132 | //util_dynarray_fini(&block->instructions); |
| 3133 | } |
| 3134 | |
| 3135 | free(source_order_bundles); |
| 3136 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 3137 | /* Report the very first tag executed */ |
| 3138 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3139 | |
| 3140 | /* Deal with off-by-one related to the fencepost problem */ |
| 3141 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3142 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 3143 | |
| 3144 | program->blend_patch_offset = ctx->blend_constant_offset; |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 3145 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3146 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 3147 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !nir->info.internal) { |
| 3148 | disassemble_midgard(stdout, |
| 3149 | program->compiled.data, |
| 3150 | program->compiled.size, |
| 3151 | inputs->gpu_id, ctx->stage); |
| 3152 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3153 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame^] | 3154 | if ((midgard_debug & MIDGARD_DBG_SHADERDB || inputs->shaderdb) && |
| 3155 | !nir->info.internal) { |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3156 | unsigned nr_bundles = 0, nr_ins = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3157 | |
| 3158 | /* Count instructions and bundles */ |
| 3159 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 3160 | mir_foreach_block(ctx, _block) { |
| 3161 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3162 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3163 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 3164 | |
Alyssa Rosenzweig | 67909c8 | 2019-08-30 13:08:16 -0700 | [diff] [blame] | 3165 | mir_foreach_bundle_in_block(block, bun) |
| 3166 | nr_ins += bun->instruction_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3167 | } |
| 3168 | |
| 3169 | /* Calculate thread count. There are certain cutoffs by |
| 3170 | * register count for thread count */ |
| 3171 | |
| 3172 | unsigned nr_registers = program->work_register_count; |
| 3173 | |
| 3174 | unsigned nr_threads = |
| 3175 | (nr_registers <= 4) ? 4 : |
| 3176 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3177 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3178 | |
| 3179 | /* Dump stats */ |
| 3180 | |
| 3181 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3182 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3183 | "%u registers, %u threads, %u loops, " |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 3184 | "%u:%u spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3185 | SHADER_DB_COUNT++, |
Alyssa Rosenzweig | 014d2e4 | 2020-05-25 13:19:43 -0400 | [diff] [blame] | 3186 | ctx->is_blend ? "PAN_SHADER_BLEND" : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3187 | gl_shader_stage_name(ctx->stage), |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 3188 | nr_ins, nr_bundles, ctx->quadword_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 3189 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 3190 | ctx->loop_count, |
| 3191 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3192 | } |
| 3193 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 3194 | ralloc_free(ctx); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 3195 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3196 | return 0; |
| 3197 | } |