Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 36 | #include "compiler/nir/nir_builder.h" |
| 37 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 38 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 39 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 40 | #include "util/u_dynarray.h" |
| 41 | #include "util/list.h" |
| 42 | #include "main/mtypes.h" |
| 43 | |
| 44 | #include "midgard.h" |
| 45 | #include "midgard_nir.h" |
| 46 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 47 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 48 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 49 | #include "compiler.h" |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 50 | #include "midgard_quirks.h" |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 51 | #include "panfrost-quirks.h" |
| 52 | #include "panfrost/util/pan_lower_framebuffer.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 53 | |
| 54 | #include "disassemble.h" |
| 55 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 56 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 57 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 58 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 59 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 60 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 61 | }; |
| 62 | |
| 63 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 64 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 65 | unsigned SHADER_DB_COUNT = 0; |
| 66 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 67 | int midgard_debug = 0; |
| 68 | |
| 69 | #define DBG(fmt, ...) \ |
| 70 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 71 | fprintf(stderr, "%s:%d: "fmt, \ |
| 72 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 73 | static midgard_block * |
| 74 | create_empty_block(compiler_context *ctx) |
| 75 | { |
| 76 | midgard_block *blk = rzalloc(ctx, midgard_block); |
| 77 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 78 | blk->base.predecessors = _mesa_set_create(blk, |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 79 | _mesa_hash_pointer, |
| 80 | _mesa_key_pointer_equal); |
| 81 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 82 | blk->base.name = ctx->block_source_count++; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 83 | |
| 84 | return blk; |
| 85 | } |
| 86 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 87 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 88 | schedule_barrier(compiler_context *ctx) |
| 89 | { |
| 90 | midgard_block *temp = ctx->after_block; |
| 91 | ctx->after_block = create_empty_block(ctx); |
| 92 | ctx->block_count++; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 93 | list_addtail(&ctx->after_block->base.link, &ctx->blocks); |
| 94 | list_inithead(&ctx->after_block->base.instructions); |
| 95 | pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 96 | ctx->current_block = ctx->after_block; |
| 97 | ctx->after_block = temp; |
| 98 | } |
| 99 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 100 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 101 | * driver seems to do it that way */ |
| 102 | |
| 103 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 104 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 105 | #define M_LOAD_STORE(name, store, T) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 106 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 107 | midgard_instruction i = { \ |
| 108 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 109 | .mask = 0xF, \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 110 | .dest = ~0, \ |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 111 | .src = { ~0, ~0, ~0, ~0 }, \ |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 112 | .swizzle = SWIZZLE_IDENTITY_4, \ |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 113 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 114 | .load_store = { \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 115 | .address = address \ |
| 116 | } \ |
| 117 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 118 | \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 119 | if (store) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 120 | i.src[0] = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 121 | i.src_types[0] = T; \ |
Alyssa Rosenzweig | 9915bb2 | 2020-05-07 10:12:38 -0400 | [diff] [blame] | 122 | i.dest_type = T; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 123 | } else { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 124 | i.dest = ssa; \ |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 125 | i.dest_type = T; \ |
| 126 | } \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 127 | return i; \ |
| 128 | } |
| 129 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 130 | #define M_LOAD(name, T) M_LOAD_STORE(name, false, T) |
| 131 | #define M_STORE(name, T) M_LOAD_STORE(name, true, T) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 132 | |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 133 | M_LOAD(ld_attr_32, nir_type_uint32); |
| 134 | M_LOAD(ld_vary_32, nir_type_uint32); |
| 135 | M_LOAD(ld_ubo_int4, nir_type_uint32); |
| 136 | M_LOAD(ld_int4, nir_type_uint32); |
| 137 | M_STORE(st_int4, nir_type_uint32); |
| 138 | M_LOAD(ld_color_buffer_32u, nir_type_uint32); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 139 | M_LOAD(ld_color_buffer_as_fp16, nir_type_float16); |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 140 | M_LOAD(ld_color_buffer_as_fp32, nir_type_float32); |
Alyssa Rosenzweig | 714eba8 | 2020-04-27 19:01:40 -0400 | [diff] [blame] | 141 | M_STORE(st_vary_32, nir_type_uint32); |
| 142 | M_LOAD(ld_cubemap_coords, nir_type_uint32); |
| 143 | M_LOAD(ld_compute_id, nir_type_uint32); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 144 | |
| 145 | static midgard_instruction |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 146 | v_branch(bool conditional, bool invert) |
| 147 | { |
| 148 | midgard_instruction ins = { |
| 149 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 150 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 151 | .compact_branch = true, |
| 152 | .branch = { |
| 153 | .conditional = conditional, |
| 154 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 155 | }, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 156 | .dest = ~0, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 157 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 158 | }; |
| 159 | |
| 160 | return ins; |
| 161 | } |
| 162 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 163 | static void |
| 164 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 165 | { |
| 166 | ins->has_constants = true; |
| 167 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 171 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | { |
| 173 | return glsl_count_attribute_slots(type, false); |
| 174 | } |
| 175 | |
| 176 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 177 | static bool |
| 178 | midgard_nir_lower_fdot2_instr(nir_builder *b, nir_instr *instr, void *data) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 179 | { |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 180 | if (instr->type != nir_instr_type_alu) |
| 181 | return false; |
| 182 | |
| 183 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 184 | if (alu->op != nir_op_fdot2) |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 185 | return false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 186 | |
| 187 | b->cursor = nir_before_instr(&alu->instr); |
| 188 | |
| 189 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 190 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 191 | |
| 192 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 193 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 194 | nir_ssa_def *sum = nir_fadd(b, |
| 195 | nir_channel(b, product, 0), |
| 196 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 197 | |
| 198 | /* Replace the fdot2 with this sum */ |
| 199 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 200 | |
| 201 | return true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 202 | } |
| 203 | |
| 204 | static bool |
| 205 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 206 | { |
Icecream95 | 27516ba | 2020-09-05 17:00:37 +1200 | [diff] [blame] | 207 | return nir_shader_instructions_pass(shader, |
| 208 | midgard_nir_lower_fdot2_instr, |
| 209 | nir_metadata_block_index | nir_metadata_dominance, |
| 210 | NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 211 | } |
| 212 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 213 | static bool |
| 214 | mdg_is_64(const nir_instr *instr, const void *_unused) |
| 215 | { |
| 216 | const nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 217 | |
| 218 | if (nir_dest_bit_size(alu->dest.dest) == 64) |
| 219 | return true; |
| 220 | |
| 221 | switch (alu->op) { |
| 222 | case nir_op_umul_high: |
| 223 | case nir_op_imul_high: |
| 224 | return true; |
| 225 | default: |
| 226 | return false; |
| 227 | } |
| 228 | } |
| 229 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 230 | /* Flushes undefined values to zero */ |
| 231 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 232 | static void |
Alyssa Rosenzweig | 7c793a4 | 2020-05-22 16:23:06 -0400 | [diff] [blame] | 233 | optimise_nir(nir_shader *nir, unsigned quirks, bool is_blend) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 234 | { |
| 235 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 236 | unsigned lower_flrp = |
| 237 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 238 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 239 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 240 | |
| 241 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
Rhys Perry | 8b98d09 | 2019-02-05 15:56:24 +0000 | [diff] [blame] | 242 | NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 243 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 244 | nir_lower_tex_options lower_tex_options = { |
| 245 | .lower_txs_lod = true, |
Alyssa Rosenzweig | 4c43b35 | 2019-11-21 13:40:00 -0500 | [diff] [blame] | 246 | .lower_txp = ~0, |
| 247 | .lower_tex_without_implicit_lod = |
| 248 | (quirks & MIDGARD_EXPLICIT_LOD), |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 249 | .lower_tg4_broadcom_swizzle = true, |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 250 | |
| 251 | /* TODO: we have native gradient.. */ |
| 252 | .lower_txd = true, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 253 | }; |
| 254 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 255 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 256 | |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 257 | /* Must lower fdot2 after tex is lowered */ |
| 258 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
| 259 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 260 | /* T720 is broken. */ |
| 261 | |
| 262 | if (quirks & MIDGARD_BROKEN_LOD) |
| 263 | NIR_PASS_V(nir, midgard_nir_lod_errata); |
| 264 | |
Alyssa Rosenzweig | c495c6c | 2020-05-12 19:07:48 -0400 | [diff] [blame] | 265 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_early); |
| 266 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 267 | do { |
| 268 | progress = false; |
| 269 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 270 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 271 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 272 | |
| 273 | NIR_PASS(progress, nir, nir_copy_prop); |
Boris Brezillon | 440b0d6 | 2020-01-06 14:31:38 +0100 | [diff] [blame] | 274 | NIR_PASS(progress, nir, nir_opt_remove_phis); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 275 | NIR_PASS(progress, nir, nir_opt_dce); |
| 276 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 277 | NIR_PASS(progress, nir, nir_opt_cse); |
| 278 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 279 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 280 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 281 | |
| 282 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 283 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 284 | NIR_PASS(lower_flrp_progress, |
| 285 | nir, |
| 286 | nir_lower_flrp, |
| 287 | lower_flrp, |
Marek Olšák | ac55b1a | 2020-07-22 22:13:16 -0400 | [diff] [blame] | 288 | false /* always_precise */); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 289 | if (lower_flrp_progress) { |
| 290 | NIR_PASS(progress, nir, |
| 291 | nir_opt_constant_folding); |
| 292 | progress = true; |
| 293 | } |
| 294 | |
| 295 | /* Nothing should rematerialize any flrps, so we only |
| 296 | * need to do this lowering once. |
| 297 | */ |
| 298 | lower_flrp = 0; |
| 299 | } |
| 300 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 301 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 302 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 303 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 304 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 305 | nir_var_shader_in | |
| 306 | nir_var_shader_out | |
| 307 | nir_var_function_temp); |
| 308 | |
Eric Anholt | f25e169 | 2020-08-27 12:49:13 -0700 | [diff] [blame] | 309 | NIR_PASS(progress, nir, nir_opt_vectorize, NULL, NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 310 | } while (progress); |
| 311 | |
Alyssa Rosenzweig | 2486fe6 | 2020-08-27 14:55:11 -0400 | [diff] [blame] | 312 | NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_is_64, NULL); |
| 313 | |
Alyssa Rosenzweig | d838cb9 | 2020-06-16 13:07:02 -0400 | [diff] [blame] | 314 | /* Run after opts so it can hit more */ |
| 315 | if (!is_blend) |
| 316 | NIR_PASS(progress, nir, nir_fuse_io_16); |
| 317 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 318 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 319 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 320 | |
| 321 | do { |
| 322 | progress = false; |
| 323 | |
| 324 | NIR_PASS(progress, nir, nir_opt_dce); |
| 325 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 326 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 327 | NIR_PASS(progress, nir, nir_copy_prop); |
| 328 | } while (progress); |
| 329 | |
| 330 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 211dee4 | 2020-04-29 20:27:16 -0400 | [diff] [blame] | 331 | NIR_PASS(progress, nir, nir_opt_algebraic_distribute_src_mods); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 332 | |
| 333 | /* We implement booleans as 32-bit 0/~0 */ |
| 334 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 335 | |
| 336 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 337 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 338 | NIR_PASS(progress, nir, midgard_nir_cancel_inot); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 339 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 340 | NIR_PASS(progress, nir, nir_copy_prop); |
| 341 | NIR_PASS(progress, nir, nir_opt_dce); |
| 342 | |
| 343 | /* Take us out of SSA */ |
| 344 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 345 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 346 | |
| 347 | /* We are a vector architecture; write combine where possible */ |
| 348 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 349 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 350 | |
| 351 | NIR_PASS(progress, nir, nir_opt_dce); |
| 352 | } |
| 353 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 354 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 355 | |
| 356 | static void |
| 357 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 358 | { |
| 359 | nir_ssa_def def = instr->def; |
| 360 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 361 | midgard_constants *consts = rzalloc(NULL, midgard_constants); |
| 362 | |
| 363 | assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8); |
| 364 | |
| 365 | #define RAW_CONST_COPY(bits) \ |
| 366 | nir_const_value_to_array(consts->u##bits, instr->value, \ |
| 367 | instr->def.num_components, u##bits) |
| 368 | |
| 369 | switch (instr->def.bit_size) { |
| 370 | case 64: |
| 371 | RAW_CONST_COPY(64); |
| 372 | break; |
| 373 | case 32: |
| 374 | RAW_CONST_COPY(32); |
| 375 | break; |
| 376 | case 16: |
| 377 | RAW_CONST_COPY(16); |
| 378 | break; |
| 379 | case 8: |
| 380 | RAW_CONST_COPY(8); |
| 381 | break; |
| 382 | default: |
| 383 | unreachable("Invalid bit_size for load_const instruction\n"); |
| 384 | } |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 385 | |
| 386 | /* Shifted for SSA, +1 for off-by-one */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 387 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 388 | } |
| 389 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 390 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 391 | * explicitly emit a move with the constant source */ |
| 392 | |
| 393 | static void |
| 394 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 395 | { |
| 396 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 397 | |
| 398 | if (constant_value) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 399 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), to); |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 400 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 401 | emit_mir_instruction(ctx, ins); |
| 402 | } |
| 403 | } |
| 404 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 405 | static bool |
| 406 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 407 | { |
| 408 | unsigned comp = src->swizzle[0]; |
| 409 | |
| 410 | for (unsigned c = 1; c < nr_components; ++c) { |
| 411 | if (src->swizzle[c] != comp) |
| 412 | return true; |
| 413 | } |
| 414 | |
| 415 | return false; |
| 416 | } |
| 417 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 418 | #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \ |
| 419 | case nir_intrinsic_##nir: \ |
| 420 | emit_atomic(ctx, instr, is_shared, midgard_op_##op); \ |
| 421 | break; |
| 422 | |
| 423 | #define ATOMIC_CASE(ctx, instr, nir, op) \ |
| 424 | ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \ |
| 425 | ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false); |
| 426 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 427 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 428 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 429 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 430 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 431 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 432 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 433 | #define ALU_CASE_RTZ(nir, _op) \ |
| 434 | case nir_op_##nir: \ |
| 435 | op = midgard_alu_op_##_op; \ |
| 436 | roundmode = MIDGARD_RTZ; \ |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 437 | break; |
| 438 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 439 | #define ALU_CHECK_CMP() \ |
Alyssa Rosenzweig | 1108eaa | 2020-05-08 17:41:49 -0400 | [diff] [blame] | 440 | assert(src_bitsize == 16 || src_bitsize == 32); \ |
| 441 | assert(dst_bitsize == 16 || dst_bitsize == 32); \ |
| 442 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 443 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 444 | case nir_op_##nir: \ |
| 445 | op = midgard_alu_op_##_op; \ |
| 446 | broadcast_swizzle = count; \ |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 447 | ALU_CHECK_CMP(); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 448 | break; |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 449 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 450 | #define ALU_CASE_CMP(nir, _op) \ |
Alyssa Rosenzweig | eb28a36 | 2020-05-07 10:12:24 -0400 | [diff] [blame] | 451 | case nir_op_##nir: \ |
| 452 | op = midgard_alu_op_##_op; \ |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 453 | ALU_CHECK_CMP(); \ |
| 454 | break; |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 455 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 456 | /* Compare mir_lower_invert */ |
| 457 | static bool |
| 458 | nir_accepts_inot(nir_op op, unsigned src) |
| 459 | { |
| 460 | switch (op) { |
| 461 | case nir_op_ior: |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 462 | case nir_op_iand: /* TODO: b2f16 */ |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 463 | case nir_op_ixor: |
| 464 | return true; |
| 465 | case nir_op_b32csel: |
| 466 | /* Only the condition */ |
| 467 | return (src == 0); |
| 468 | default: |
| 469 | return false; |
| 470 | } |
| 471 | } |
| 472 | |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 473 | static bool |
| 474 | mir_accept_dest_mod(compiler_context *ctx, nir_dest **dest, nir_op op) |
| 475 | { |
| 476 | if (pan_has_dest_mod(dest, op)) { |
| 477 | assert((*dest)->is_ssa); |
| 478 | BITSET_SET(ctx->already_emitted, (*dest)->ssa.index); |
| 479 | return true; |
| 480 | } |
| 481 | |
| 482 | return false; |
| 483 | } |
| 484 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 485 | /* Look for floating point mods. We have the mods fsat, fsat_signed, |
| 486 | * and fpos. We also have the relations (note 3 * 2 = 6 cases): |
| 487 | * |
| 488 | * fsat_signed(fpos(x)) = fsat(x) |
| 489 | * fsat_signed(fsat(x)) = fsat(x) |
| 490 | * fpos(fsat_signed(x)) = fsat(x) |
| 491 | * fpos(fsat(x)) = fsat(x) |
| 492 | * fsat(fsat_signed(x)) = fsat(x) |
| 493 | * fsat(fpos(x)) = fsat(x) |
| 494 | * |
| 495 | * So by cases any composition of output modifiers is equivalent to |
| 496 | * fsat alone. |
| 497 | */ |
| 498 | static unsigned |
| 499 | mir_determine_float_outmod(compiler_context *ctx, nir_dest **dest, unsigned prior_outmod) |
| 500 | { |
| 501 | bool fpos = mir_accept_dest_mod(ctx, dest, nir_op_fclamp_pos); |
| 502 | bool fsat = mir_accept_dest_mod(ctx, dest, nir_op_fsat); |
| 503 | bool ssat = mir_accept_dest_mod(ctx, dest, nir_op_fsat_signed); |
| 504 | bool prior = (prior_outmod != midgard_outmod_none); |
| 505 | int count = (int) prior + (int) fpos + (int) ssat + (int) fsat; |
| 506 | |
| 507 | return ((count > 1) || fsat) ? midgard_outmod_sat : |
| 508 | fpos ? midgard_outmod_pos : |
| 509 | ssat ? midgard_outmod_sat_signed : |
| 510 | prior_outmod; |
| 511 | } |
| 512 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 513 | static void |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 514 | mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i, unsigned to, bool *abs, bool *neg, bool *not, enum midgard_roundmode *roundmode, bool is_int, unsigned bcast_count) |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 515 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 516 | nir_alu_src src = instr->src[i]; |
Alyssa Rosenzweig | b124f53 | 2020-04-29 18:10:43 -0400 | [diff] [blame] | 517 | |
| 518 | if (!is_int) { |
| 519 | if (pan_has_source_mod(&src, nir_op_fneg)) |
| 520 | *neg = !(*neg); |
| 521 | |
| 522 | if (pan_has_source_mod(&src, nir_op_fabs)) |
| 523 | *abs = true; |
| 524 | } |
| 525 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 526 | if (nir_accepts_inot(instr->op, i) && pan_has_source_mod(&src, nir_op_inot)) |
| 527 | *not = true; |
| 528 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 529 | if (roundmode) { |
| 530 | if (pan_has_source_mod(&src, nir_op_fround_even)) |
| 531 | *roundmode = MIDGARD_RTE; |
| 532 | |
| 533 | if (pan_has_source_mod(&src, nir_op_ftrunc)) |
| 534 | *roundmode = MIDGARD_RTZ; |
| 535 | |
| 536 | if (pan_has_source_mod(&src, nir_op_ffloor)) |
| 537 | *roundmode = MIDGARD_RTN; |
| 538 | |
| 539 | if (pan_has_source_mod(&src, nir_op_fceil)) |
| 540 | *roundmode = MIDGARD_RTP; |
| 541 | } |
| 542 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 543 | unsigned bits = nir_src_bit_size(src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 544 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 545 | ins->src[to] = nir_src_index(NULL, &src.src); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 546 | ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 547 | |
| 548 | for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 549 | ins->swizzle[to][c] = src.swizzle[ |
| 550 | (!bcast_count || c < bcast_count) ? c : |
| 551 | (bcast_count - 1)]; |
| 552 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 553 | } |
| 554 | |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 555 | /* Midgard features both fcsel and icsel, depending on whether you want int or |
| 556 | * float modifiers. NIR's csel is typeless, so we want a heuristic to guess if |
| 557 | * we should emit an int or float csel depending on what modifiers could be |
| 558 | * placed. In the absense of modifiers, this is probably arbitrary. */ |
| 559 | |
| 560 | static bool |
| 561 | mir_is_bcsel_float(nir_alu_instr *instr) |
| 562 | { |
| 563 | nir_op intmods[] = { |
| 564 | nir_op_i2i8, nir_op_i2i16, |
| 565 | nir_op_i2i32, nir_op_i2i64 |
| 566 | }; |
| 567 | |
| 568 | nir_op floatmods[] = { |
| 569 | nir_op_fabs, nir_op_fneg, |
| 570 | nir_op_f2f16, nir_op_f2f32, |
| 571 | nir_op_f2f64 |
| 572 | }; |
| 573 | |
| 574 | nir_op floatdestmods[] = { |
| 575 | nir_op_fsat, nir_op_fsat_signed, nir_op_fclamp_pos, |
| 576 | nir_op_f2f16, nir_op_f2f32 |
| 577 | }; |
| 578 | |
| 579 | signed score = 0; |
| 580 | |
| 581 | for (unsigned i = 1; i < 3; ++i) { |
| 582 | nir_alu_src s = instr->src[i]; |
| 583 | for (unsigned q = 0; q < ARRAY_SIZE(intmods); ++q) { |
| 584 | if (pan_has_source_mod(&s, intmods[q])) |
| 585 | score--; |
| 586 | } |
| 587 | } |
| 588 | |
| 589 | for (unsigned i = 1; i < 3; ++i) { |
| 590 | nir_alu_src s = instr->src[i]; |
| 591 | for (unsigned q = 0; q < ARRAY_SIZE(floatmods); ++q) { |
| 592 | if (pan_has_source_mod(&s, floatmods[q])) |
| 593 | score++; |
| 594 | } |
| 595 | } |
| 596 | |
| 597 | for (unsigned q = 0; q < ARRAY_SIZE(floatdestmods); ++q) { |
| 598 | nir_dest *dest = &instr->dest.dest; |
| 599 | if (pan_has_dest_mod(&dest, floatdestmods[q])) |
| 600 | score++; |
| 601 | } |
| 602 | |
| 603 | return (score > 0); |
| 604 | } |
| 605 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 606 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 607 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 608 | { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 609 | nir_dest *dest = &instr->dest.dest; |
| 610 | |
| 611 | if (dest->is_ssa && BITSET_TEST(ctx->already_emitted, dest->ssa.index)) |
| 612 | return; |
| 613 | |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 614 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 615 | * is handled elsewhere */ |
| 616 | |
| 617 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 618 | midgard_emit_derivatives(ctx, instr); |
| 619 | return; |
| 620 | } |
| 621 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 622 | bool is_ssa = dest->is_ssa; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 623 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 624 | unsigned nr_components = nir_dest_num_components(*dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 625 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | 04f76ad | 2020-04-27 18:58:21 -0400 | [diff] [blame] | 626 | unsigned op = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 627 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 628 | /* Number of components valid to check for the instruction (the rest |
| 629 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 630 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 631 | * in Midgard */ |
| 632 | |
| 633 | unsigned broadcast_swizzle = 0; |
| 634 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 635 | /* Should we swap arguments? */ |
| 636 | bool flip_src12 = false; |
| 637 | |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 638 | ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 639 | ASSERTED unsigned dst_bitsize = nir_dest_bit_size(*dest); |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 640 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 641 | enum midgard_roundmode roundmode = MIDGARD_RTE; |
| 642 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 643 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 644 | ALU_CASE(fadd, fadd); |
| 645 | ALU_CASE(fmul, fmul); |
| 646 | ALU_CASE(fmin, fmin); |
| 647 | ALU_CASE(fmax, fmax); |
| 648 | ALU_CASE(imin, imin); |
| 649 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 650 | ALU_CASE(umin, umin); |
| 651 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 652 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 653 | ALU_CASE(fround_even, froundeven); |
| 654 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 655 | ALU_CASE(fceil, fceil); |
| 656 | ALU_CASE(fdot3, fdot3); |
| 657 | ALU_CASE(fdot4, fdot4); |
| 658 | ALU_CASE(iadd, iadd); |
| 659 | ALU_CASE(isub, isub); |
| 660 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 661 | ALU_CASE(imul_high, imul); |
| 662 | ALU_CASE(umul_high, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 663 | |
| 664 | /* Zero shoved as second-arg */ |
| 665 | ALU_CASE(iabs, iabsdiff); |
| 666 | |
Italo Nicola | c9192d1 | 2020-09-19 10:36:08 +0000 | [diff] [blame] | 667 | ALU_CASE(uabs_isub, iabsdiff); |
| 668 | ALU_CASE(uabs_usub, uabsdiff); |
| 669 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 670 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 671 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 672 | ALU_CASE_CMP(feq32, feq); |
| 673 | ALU_CASE_CMP(fneu32, fne); |
| 674 | ALU_CASE_CMP(flt32, flt); |
| 675 | ALU_CASE_CMP(ieq32, ieq); |
| 676 | ALU_CASE_CMP(ine32, ine); |
| 677 | ALU_CASE_CMP(ilt32, ilt); |
| 678 | ALU_CASE_CMP(ult32, ult); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 679 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 680 | /* We don't have a native b2f32 instruction. Instead, like many |
| 681 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 682 | * correspondingly AND |
| 683 | * by 1.0 to do the type conversion. For the moment, prime us |
| 684 | * to emit: |
| 685 | * |
| 686 | * iand [whatever], #0 |
| 687 | * |
| 688 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 689 | */ |
| 690 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 691 | ALU_CASE_CMP(b2f32, iand); |
| 692 | ALU_CASE_CMP(b2f16, iand); |
| 693 | ALU_CASE_CMP(b2i32, iand); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 694 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 695 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 696 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 697 | |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 698 | ALU_CASE_CMP(f2b32, fne); |
| 699 | ALU_CASE_CMP(i2b32, ine); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 700 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 701 | ALU_CASE(frcp, frcp); |
| 702 | ALU_CASE(frsq, frsqrt); |
| 703 | ALU_CASE(fsqrt, fsqrt); |
| 704 | ALU_CASE(fexp2, fexp2); |
| 705 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 706 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 707 | ALU_CASE_RTZ(f2i64, f2i_rte); |
| 708 | ALU_CASE_RTZ(f2u64, f2u_rte); |
| 709 | ALU_CASE_RTZ(i2f64, i2f_rte); |
| 710 | ALU_CASE_RTZ(u2f64, u2f_rte); |
Boris Brezillon | fcceeaf | 2020-01-20 22:05:14 +0100 | [diff] [blame] | 711 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 712 | ALU_CASE_RTZ(f2i32, f2i_rte); |
| 713 | ALU_CASE_RTZ(f2u32, f2u_rte); |
| 714 | ALU_CASE_RTZ(i2f32, i2f_rte); |
| 715 | ALU_CASE_RTZ(u2f32, u2f_rte); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 716 | |
Alyssa Rosenzweig | 0ae0141 | 2020-05-25 14:46:40 -0400 | [diff] [blame] | 717 | ALU_CASE_RTZ(f2i8, f2i_rte); |
| 718 | ALU_CASE_RTZ(f2u8, f2u_rte); |
| 719 | |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 720 | ALU_CASE_RTZ(f2i16, f2i_rte); |
| 721 | ALU_CASE_RTZ(f2u16, f2u_rte); |
| 722 | ALU_CASE_RTZ(i2f16, i2f_rte); |
| 723 | ALU_CASE_RTZ(u2f16, u2f_rte); |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 724 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 725 | ALU_CASE(fsin, fsin); |
| 726 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 727 | |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 728 | /* We'll get 0 in the second arg, so: |
| 729 | * ~a = ~(a | 0) = nor(a, 0) */ |
| 730 | ALU_CASE(inot, inor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 731 | ALU_CASE(iand, iand); |
| 732 | ALU_CASE(ior, ior); |
| 733 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 734 | ALU_CASE(ishl, ishl); |
| 735 | ALU_CASE(ishr, iasr); |
| 736 | ALU_CASE(ushr, ilsr); |
| 737 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 738 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 739 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 740 | ALU_CASE_CMP(b32all_fequal4, fball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 741 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 742 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 743 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 744 | ALU_CASE_CMP(b32any_fnequal4, fbany_neq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 745 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 746 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 747 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 748 | ALU_CASE_CMP(b32all_iequal4, iball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 749 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 750 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 751 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 752 | ALU_CASE_CMP(b32any_inequal4, ibany_neq); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 753 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 754 | /* Source mods will be shoved in later */ |
| 755 | ALU_CASE(fabs, fmov); |
| 756 | ALU_CASE(fneg, fmov); |
| 757 | ALU_CASE(fsat, fmov); |
Alyssa Rosenzweig | 24e2e24 | 2020-05-04 16:12:41 -0400 | [diff] [blame] | 758 | ALU_CASE(fsat_signed, fmov); |
| 759 | ALU_CASE(fclamp_pos, fmov); |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 760 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 761 | /* For size conversion, we use a move. Ideally though we would squash |
| 762 | * these ops together; maybe that has to happen after in NIR as part of |
| 763 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 764 | * only / exactly one size. If stepping down, we use a dest override to |
| 765 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 766 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 767 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 768 | case nir_op_i2i8: |
| 769 | case nir_op_i2i16: |
| 770 | case nir_op_i2i32: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 771 | case nir_op_i2i64: |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 772 | case nir_op_u2u8: |
| 773 | case nir_op_u2u16: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 774 | case nir_op_u2u32: |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 775 | case nir_op_u2u64: |
| 776 | case nir_op_f2f16: |
Boris Brezillon | e1f9e8d | 2020-01-20 16:05:31 +0100 | [diff] [blame] | 777 | case nir_op_f2f32: |
| 778 | case nir_op_f2f64: { |
| 779 | if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 || |
| 780 | instr->op == nir_op_f2f64) |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 781 | op = midgard_alu_op_fmov; |
| 782 | else |
| 783 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 784 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 785 | break; |
| 786 | } |
| 787 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 788 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 789 | * arguments */ |
| 790 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 791 | case nir_op_fge: |
| 792 | case nir_op_fge32: |
| 793 | case nir_op_ige32: |
| 794 | case nir_op_uge32: { |
| 795 | op = |
| 796 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 797 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 798 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 799 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 800 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 801 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 802 | flip_src12 = true; |
Italo Nicola | cea032a | 2020-09-23 05:41:38 +0000 | [diff] [blame] | 803 | ALU_CHECK_CMP(); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 804 | break; |
| 805 | } |
| 806 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 807 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 808 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
Alyssa Rosenzweig | d39f95b | 2020-05-04 15:45:47 -0400 | [diff] [blame] | 809 | bool is_float = mir_is_bcsel_float(instr); |
| 810 | op = is_float ? |
| 811 | (mixed ? midgard_alu_op_fcsel_v : midgard_alu_op_fcsel) : |
| 812 | (mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 813 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 814 | break; |
| 815 | } |
| 816 | |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 817 | case nir_op_unpack_32_2x16: |
| 818 | case nir_op_unpack_32_4x8: |
| 819 | case nir_op_pack_32_2x16: |
| 820 | case nir_op_pack_32_4x8: { |
| 821 | op = midgard_alu_op_imov; |
| 822 | break; |
| 823 | } |
| 824 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 825 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 826 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 827 | assert(0); |
| 828 | return; |
| 829 | } |
| 830 | |
Alyssa Rosenzweig | 72c1e3a | 2020-05-21 12:31:40 -0400 | [diff] [blame] | 831 | /* Promote imov to fmov if it might help inline a constant */ |
| 832 | if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src) |
| 833 | && nir_src_bit_size(instr->src[0].src) == 32 |
| 834 | && nir_is_same_comp_swizzle(instr->src[0].swizzle, |
| 835 | nir_src_num_components(instr->src[0].src))) { |
| 836 | op = midgard_alu_op_fmov; |
| 837 | } |
| 838 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 839 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 840 | |
| 841 | unsigned outmod = 0; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 842 | bool is_int = midgard_is_integer_op(op); |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 843 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 844 | if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) { |
| 845 | outmod = midgard_outmod_int_high; |
| 846 | } else if (midgard_is_integer_out_op(op)) { |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 847 | outmod = midgard_outmod_int_wrap; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 848 | } else if (instr->op == nir_op_fsat) { |
| 849 | outmod = midgard_outmod_sat; |
| 850 | } else if (instr->op == nir_op_fsat_signed) { |
| 851 | outmod = midgard_outmod_sat_signed; |
| 852 | } else if (instr->op == nir_op_fclamp_pos) { |
| 853 | outmod = midgard_outmod_pos; |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 854 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 855 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 856 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 857 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 858 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 859 | |
Italo Nicola | 2096903 | 2020-07-13 16:19:08 +0000 | [diff] [blame] | 860 | if (!midgard_is_integer_out_op(op)) { |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 861 | outmod = mir_determine_float_outmod(ctx, &dest, outmod); |
Alyssa Rosenzweig | 29afa88 | 2020-05-04 17:33:52 -0400 | [diff] [blame] | 862 | } |
| 863 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 864 | midgard_instruction ins = { |
| 865 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 866 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ecf9466 | 2020-04-27 18:57:34 -0400 | [diff] [blame] | 867 | .dest_type = nir_op_infos[instr->op].output_type |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 868 | | nir_dest_bit_size(*dest), |
Alyssa Rosenzweig | 93513cd | 2020-05-25 14:19:11 -0400 | [diff] [blame] | 869 | .roundmode = roundmode, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 870 | }; |
| 871 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 872 | enum midgard_roundmode *roundptr = (opcode_props & MIDGARD_ROUNDS) ? |
| 873 | &ins.roundmode : NULL; |
| 874 | |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 875 | for (unsigned i = nr_inputs; i < ARRAY_SIZE(ins.src); ++i) |
| 876 | ins.src[i] = ~0; |
| 877 | |
| 878 | if (quirk_flipped_r24) { |
| 879 | ins.src[0] = ~0; |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 880 | mir_copy_src(&ins, instr, 0, 1, &ins.src_abs[1], &ins.src_neg[1], &ins.src_invert[1], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 881 | } else { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 882 | for (unsigned i = 0; i < nr_inputs; ++i) { |
| 883 | unsigned to = i; |
| 884 | |
| 885 | if (instr->op == nir_op_b32csel) { |
| 886 | /* The condition is the first argument; move |
| 887 | * the other arguments up one to be a binary |
| 888 | * instruction for Midgard with the condition |
| 889 | * last */ |
| 890 | |
| 891 | if (i == 0) |
| 892 | to = 2; |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 893 | else if (flip_src12) |
| 894 | to = 2 - i; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 895 | else |
| 896 | to = i - 1; |
| 897 | } else if (flip_src12) { |
| 898 | to = 1 - to; |
| 899 | } |
| 900 | |
Alyssa Rosenzweig | f8b881f | 2020-05-25 14:19:24 -0400 | [diff] [blame] | 901 | mir_copy_src(&ins, instr, i, to, &ins.src_abs[to], &ins.src_neg[to], &ins.src_invert[to], roundptr, is_int, broadcast_swizzle); |
Alyssa Rosenzweig | 449e5de | 2020-04-30 13:46:35 -0400 | [diff] [blame] | 902 | |
| 903 | /* (!c) ? a : b = c ? b : a */ |
| 904 | if (instr->op == nir_op_b32csel && ins.src_invert[2]) { |
| 905 | ins.src_invert[2] = false; |
| 906 | flip_src12 ^= true; |
| 907 | } |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 908 | } |
Alyssa Rosenzweig | 6757c48 | 2020-04-27 18:55:11 -0400 | [diff] [blame] | 909 | } |
| 910 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 911 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 912 | /* Lowered to move */ |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 913 | if (instr->op == nir_op_fneg) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 914 | ins.src_neg[1] ^= true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 915 | |
| 916 | if (instr->op == nir_op_fabs) |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 917 | ins.src_abs[1] = true; |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 918 | } |
| 919 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 920 | ins.mask = mask_of(nr_components); |
| 921 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 922 | /* Apply writemask if non-SSA, keeping in mind that we can't write to |
| 923 | * components that don't exist. Note modifier => SSA => !reg => no |
| 924 | * writemask, so we don't have to worry about writemasks here.*/ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 925 | |
| 926 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 927 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 928 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 929 | ins.op = op; |
Italo Nicola | 5011373 | 2020-07-15 18:43:18 +0000 | [diff] [blame] | 930 | ins.outmod = outmod; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 931 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 932 | /* Late fixup for emulated instructions */ |
| 933 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 934 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 935 | /* Presently, our second argument is an inline #0 constant. |
| 936 | * Switch over to an embedded 1.0 constant (that can't fit |
| 937 | * inline, since we're 32-bit, not 16-bit like the inline |
| 938 | * constants) */ |
| 939 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 940 | ins.has_inline_constant = false; |
| 941 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 942 | ins.src_types[1] = nir_type_float32; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 943 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 944 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 945 | if (instr->op == nir_op_b2f32) |
| 946 | ins.constants.f32[0] = 1.0f; |
| 947 | else |
| 948 | ins.constants.i32[0] = 1; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 949 | |
| 950 | for (unsigned c = 0; c < 16; ++c) |
| 951 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 6b023b3 | 2020-05-08 17:42:40 -0400 | [diff] [blame] | 952 | } else if (instr->op == nir_op_b2f16) { |
| 953 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
| 954 | ins.src_types[1] = nir_type_float16; |
| 955 | ins.has_constants = true; |
| 956 | ins.constants.i16[0] = _mesa_float_to_half(1.0); |
| 957 | |
| 958 | for (unsigned c = 0; c < 16; ++c) |
| 959 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 960 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 961 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 962 | ins.has_inline_constant = false; |
| 963 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Italo Nicola | b1b0ce0 | 2020-07-10 14:51:52 +0000 | [diff] [blame] | 964 | ins.src_types[1] = ins.src_types[0]; |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 965 | ins.has_constants = true; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 966 | ins.constants.u32[0] = 0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 967 | |
| 968 | for (unsigned c = 0; c < 16; ++c) |
| 969 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 970 | } else if (instr->op == nir_op_pack_32_2x16) { |
| 971 | ins.dest_type = nir_type_uint16; |
| 972 | ins.mask = mask_of(nr_components * 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 973 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 974 | } else if (instr->op == nir_op_pack_32_4x8) { |
| 975 | ins.dest_type = nir_type_uint8; |
| 976 | ins.mask = mask_of(nr_components * 4); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 977 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 978 | } else if (instr->op == nir_op_unpack_32_2x16) { |
| 979 | ins.dest_type = nir_type_uint32; |
| 980 | ins.mask = mask_of(nr_components >> 1); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 981 | ins.is_pack = true; |
Alyssa Rosenzweig | 551d990 | 2020-05-13 16:17:46 -0400 | [diff] [blame] | 982 | } else if (instr->op == nir_op_unpack_32_4x8) { |
| 983 | ins.dest_type = nir_type_uint32; |
| 984 | ins.mask = mask_of(nr_components >> 2); |
Alyssa Rosenzweig | e9c780b | 2020-05-13 18:41:52 -0400 | [diff] [blame] | 985 | ins.is_pack = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 986 | } |
| 987 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 988 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 989 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 990 | * instructions can only operate as if they were scalars. Lower |
| 991 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 992 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 993 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 994 | |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 995 | unsigned swizzle_back[MIR_VEC_COMPONENTS]; |
| 996 | memcpy(&swizzle_back, ins.swizzle[0], sizeof(swizzle_back)); |
| 997 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 998 | midgard_instruction ins_split[MIR_VEC_COMPONENTS]; |
| 999 | unsigned ins_count = 0; |
| 1000 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1001 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1002 | /* Mask the associated component, dropping the |
| 1003 | * instruction if needed */ |
| 1004 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1005 | ins.mask = 1 << i; |
| 1006 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1007 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1008 | for (unsigned j = 0; j < ins_count; ++j) { |
| 1009 | if (swizzle_back[i] == ins_split[j].swizzle[0][0]) { |
| 1010 | ins_split[j].mask |= ins.mask; |
| 1011 | ins.mask = 0; |
| 1012 | break; |
| 1013 | } |
| 1014 | } |
| 1015 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1016 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1017 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1018 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1019 | for (unsigned j = 0; j < MIR_VEC_COMPONENTS; ++j) |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 1020 | ins.swizzle[0][j] = swizzle_back[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1021 | |
Icecream95 | a6f0d7f | 2020-05-24 00:23:25 +1200 | [diff] [blame] | 1022 | ins_split[ins_count] = ins; |
| 1023 | |
| 1024 | ++ins_count; |
| 1025 | } |
| 1026 | |
| 1027 | for (unsigned i = 0; i < ins_count; ++i) { |
| 1028 | emit_mir_instruction(ctx, ins_split[i]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1029 | } |
| 1030 | } else { |
| 1031 | emit_mir_instruction(ctx, ins); |
| 1032 | } |
| 1033 | } |
| 1034 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1035 | #undef ALU_CASE |
| 1036 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1037 | static void |
| 1038 | mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read) |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1039 | { |
| 1040 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1041 | unsigned nir_mask = 0; |
| 1042 | unsigned dsize = 0; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1043 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1044 | if (is_read) { |
| 1045 | nir_mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1046 | dsize = nir_dest_bit_size(intr->dest); |
| 1047 | } else { |
| 1048 | nir_mask = nir_intrinsic_write_mask(intr); |
| 1049 | dsize = 32; |
| 1050 | } |
| 1051 | |
| 1052 | /* Once we have the NIR mask, we need to normalize to work in 32-bit space */ |
Alyssa Rosenzweig | 9b8cb9f | 2020-03-09 20:19:29 -0400 | [diff] [blame] | 1053 | unsigned bytemask = pan_to_bytemask(dsize, nir_mask); |
Alyssa Rosenzweig | b91d715 | 2020-05-11 15:06:53 -0400 | [diff] [blame] | 1054 | ins->dest_type = nir_type_uint | dsize; |
Italo Nicola | 1101261 | 2020-08-26 14:56:13 +0000 | [diff] [blame] | 1055 | mir_set_bytemask(ins, bytemask); |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1056 | } |
| 1057 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1058 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1059 | * optimized) versions of UBO #0 */ |
| 1060 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1061 | static midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1062 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1063 | compiler_context *ctx, |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1064 | nir_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1065 | unsigned dest, |
| 1066 | unsigned offset, |
| 1067 | nir_src *indirect_offset, |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1068 | unsigned indirect_shift, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1069 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1070 | { |
| 1071 | /* TODO: half-floats */ |
| 1072 | |
Alyssa Rosenzweig | bc9a7d0 | 2019-11-15 14:19:34 -0500 | [diff] [blame] | 1073 | midgard_instruction ins = m_ld_ubo_int4(dest, 0); |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1074 | ins.constants.u32[0] = offset; |
Alyssa Rosenzweig | da73651 | 2019-12-19 11:12:25 -0500 | [diff] [blame] | 1075 | |
| 1076 | if (instr->type == nir_instr_type_intrinsic) |
| 1077 | mir_set_intr_mask(instr, &ins, true); |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1078 | |
| 1079 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1080 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1081 | ins.src_types[2] = nir_type_uint32; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1082 | ins.load_store.arg_2 = (indirect_shift << 5); |
Alyssa Rosenzweig | 797fa87 | 2020-07-06 10:57:04 -0400 | [diff] [blame] | 1083 | |
| 1084 | /* X component for the whole swizzle to prevent register |
| 1085 | * pressure from ballooning from the extra components */ |
| 1086 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[2]); ++i) |
| 1087 | ins.swizzle[2][i] = 0; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1088 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1089 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1090 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1091 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1092 | ins.load_store.arg_1 = index; |
| 1093 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1094 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1095 | } |
| 1096 | |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1097 | /* Globals are like UBOs if you squint. And shared memory is like globals if |
| 1098 | * you squint even harder */ |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1099 | |
| 1100 | static void |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1101 | emit_global( |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1102 | compiler_context *ctx, |
| 1103 | nir_instr *instr, |
| 1104 | bool is_read, |
| 1105 | unsigned srcdest, |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1106 | nir_src *offset, |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1107 | bool is_shared) |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1108 | { |
| 1109 | /* TODO: types */ |
| 1110 | |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 1111 | midgard_instruction ins; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1112 | |
| 1113 | if (is_read) |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1114 | ins = m_ld_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1115 | else |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1116 | ins = m_st_int4(srcdest, 0); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1117 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1118 | mir_set_offset(ctx, &ins, offset, is_shared); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1119 | mir_set_intr_mask(instr, &ins, is_read); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1120 | |
Alyssa Rosenzweig | 41184f8 | 2020-08-27 15:13:19 -0400 | [diff] [blame] | 1121 | /* Set a valid swizzle for masked out components */ |
| 1122 | assert(ins.mask); |
| 1123 | unsigned first_component = __builtin_ffs(ins.mask) - 1; |
| 1124 | |
| 1125 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) { |
| 1126 | if (!(ins.mask & (1 << i))) |
| 1127 | ins.swizzle[0][i] = first_component; |
| 1128 | } |
| 1129 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1130 | emit_mir_instruction(ctx, ins); |
| 1131 | } |
| 1132 | |
Italo Nicola | 8e221f5 | 2020-08-31 11:17:48 +0000 | [diff] [blame] | 1133 | /* If is_shared is off, the only other possible value are globals, since |
| 1134 | * SSBO's are being lowered to globals through a NIR pass. */ |
| 1135 | static void |
| 1136 | emit_atomic( |
| 1137 | compiler_context *ctx, |
| 1138 | nir_intrinsic_instr *instr, |
| 1139 | bool is_shared, |
| 1140 | midgard_load_store_op op) |
| 1141 | { |
| 1142 | unsigned bitsize = nir_src_bit_size(instr->src[1]); |
| 1143 | nir_alu_type type = |
| 1144 | (op == midgard_op_atomic_imin || op == midgard_op_atomic_imax) ? |
| 1145 | nir_type_int : nir_type_uint; |
| 1146 | |
| 1147 | unsigned dest = nir_dest_index(&instr->dest); |
| 1148 | unsigned val = nir_src_index(ctx, &instr->src[1]); |
| 1149 | emit_explicit_constant(ctx, val, val); |
| 1150 | |
| 1151 | midgard_instruction ins = { |
| 1152 | .type = TAG_LOAD_STORE_4, |
| 1153 | .mask = 0xF, |
| 1154 | .dest = dest, |
| 1155 | .src = { ~0, ~0, ~0, val }, |
| 1156 | .src_types = { 0, 0, 0, type | bitsize }, |
| 1157 | .op = op |
| 1158 | }; |
| 1159 | |
| 1160 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1161 | |
| 1162 | /* cmpxchg takes an extra value in arg_2, so we don't use it for the offset */ |
| 1163 | if (op == midgard_op_atomic_cmpxchg) { |
| 1164 | unsigned addr = nir_src_index(ctx, src_offset); |
| 1165 | |
| 1166 | ins.src[1] = addr; |
| 1167 | ins.src_types[1] = nir_type_uint | nir_src_bit_size(*src_offset); |
| 1168 | |
| 1169 | unsigned xchg_val = nir_src_index(ctx, &instr->src[2]); |
| 1170 | emit_explicit_constant(ctx, xchg_val, xchg_val); |
| 1171 | |
| 1172 | ins.src[2] = val; |
| 1173 | ins.src_types[2] = type | bitsize; |
| 1174 | ins.src[3] = xchg_val; |
| 1175 | |
| 1176 | if (is_shared) |
| 1177 | ins.load_store.arg_1 |= 0x6E; |
| 1178 | } else { |
| 1179 | mir_set_offset(ctx, &ins, src_offset, is_shared); |
| 1180 | } |
| 1181 | |
| 1182 | mir_set_intr_mask(&instr->instr, &ins, true); |
| 1183 | |
| 1184 | emit_mir_instruction(ctx, ins); |
| 1185 | } |
| 1186 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1187 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1188 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1189 | compiler_context *ctx, |
| 1190 | unsigned dest, unsigned offset, |
| 1191 | unsigned nr_comp, unsigned component, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1192 | nir_src *indirect_offset, nir_alu_type type, bool flat) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1193 | { |
| 1194 | /* XXX: Half-floats? */ |
| 1195 | /* TODO: swizzle, mask */ |
| 1196 | |
| 1197 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1198 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 2d16883 | 2020-06-04 11:32:59 -0400 | [diff] [blame] | 1199 | ins.dest_type = type; |
| 1200 | |
| 1201 | if (type == nir_type_float16) { |
| 1202 | /* Ensure we are aligned so we can pack it later */ |
| 1203 | ins.mask = mask_of(ALIGN_POT(nr_comp, 2)); |
| 1204 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1205 | |
| 1206 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) |
| 1207 | ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1208 | |
| 1209 | midgard_varying_parameter p = { |
| 1210 | .is_varying = 1, |
| 1211 | .interpolation = midgard_interp_default, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1212 | .flat = flat, |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1213 | }; |
| 1214 | |
| 1215 | unsigned u; |
| 1216 | memcpy(&u, &p, sizeof(p)); |
| 1217 | ins.load_store.varying_parameters = u; |
| 1218 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1219 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1220 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1221 | ins.src_types[2] = nir_type_uint32; |
| 1222 | } else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1223 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1224 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1225 | ins.load_store.arg_1 = 0x9E; |
| 1226 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1227 | /* Use the type appropriate load */ |
| 1228 | switch (type) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1229 | case nir_type_uint32: |
| 1230 | case nir_type_bool32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1231 | ins.op = midgard_op_ld_vary_32u; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1232 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1233 | case nir_type_int32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1234 | ins.op = midgard_op_ld_vary_32i; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1235 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1236 | case nir_type_float32: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1237 | ins.op = midgard_op_ld_vary_32; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1238 | break; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1239 | case nir_type_float16: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1240 | ins.op = midgard_op_ld_vary_16; |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1241 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1242 | default: |
| 1243 | unreachable("Attempted to load unknown type"); |
| 1244 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1245 | } |
| 1246 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1247 | emit_mir_instruction(ctx, ins); |
| 1248 | } |
| 1249 | |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1250 | static void |
| 1251 | emit_attr_read( |
| 1252 | compiler_context *ctx, |
| 1253 | unsigned dest, unsigned offset, |
| 1254 | unsigned nr_comp, nir_alu_type t) |
| 1255 | { |
| 1256 | midgard_instruction ins = m_ld_attr_32(dest, offset); |
| 1257 | ins.load_store.arg_1 = 0x1E; |
| 1258 | ins.load_store.arg_2 = 0x1E; |
| 1259 | ins.mask = mask_of(nr_comp); |
| 1260 | |
| 1261 | /* Use the type appropriate load */ |
| 1262 | switch (t) { |
| 1263 | case nir_type_uint: |
| 1264 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1265 | ins.op = midgard_op_ld_attr_32u; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1266 | break; |
| 1267 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1268 | ins.op = midgard_op_ld_attr_32i; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1269 | break; |
| 1270 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1271 | ins.op = midgard_op_ld_attr_32; |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1272 | break; |
| 1273 | default: |
| 1274 | unreachable("Attempted to load unknown type"); |
| 1275 | break; |
| 1276 | } |
| 1277 | |
| 1278 | emit_mir_instruction(ctx, ins); |
| 1279 | } |
| 1280 | |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1281 | static void |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1282 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1283 | unsigned nr_components, unsigned offset) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1284 | { |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1285 | nir_dest nir_dest; |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1286 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1287 | /* Figure out which uniform this is */ |
Alyssa Rosenzweig | e610267 | 2020-03-10 16:06:30 -0400 | [diff] [blame] | 1288 | int sysval = panfrost_sysval_for_instr(instr, &nir_dest); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1289 | void *val = _mesa_hash_table_u64_search(ctx->sysvals.sysval_to_id, sysval); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1290 | |
Alyssa Rosenzweig | 674b24d | 2020-03-10 15:54:17 -0400 | [diff] [blame] | 1291 | unsigned dest = nir_dest_index(&nir_dest); |
| 1292 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1293 | /* Sysvals are prefix uniforms */ |
| 1294 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1295 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1296 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1297 | midgard_instruction *ins = |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1298 | emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0, 0); |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1299 | |
| 1300 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1301 | } |
| 1302 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1303 | static unsigned |
| 1304 | compute_builtin_arg(nir_op op) |
| 1305 | { |
| 1306 | switch (op) { |
| 1307 | case nir_intrinsic_load_work_group_id: |
| 1308 | return 0x14; |
| 1309 | case nir_intrinsic_load_local_invocation_id: |
| 1310 | return 0x10; |
| 1311 | default: |
| 1312 | unreachable("Invalid compute paramater loaded"); |
| 1313 | } |
| 1314 | } |
| 1315 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1316 | static void |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1317 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z, unsigned src_s, enum midgard_rt_id rt) |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1318 | { |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1319 | assert(rt < ARRAY_SIZE(ctx->writeout_branch)); |
| 1320 | |
| 1321 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 1322 | |
| 1323 | assert(!br); |
| 1324 | |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 1325 | emit_explicit_constant(ctx, src, src); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1326 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1327 | struct midgard_instruction ins = |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1328 | v_branch(false, false); |
| 1329 | |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1330 | bool depth_only = (rt == MIDGARD_ZS_RT); |
| 1331 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1332 | ins.writeout = depth_only ? 0 : PAN_WRITEOUT_C; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1333 | |
| 1334 | /* Add dependencies */ |
Alyssa Rosenzweig | 7652983 | 2019-08-30 11:01:15 -0700 | [diff] [blame] | 1335 | ins.src[0] = src; |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1336 | ins.src_types[0] = nir_type_uint32; |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 1337 | ins.constants.u32[0] = depth_only ? 0xFF : (rt - MIDGARD_COLOR_RT0) * 0x100; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 1338 | for (int i = 0; i < 4; ++i) |
| 1339 | ins.swizzle[0][i] = i; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1340 | |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1341 | if (~src_z) { |
| 1342 | emit_explicit_constant(ctx, src_z, src_z); |
| 1343 | ins.src[2] = src_z; |
| 1344 | ins.src_types[2] = nir_type_uint32; |
| 1345 | ins.writeout |= PAN_WRITEOUT_Z; |
| 1346 | } |
| 1347 | if (~src_s) { |
| 1348 | emit_explicit_constant(ctx, src_s, src_s); |
| 1349 | ins.src[3] = src_s; |
| 1350 | ins.src_types[3] = nir_type_uint32; |
| 1351 | ins.writeout |= PAN_WRITEOUT_S; |
| 1352 | } |
| 1353 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1354 | /* Emit the branch */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 1355 | br = emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 281cc6f | 2019-11-23 12:43:55 -0500 | [diff] [blame] | 1356 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 1357 | ctx->writeout_branch[rt] = br; |
| 1358 | |
| 1359 | /* Push our current location = current block count - 1 = where we'll |
| 1360 | * jump to. Maybe a bit too clever for my own good */ |
| 1361 | |
| 1362 | br->branch.target_block = ctx->block_count - 1; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1363 | } |
| 1364 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1365 | static void |
| 1366 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1367 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1368 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1369 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1370 | ins.mask = mask_of(3); |
Alyssa Rosenzweig | d3747fb | 2020-02-12 08:39:29 -0500 | [diff] [blame] | 1371 | ins.swizzle[0][3] = COMPONENT_X; /* xyzx */ |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1372 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1373 | emit_mir_instruction(ctx, ins); |
| 1374 | } |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1375 | |
| 1376 | static unsigned |
| 1377 | vertex_builtin_arg(nir_op op) |
| 1378 | { |
| 1379 | switch (op) { |
| 1380 | case nir_intrinsic_load_vertex_id: |
| 1381 | return PAN_VERTEX_ID; |
| 1382 | case nir_intrinsic_load_instance_id: |
| 1383 | return PAN_INSTANCE_ID; |
| 1384 | default: |
| 1385 | unreachable("Invalid vertex builtin"); |
| 1386 | } |
| 1387 | } |
| 1388 | |
| 1389 | static void |
| 1390 | emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1391 | { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1392 | unsigned reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1393 | emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int); |
| 1394 | } |
| 1395 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1396 | static void |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1397 | emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx) |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1398 | { |
| 1399 | unsigned reg = nir_dest_index(&instr->dest); |
| 1400 | |
| 1401 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1402 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1403 | ld.load_store.address = idx; |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1404 | ld.load_store.arg_2 = 0x1E; |
| 1405 | |
| 1406 | for (int i = 0; i < 4; ++i) |
| 1407 | ld.swizzle[0][i] = COMPONENT_X; |
| 1408 | |
| 1409 | emit_mir_instruction(ctx, ld); |
| 1410 | } |
| 1411 | |
| 1412 | static void |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1413 | emit_control_barrier(compiler_context *ctx) |
| 1414 | { |
| 1415 | midgard_instruction ins = { |
| 1416 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | fde1f2b | 2020-05-13 11:05:34 -0400 | [diff] [blame] | 1417 | .dest = ~0, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1418 | .src = { ~0, ~0, ~0, ~0 }, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 1419 | .op = TEXTURE_OP_BARRIER, |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1420 | }; |
| 1421 | |
| 1422 | emit_mir_instruction(ctx, ins); |
| 1423 | } |
| 1424 | |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1425 | static unsigned |
| 1426 | mir_get_branch_cond(nir_src *src, bool *invert) |
| 1427 | { |
| 1428 | /* Wrap it. No swizzle since it's a scalar */ |
| 1429 | |
| 1430 | nir_alu_src alu = { |
| 1431 | .src = *src |
| 1432 | }; |
| 1433 | |
| 1434 | *invert = pan_has_source_mod(&alu, nir_op_inot); |
| 1435 | return nir_src_index(NULL, &alu.src); |
| 1436 | } |
| 1437 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1438 | static uint8_t |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1439 | output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr) |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1440 | { |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1441 | if (ctx->is_blend) |
| 1442 | return ctx->blend_rt; |
| 1443 | |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1444 | const nir_variable *var; |
Alyssa Rosenzweig | dfaa4c5 | 2020-11-04 08:32:16 -0500 | [diff] [blame] | 1445 | var = nir_find_variable_with_driver_location(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr)); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1446 | assert(var); |
| 1447 | |
| 1448 | unsigned loc = var->data.location; |
| 1449 | |
| 1450 | if (loc == FRAG_RESULT_COLOR) |
| 1451 | loc = FRAG_RESULT_DATA0; |
| 1452 | |
| 1453 | if (loc >= FRAG_RESULT_DATA0) |
| 1454 | return loc - FRAG_RESULT_DATA0; |
| 1455 | |
| 1456 | if (loc == FRAG_RESULT_DEPTH) |
| 1457 | return 0x1F; |
| 1458 | if (loc == FRAG_RESULT_STENCIL) |
| 1459 | return 0x1E; |
| 1460 | |
Icecream95 | 6493d29 | 2020-07-14 15:06:09 +1200 | [diff] [blame] | 1461 | unreachable("Invalid RT to load from"); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1462 | } |
| 1463 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1464 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1465 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1466 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1467 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1468 | |
| 1469 | switch (instr->intrinsic) { |
| 1470 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1471 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1472 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1473 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1474 | discard.branch.target_type = TARGET_DISCARD; |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1475 | |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1476 | if (conditional) { |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 1477 | discard.src[0] = mir_get_branch_cond(&instr->src[0], |
| 1478 | &discard.branch.invert_conditional); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 1479 | discard.src_types[0] = nir_type_uint32; |
| 1480 | } |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1481 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1482 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1483 | schedule_barrier(ctx); |
| 1484 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1485 | break; |
| 1486 | } |
| 1487 | |
| 1488 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1489 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1490 | case nir_intrinsic_load_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1491 | case nir_intrinsic_load_shared: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1492 | case nir_intrinsic_load_input: |
| 1493 | case nir_intrinsic_load_interpolated_input: { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1494 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1495 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1496 | bool is_global = instr->intrinsic == nir_intrinsic_load_global; |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1497 | bool is_shared = instr->intrinsic == nir_intrinsic_load_shared; |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1498 | bool is_flat = instr->intrinsic == nir_intrinsic_load_input; |
| 1499 | bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1500 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1501 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1502 | /* TODO: Infer type? Does it matter? */ |
| 1503 | nir_alu_type t = |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1504 | (is_ubo || is_global || is_shared) ? nir_type_uint : |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1505 | (is_interp) ? nir_type_float : |
Jason Ekstrand | 0aa08ae | 2020-09-30 21:20:53 -0500 | [diff] [blame] | 1506 | nir_intrinsic_dest_type(instr); |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1507 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1508 | t = nir_alu_type_get_base_type(t); |
| 1509 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1510 | if (!(is_ubo || is_global)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1511 | offset = nir_intrinsic_base(instr); |
| 1512 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1513 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1514 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1515 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1516 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1517 | |
| 1518 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1519 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1520 | |
| 1521 | if (direct) |
| 1522 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1523 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1524 | /* We may need to apply a fractional offset */ |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1525 | int component = (is_flat || is_interp) ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1526 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1527 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1528 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1529 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 1530 | emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysvals.sysval_count + offset) * 16, indirect_offset, 4, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1531 | } else if (is_ubo) { |
| 1532 | nir_src index = instr->src[0]; |
| 1533 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1534 | /* TODO: Is indirect block number possible? */ |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1535 | assert(nir_src_is_const(index)); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1536 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1537 | uint32_t uindex = nir_src_as_uint(index) + 1; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1538 | emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1539 | } else if (is_global || is_shared) { |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1540 | emit_global(ctx, &instr->instr, true, reg, src_offset, is_shared); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1541 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | 5f8dd41 | 2020-05-22 16:22:48 -0400 | [diff] [blame] | 1542 | emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1543 | } else if (ctx->is_blend) { |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1544 | /* ctx->blend_input will be precoloured to r0/r2, where |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1545 | * the input is preloaded */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1546 | |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1547 | unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input; |
| 1548 | |
| 1549 | if (*input == ~0) |
| 1550 | *input = reg; |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 1551 | else |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1552 | emit_mir_instruction(ctx, v_mov(*input, reg)); |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1553 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
| 1554 | emit_attr_read(ctx, reg, offset, nr_comp, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1555 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1556 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1557 | assert(0); |
| 1558 | } |
| 1559 | |
| 1560 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1561 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1562 | |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1563 | /* Artefact of load_interpolated_input. TODO: other barycentric modes */ |
| 1564 | case nir_intrinsic_load_barycentric_pixel: |
Tomeu Vizoso | 2504206 | 2020-01-03 09:42:11 +0100 | [diff] [blame] | 1565 | case nir_intrinsic_load_barycentric_centroid: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1566 | break; |
| 1567 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1568 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1569 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1570 | case nir_intrinsic_load_raw_output_pan: { |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1571 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1572 | |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1573 | /* T720 and below use different blend opcodes with slightly |
| 1574 | * different semantics than T760 and up */ |
| 1575 | |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 1576 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1577 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1578 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1579 | |
Icecream95 | c20d166 | 2020-07-16 14:16:11 +1200 | [diff] [blame] | 1580 | if (nir_src_is_const(instr->src[0])) { |
| 1581 | ld.load_store.arg_1 = nir_src_as_uint(instr->src[0]); |
| 1582 | } else { |
| 1583 | ld.load_store.varying_parameters = 2; |
| 1584 | ld.src[1] = nir_src_index(ctx, &instr->src[0]); |
| 1585 | ld.src_types[1] = nir_type_int32; |
| 1586 | } |
| 1587 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1588 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1589 | ld.op = midgard_op_ld_color_buffer_32u_old; |
Alyssa Rosenzweig | 5a175e4 | 2020-05-29 21:11:11 -0400 | [diff] [blame] | 1590 | ld.load_store.address = 16; |
| 1591 | ld.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1592 | } |
| 1593 | |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1594 | emit_mir_instruction(ctx, ld); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1595 | break; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1596 | } |
| 1597 | |
| 1598 | case nir_intrinsic_load_output: { |
| 1599 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1600 | |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1601 | unsigned bits = nir_dest_bit_size(instr->dest); |
| 1602 | |
| 1603 | midgard_instruction ld; |
| 1604 | if (bits == 16) |
| 1605 | ld = m_ld_color_buffer_as_fp16(reg, 0); |
| 1606 | else |
| 1607 | ld = m_ld_color_buffer_as_fp32(reg, 0); |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1608 | |
Icecream95 | e764192 | 2020-07-19 22:31:26 +1200 | [diff] [blame] | 1609 | ld.load_store.arg_2 = output_load_rt_addr(ctx, instr); |
Icecream95 | 7781d2c | 2020-07-06 19:54:56 +1200 | [diff] [blame] | 1610 | |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1611 | for (unsigned c = 4; c < 16; ++c) |
| 1612 | ld.swizzle[0][c] = 0; |
| 1613 | |
| 1614 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1615 | if (bits == 16) |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1616 | ld.op = midgard_op_ld_color_buffer_as_fp16_old; |
Icecream95 | 2fbe7ca | 2020-07-09 23:44:41 +1200 | [diff] [blame] | 1617 | else |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1618 | ld.op = midgard_op_ld_color_buffer_as_fp32_old; |
Alyssa Rosenzweig | 36af05b | 2020-06-01 14:14:33 -0400 | [diff] [blame] | 1619 | ld.load_store.address = 1; |
| 1620 | ld.load_store.arg_2 = 0x1E; |
| 1621 | } |
| 1622 | |
| 1623 | emit_mir_instruction(ctx, ld); |
| 1624 | break; |
| 1625 | } |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1626 | |
| 1627 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1628 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 7c2647f | 2020-03-10 15:48:52 -0400 | [diff] [blame] | 1629 | reg = nir_dest_index(&instr->dest); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1630 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1631 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1632 | ins.has_constants = true; |
Boris Brezillon | a5005c3 | 2020-10-08 10:58:53 +0200 | [diff] [blame] | 1633 | memcpy(ins.constants.f32, ctx->blend_constants, sizeof(ctx->blend_constants)); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1634 | emit_mir_instruction(ctx, ins); |
| 1635 | break; |
| 1636 | } |
| 1637 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1638 | case nir_intrinsic_store_output: |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1639 | case nir_intrinsic_store_combined_output_pan: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1640 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1641 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1642 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1643 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1644 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1645 | |
| 1646 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1647 | bool combined = instr->intrinsic == |
| 1648 | nir_intrinsic_store_combined_output_pan; |
| 1649 | |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1650 | const nir_variable *var; |
Alyssa Rosenzweig | dfaa4c5 | 2020-11-04 08:32:16 -0500 | [diff] [blame] | 1651 | var = nir_find_variable_with_driver_location(ctx->nir, nir_var_shader_out, |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1652 | nir_intrinsic_base(instr)); |
| 1653 | assert(var); |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 1654 | |
| 1655 | /* Dual-source blend writeout is done by leaving the |
| 1656 | * value in r2 for the blend shader to use. */ |
| 1657 | if (var->data.index) { |
| 1658 | if (instr->src[0].is_ssa) { |
| 1659 | emit_explicit_constant(ctx, reg, reg); |
| 1660 | |
| 1661 | unsigned out = make_compiler_temp(ctx); |
| 1662 | |
| 1663 | midgard_instruction ins = v_mov(reg, out); |
| 1664 | emit_mir_instruction(ctx, ins); |
| 1665 | |
| 1666 | ctx->blend_src1 = out; |
| 1667 | } else { |
| 1668 | ctx->blend_src1 = reg; |
| 1669 | } |
| 1670 | |
| 1671 | break; |
| 1672 | } |
| 1673 | |
| 1674 | enum midgard_rt_id rt; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1675 | if (var->data.location == FRAG_RESULT_COLOR) |
| 1676 | rt = MIDGARD_COLOR_RT0; |
| 1677 | else if (var->data.location >= FRAG_RESULT_DATA0) |
| 1678 | rt = MIDGARD_COLOR_RT0 + var->data.location - |
| 1679 | FRAG_RESULT_DATA0; |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1680 | else if (combined) |
| 1681 | rt = MIDGARD_ZS_RT; |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1682 | else |
Eric Anholt | 4c24c82 | 2020-08-25 10:15:27 -0700 | [diff] [blame] | 1683 | unreachable("bad rt"); |
Boris Brezillon | c68cd39 | 2020-01-31 09:22:50 +0100 | [diff] [blame] | 1684 | |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1685 | unsigned reg_z = ~0, reg_s = ~0; |
| 1686 | if (combined) { |
| 1687 | unsigned writeout = nir_intrinsic_component(instr); |
| 1688 | if (writeout & PAN_WRITEOUT_Z) |
| 1689 | reg_z = nir_src_index(ctx, &instr->src[2]); |
| 1690 | if (writeout & PAN_WRITEOUT_S) |
| 1691 | reg_s = nir_src_index(ctx, &instr->src[3]); |
| 1692 | } |
| 1693 | |
| 1694 | emit_fragment_store(ctx, reg, reg_z, reg_s, rt); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1695 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Icecream95 | d37e901 | 2020-06-06 17:25:08 +1200 | [diff] [blame] | 1696 | assert(instr->intrinsic == nir_intrinsic_store_output); |
| 1697 | |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1698 | /* We should have been vectorized, though we don't |
| 1699 | * currently check that st_vary is emitted only once |
| 1700 | * per slot (this is relevant, since there's not a mask |
| 1701 | * parameter available on the store [set to 0 by the |
| 1702 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1703 | * swizzle. If this is a constant source, we'll need to |
| 1704 | * emit that explicitly. */ |
| 1705 | |
| 1706 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1707 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1708 | unsigned dst_component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | 2788721 | 2019-08-15 16:53:03 -0700 | [diff] [blame] | 1709 | unsigned nr_comp = nir_src_num_components(instr->src[0]); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1710 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1711 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1712 | st.load_store.arg_1 = 0x9E; |
| 1713 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1714 | |
Jason Ekstrand | 0aa08ae | 2020-09-30 21:20:53 -0500 | [diff] [blame] | 1715 | switch (nir_alu_type_get_base_type(nir_intrinsic_src_type(instr))) { |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1716 | case nir_type_uint: |
| 1717 | case nir_type_bool: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1718 | st.op = midgard_op_st_vary_32u; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1719 | break; |
| 1720 | case nir_type_int: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1721 | st.op = midgard_op_st_vary_32i; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1722 | break; |
| 1723 | case nir_type_float: |
Italo Nicola | bea6a65 | 2020-07-23 19:24:39 +0000 | [diff] [blame] | 1724 | st.op = midgard_op_st_vary_32; |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1725 | break; |
| 1726 | default: |
| 1727 | unreachable("Attempted to store unknown type"); |
| 1728 | break; |
| 1729 | } |
| 1730 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1731 | /* nir_intrinsic_component(store_intr) encodes the |
| 1732 | * destination component start. Source component offset |
| 1733 | * adjustment is taken care of in |
| 1734 | * install_registers_instr(), when offset_swizzle() is |
| 1735 | * called. |
| 1736 | */ |
| 1737 | unsigned src_component = COMPONENT_X; |
| 1738 | |
| 1739 | assert(nr_comp > 0); |
| 1740 | for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) { |
| 1741 | st.swizzle[0][i] = src_component; |
| 1742 | if (i >= dst_component && i < dst_component + nr_comp - 1) |
| 1743 | src_component++; |
| 1744 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1745 | |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1746 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1747 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1748 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1749 | assert(0); |
| 1750 | } |
| 1751 | |
| 1752 | break; |
| 1753 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1754 | /* Special case of store_output for lowered blend shaders */ |
| 1755 | case nir_intrinsic_store_raw_output_pan: |
| 1756 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 1757 | reg = nir_src_index(ctx, &instr->src[0]); |
Icecream95 | a680634 | 2020-06-06 15:41:51 +1200 | [diff] [blame] | 1758 | emit_fragment_store(ctx, reg, ~0, ~0, ctx->blend_rt); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1759 | break; |
| 1760 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1761 | case nir_intrinsic_store_global: |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1762 | case nir_intrinsic_store_shared: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1763 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1764 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | 3a310fb | 2020-02-05 15:17:44 -0500 | [diff] [blame] | 1765 | |
Alyssa Rosenzweig | 0bb25e4 | 2020-02-27 09:41:17 -0500 | [diff] [blame] | 1766 | emit_global(ctx, &instr->instr, false, reg, &instr->src[1], instr->intrinsic == nir_intrinsic_store_shared); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1767 | break; |
| 1768 | |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1769 | case nir_intrinsic_load_ssbo_address: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1770 | emit_sysval_read(ctx, &instr->instr, 1, 0); |
Alyssa Rosenzweig | fcbb3d4 | 2020-02-04 09:46:17 -0500 | [diff] [blame] | 1771 | break; |
| 1772 | |
Jason Ekstrand | 9750164 | 2020-09-22 03:24:45 -0500 | [diff] [blame] | 1773 | case nir_intrinsic_get_ssbo_size: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1774 | emit_sysval_read(ctx, &instr->instr, 1, 8); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 1775 | break; |
Dylan Baker | 8e36961 | 2018-09-14 12:57:32 -0700 | [diff] [blame] | 1776 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1777 | case nir_intrinsic_load_viewport_scale: |
| 1778 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 1779 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 1780 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 1781 | emit_sysval_read(ctx, &instr->instr, 3, 0); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1782 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1783 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1784 | case nir_intrinsic_load_work_group_id: |
| 1785 | case nir_intrinsic_load_local_invocation_id: |
| 1786 | emit_compute_builtin(ctx, instr); |
| 1787 | break; |
| 1788 | |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1789 | case nir_intrinsic_load_vertex_id: |
| 1790 | case nir_intrinsic_load_instance_id: |
| 1791 | emit_vertex_builtin(ctx, instr); |
| 1792 | break; |
| 1793 | |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1794 | case nir_intrinsic_load_sample_mask_in: |
| 1795 | emit_special(ctx, instr, 96); |
| 1796 | break; |
| 1797 | |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1798 | case nir_intrinsic_load_sample_id: |
Alyssa Rosenzweig | 80ebf11 | 2020-08-27 19:55:53 -0400 | [diff] [blame] | 1799 | emit_special(ctx, instr, 97); |
Alyssa Rosenzweig | da2eed3 | 2020-07-15 09:56:24 -0400 | [diff] [blame] | 1800 | break; |
| 1801 | |
Alyssa Rosenzweig | 3f59098 | 2020-02-03 20:23:41 -0500 | [diff] [blame] | 1802 | case nir_intrinsic_memory_barrier_buffer: |
| 1803 | case nir_intrinsic_memory_barrier_shared: |
| 1804 | break; |
| 1805 | |
| 1806 | case nir_intrinsic_control_barrier: |
| 1807 | schedule_barrier(ctx); |
| 1808 | emit_control_barrier(ctx); |
| 1809 | schedule_barrier(ctx); |
| 1810 | break; |
| 1811 | |
Italo Nicola | d7b6d2e | 2020-08-31 17:32:30 +0000 | [diff] [blame] | 1812 | ATOMIC_CASE(ctx, instr, add, add); |
| 1813 | ATOMIC_CASE(ctx, instr, and, and); |
| 1814 | ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg); |
| 1815 | ATOMIC_CASE(ctx, instr, exchange, xchg); |
| 1816 | ATOMIC_CASE(ctx, instr, imax, imax); |
| 1817 | ATOMIC_CASE(ctx, instr, imin, imin); |
| 1818 | ATOMIC_CASE(ctx, instr, or, or); |
| 1819 | ATOMIC_CASE(ctx, instr, umax, umax); |
| 1820 | ATOMIC_CASE(ctx, instr, umin, umin); |
| 1821 | ATOMIC_CASE(ctx, instr, xor, xor); |
| 1822 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1823 | default: |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 1824 | fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1825 | assert(0); |
| 1826 | break; |
| 1827 | } |
| 1828 | } |
| 1829 | |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 1830 | /* Returns dimension with 0 special casing cubemaps */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1831 | static unsigned |
| 1832 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 1833 | { |
| 1834 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1835 | case GLSL_SAMPLER_DIM_1D: |
| 1836 | case GLSL_SAMPLER_DIM_BUF: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 1837 | return 1; |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1838 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1839 | case GLSL_SAMPLER_DIM_2D: |
Alyssa Rosenzweig | a2748d4 | 2020-06-30 15:31:30 -0400 | [diff] [blame] | 1840 | case GLSL_SAMPLER_DIM_MS: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1841 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 1842 | case GLSL_SAMPLER_DIM_RECT: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 1843 | return 2; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1844 | |
| 1845 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 1846 | return 3; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1847 | |
| 1848 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 1d0b3ef | 2020-08-05 18:11:15 -0400 | [diff] [blame] | 1849 | return 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1850 | |
| 1851 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1852 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1853 | assert(0); |
| 1854 | return 0; |
| 1855 | } |
| 1856 | } |
| 1857 | |
Alyssa Rosenzweig | c6c906e | 2020-05-21 18:02:38 -0400 | [diff] [blame] | 1858 | /* Tries to attach an explicit LOD or bias as a constant. Returns whether this |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1859 | * was successful */ |
| 1860 | |
| 1861 | static bool |
| 1862 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1863 | compiler_context *ctx, |
| 1864 | nir_src lod, |
| 1865 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1866 | { |
| 1867 | /* To attach as constant, it has to *be* constant */ |
| 1868 | |
| 1869 | if (!nir_src_is_const(lod)) |
| 1870 | return false; |
| 1871 | |
| 1872 | float f = nir_src_as_float(lod); |
| 1873 | |
| 1874 | /* Break into fixed-point */ |
| 1875 | signed lod_int = f; |
| 1876 | float lod_frac = f - lod_int; |
| 1877 | |
| 1878 | /* Carry over negative fractions */ |
| 1879 | if (lod_frac < 0.0) { |
| 1880 | lod_int--; |
| 1881 | lod_frac += 1.0; |
| 1882 | } |
| 1883 | |
| 1884 | /* Encode */ |
| 1885 | word->bias = float_to_ubyte(lod_frac); |
| 1886 | word->bias_int = lod_int; |
| 1887 | |
| 1888 | return true; |
| 1889 | } |
| 1890 | |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 1891 | static enum mali_texture_mode |
| 1892 | mdg_texture_mode(nir_tex_instr *instr) |
| 1893 | { |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 1894 | if (instr->op == nir_texop_tg4 && instr->is_shadow) |
| 1895 | return TEXTURE_GATHER_SHADOW; |
| 1896 | else if (instr->op == nir_texop_tg4) |
| 1897 | return TEXTURE_GATHER_X + instr->component; |
| 1898 | else if (instr->is_shadow) |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 1899 | return TEXTURE_SHADOW; |
| 1900 | else |
| 1901 | return TEXTURE_NORMAL; |
| 1902 | } |
| 1903 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1904 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1905 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1906 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1907 | { |
| 1908 | /* TODO */ |
| 1909 | //assert (!instr->sampler); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1910 | |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1911 | nir_dest *dest = &instr->dest; |
| 1912 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1913 | int texture_index = instr->texture_index; |
| 1914 | int sampler_index = texture_index; |
| 1915 | |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 1916 | nir_alu_type dest_base = nir_alu_type_get_base_type(instr->dest_type); |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1917 | nir_alu_type dest_type = dest_base | nir_dest_bit_size(*dest); |
| 1918 | |
| 1919 | /* texture instructions support float outmods */ |
| 1920 | unsigned outmod = midgard_outmod_none; |
| 1921 | if (dest_base == nir_type_float) { |
| 1922 | outmod = mir_determine_float_outmod(ctx, &dest, 0); |
| 1923 | } |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 1924 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1925 | midgard_instruction ins = { |
| 1926 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1927 | .mask = 0xF, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1928 | .dest = nir_dest_index(dest), |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 1929 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 1930 | .dest_type = dest_type, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1931 | .swizzle = SWIZZLE_IDENTITY_4, |
Italo Nicola | 83592de | 2020-07-15 18:48:42 +0000 | [diff] [blame] | 1932 | .outmod = outmod, |
Italo Nicola | 92c808c | 2020-07-29 19:10:25 +0000 | [diff] [blame] | 1933 | .op = midgard_texop, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1934 | .texture = { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1935 | .format = midgard_tex_format(instr->sampler_dim), |
| 1936 | .texture_handle = texture_index, |
| 1937 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | f6e19dd | 2020-08-28 08:35:19 -0400 | [diff] [blame] | 1938 | .mode = mdg_texture_mode(instr) |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1939 | } |
| 1940 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 1941 | |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 1942 | if (instr->is_shadow && !instr->is_new_style_shadow && instr->op != nir_texop_tg4) |
Icecream95 | d1290e7 | 2020-05-12 10:16:31 +1200 | [diff] [blame] | 1943 | for (int i = 0; i < 4; ++i) |
| 1944 | ins.swizzle[0][i] = COMPONENT_X; |
| 1945 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1946 | /* We may need a temporary for the coordinate */ |
| 1947 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1948 | bool needs_temp_coord = |
| 1949 | (midgard_texop == TEXTURE_OP_TEXEL_FETCH) || |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1950 | (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) || |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1951 | (instr->is_shadow); |
| 1952 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1953 | unsigned coords = needs_temp_coord ? make_compiler_temp_reg(ctx) : 0; |
| 1954 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1955 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1956 | int index = nir_src_index(ctx, &instr->src[i].src); |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 1957 | unsigned nr_components = nir_src_num_components(instr->src[i].src); |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 1958 | unsigned sz = nir_src_bit_size(instr->src[i].src); |
| 1959 | nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1960 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1961 | switch (instr->src[i].src_type) { |
| 1962 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1963 | emit_explicit_constant(ctx, index, index); |
| 1964 | |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1965 | unsigned coord_mask = mask_of(instr->coord_components); |
| 1966 | |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 1967 | bool flip_zw = (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) && (coord_mask & (1 << COMPONENT_Z)); |
| 1968 | |
| 1969 | if (flip_zw) |
| 1970 | coord_mask ^= ((1 << COMPONENT_Z) | (1 << COMPONENT_W)); |
| 1971 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1972 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
| 1973 | /* texelFetch is undefined on samplerCube */ |
| 1974 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 1975 | |
| 1976 | /* For cubemaps, we use a special ld/st op to |
| 1977 | * select the face and copy the xy into the |
| 1978 | * texture register */ |
| 1979 | |
| 1980 | midgard_instruction ld = m_ld_cubemap_coords(coords, 0); |
| 1981 | ld.src[1] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 1982 | ld.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1983 | ld.mask = 0x3; /* xy */ |
| 1984 | ld.load_store.arg_1 = 0x20; |
| 1985 | ld.swizzle[1][3] = COMPONENT_X; |
| 1986 | emit_mir_instruction(ctx, ld); |
| 1987 | |
| 1988 | /* xyzw -> xyxx */ |
| 1989 | ins.swizzle[1][2] = instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
| 1990 | ins.swizzle[1][3] = COMPONENT_X; |
| 1991 | } else if (needs_temp_coord) { |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1992 | /* mov coord_temp, coords */ |
| 1993 | midgard_instruction mov = v_mov(index, coords); |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1994 | mov.mask = coord_mask; |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 1995 | |
| 1996 | if (flip_zw) |
| 1997 | mov.swizzle[1][COMPONENT_W] = COMPONENT_Z; |
| 1998 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1999 | emit_mir_instruction(ctx, mov); |
| 2000 | } else { |
| 2001 | coords = index; |
| 2002 | } |
| 2003 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2004 | ins.src[1] = coords; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2005 | ins.src_types[1] = T; |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 2006 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2007 | /* Texelfetch coordinates uses all four elements |
| 2008 | * (xyz/index) regardless of texture dimensionality, |
| 2009 | * which means it's necessary to zero the unused |
| 2010 | * components to keep everything happy */ |
| 2011 | |
| 2012 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2013 | /* mov index.zw, #0, or generalized */ |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 2014 | midgard_instruction mov = |
| 2015 | v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), coords); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2016 | mov.has_constants = true; |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 2017 | mov.mask = coord_mask ^ 0xF; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2018 | emit_mir_instruction(ctx, mov); |
| 2019 | } |
| 2020 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2021 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2022 | /* Array component in w but NIR wants it in z, |
| 2023 | * but if we have a temp coord we already fixed |
| 2024 | * that up */ |
| 2025 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2026 | if (nr_components == 3) { |
| 2027 | ins.swizzle[1][2] = COMPONENT_Z; |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 2028 | ins.swizzle[1][3] = needs_temp_coord ? COMPONENT_W : COMPONENT_Z; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2029 | } else if (nr_components == 2) { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2030 | ins.swizzle[1][2] = |
| 2031 | instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2032 | ins.swizzle[1][3] = COMPONENT_X; |
| 2033 | } else |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 2034 | unreachable("Invalid texture 2D components"); |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 2035 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2036 | |
Alyssa Rosenzweig | 64b2fe9 | 2019-12-20 12:38:24 -0500 | [diff] [blame] | 2037 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 2038 | /* We zeroed */ |
| 2039 | ins.swizzle[1][2] = COMPONENT_Z; |
| 2040 | ins.swizzle[1][3] = COMPONENT_W; |
| 2041 | } |
| 2042 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2043 | break; |
| 2044 | } |
| 2045 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 2046 | case nir_tex_src_bias: |
| 2047 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2048 | /* Try as a constant if we can */ |
| 2049 | |
| 2050 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 2051 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 2052 | break; |
| 2053 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2054 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2055 | ins.src[2] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2056 | ins.src_types[2] = T; |
Alyssa Rosenzweig | 72e5749 | 2019-12-20 12:34:20 -0500 | [diff] [blame] | 2057 | |
| 2058 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2059 | ins.swizzle[2][c] = COMPONENT_X; |
| 2060 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2061 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 2062 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2063 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2064 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 2065 | |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2066 | case nir_tex_src_offset: { |
| 2067 | ins.texture.offset_register = true; |
| 2068 | ins.src[3] = index; |
Alyssa Rosenzweig | 4fb0217 | 2020-04-27 19:11:19 -0400 | [diff] [blame] | 2069 | ins.src_types[3] = T; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2070 | |
| 2071 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 2072 | ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c; |
| 2073 | |
| 2074 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | 4ec1f95 | 2019-12-20 12:58:10 -0500 | [diff] [blame] | 2075 | break; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 2076 | }; |
| 2077 | |
Alyssa Rosenzweig | 6d9f951 | 2020-06-30 15:31:39 -0400 | [diff] [blame] | 2078 | case nir_tex_src_comparator: |
| 2079 | case nir_tex_src_ms_index: { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 2080 | unsigned comp = COMPONENT_Z; |
| 2081 | |
| 2082 | /* mov coord_temp.foo, coords */ |
| 2083 | midgard_instruction mov = v_mov(index, coords); |
| 2084 | mov.mask = 1 << comp; |
| 2085 | |
| 2086 | for (unsigned i = 0; i < MIR_VEC_COMPONENTS; ++i) |
| 2087 | mov.swizzle[1][i] = COMPONENT_X; |
| 2088 | |
| 2089 | emit_mir_instruction(ctx, mov); |
| 2090 | break; |
| 2091 | } |
| 2092 | |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2093 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2094 | fprintf(stderr, "Unknown texture source type: %d\n", instr->src[i].src_type); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2095 | assert(0); |
| 2096 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2097 | } |
| 2098 | } |
| 2099 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2100 | emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2101 | } |
| 2102 | |
| 2103 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2104 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 2105 | { |
| 2106 | switch (instr->op) { |
| 2107 | case nir_texop_tex: |
| 2108 | case nir_texop_txb: |
| 2109 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 2110 | break; |
| 2111 | case nir_texop_txl: |
Alyssa Rosenzweig | 7dab574 | 2020-08-28 09:48:38 -0400 | [diff] [blame] | 2112 | case nir_texop_tg4: |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2113 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 2114 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2115 | case nir_texop_txf: |
Alyssa Rosenzweig | 63a8722 | 2020-06-30 15:32:01 -0400 | [diff] [blame] | 2116 | case nir_texop_txf_ms: |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 2117 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 2118 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2119 | case nir_texop_txs: |
Alyssa Rosenzweig | b756a66 | 2020-03-10 16:19:33 -0400 | [diff] [blame] | 2120 | emit_sysval_read(ctx, &instr->instr, 4, 0); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 2121 | break; |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2122 | default: { |
Tomeu Vizoso | ae5e640 | 2020-02-21 13:47:38 +0100 | [diff] [blame] | 2123 | fprintf(stderr, "Unhandled texture op: %d\n", instr->op); |
Tomeu Vizoso | 226c1ef | 2019-12-19 15:07:39 +0100 | [diff] [blame] | 2124 | assert(0); |
| 2125 | } |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 2126 | } |
| 2127 | } |
| 2128 | |
| 2129 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2130 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 2131 | { |
| 2132 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2133 | case nir_jump_break: { |
| 2134 | /* Emit a branch out of the loop */ |
| 2135 | struct midgard_instruction br = v_branch(false, false); |
| 2136 | br.branch.target_type = TARGET_BREAK; |
| 2137 | br.branch.target_break = ctx->current_loop_depth; |
| 2138 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2139 | break; |
| 2140 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2141 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2142 | default: |
| 2143 | DBG("Unknown jump type %d\n", instr->type); |
| 2144 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2145 | } |
| 2146 | } |
| 2147 | |
| 2148 | static void |
| 2149 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 2150 | { |
| 2151 | switch (instr->type) { |
| 2152 | case nir_instr_type_load_const: |
| 2153 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 2154 | break; |
| 2155 | |
| 2156 | case nir_instr_type_intrinsic: |
| 2157 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 2158 | break; |
| 2159 | |
| 2160 | case nir_instr_type_alu: |
| 2161 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 2162 | break; |
| 2163 | |
| 2164 | case nir_instr_type_tex: |
| 2165 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 2166 | break; |
| 2167 | |
| 2168 | case nir_instr_type_jump: |
| 2169 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 2170 | break; |
| 2171 | |
| 2172 | case nir_instr_type_ssa_undef: |
| 2173 | /* Spurious */ |
| 2174 | break; |
| 2175 | |
| 2176 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2177 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2178 | break; |
| 2179 | } |
| 2180 | } |
| 2181 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2182 | |
| 2183 | /* ALU instructions can inline or embed constants, which decreases register |
| 2184 | * pressure and saves space. */ |
| 2185 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2186 | #define CONDITIONAL_ATTACH(idx) { \ |
| 2187 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2188 | \ |
| 2189 | if (entry) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2190 | attach_constants(ctx, alu, entry, alu->src[idx] + 1); \ |
| 2191 | alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2192 | } \ |
| 2193 | } |
| 2194 | |
| 2195 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2196 | inline_alu_constants(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2197 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2198 | mir_foreach_instr_in_block(block, alu) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2199 | /* Other instructions cannot inline constants */ |
| 2200 | if (alu->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 2201 | if (alu->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2202 | |
| 2203 | /* If there is already a constant here, we can do nothing */ |
| 2204 | if (alu->has_constants) continue; |
| 2205 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2206 | CONDITIONAL_ATTACH(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2207 | |
| 2208 | if (!alu->has_constants) { |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2209 | CONDITIONAL_ATTACH(1) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2210 | } else if (!alu->inline_constant) { |
| 2211 | /* Corner case: _two_ vec4 constants, for instance with a |
| 2212 | * csel. For this case, we can only use a constant |
| 2213 | * register for one, we'll have to emit a move for the |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2214 | * other. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2215 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2216 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1); |
Alyssa Rosenzweig | 3b10bcd | 2020-04-27 17:47:13 -0400 | [diff] [blame] | 2217 | unsigned scratch = make_compiler_temp(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2218 | |
| 2219 | if (entry) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 2220 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch); |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2221 | attach_constants(ctx, &ins, entry, alu->src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2222 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2223 | /* Set the source */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2224 | alu->src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2225 | |
| 2226 | /* Inject us -before- the last instruction which set r31 */ |
Boris Brezillon | 938c5b0 | 2019-08-28 09:17:21 +0200 | [diff] [blame] | 2227 | mir_insert_instruction_before(ctx, mir_prev_op(alu), ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2228 | } |
| 2229 | } |
| 2230 | } |
| 2231 | } |
| 2232 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2233 | unsigned |
| 2234 | max_bitsize_for_alu(midgard_instruction *ins) |
| 2235 | { |
| 2236 | unsigned max_bitsize = 0; |
| 2237 | for (int i = 0; i < MIR_SRC_COUNT; i++) { |
| 2238 | if (ins->src[i] == ~0) continue; |
| 2239 | unsigned src_bitsize = nir_alu_type_get_type_size(ins->src_types[i]); |
| 2240 | max_bitsize = MAX2(src_bitsize, max_bitsize); |
| 2241 | } |
| 2242 | unsigned dst_bitsize = nir_alu_type_get_type_size(ins->dest_type); |
| 2243 | max_bitsize = MAX2(dst_bitsize, max_bitsize); |
| 2244 | |
| 2245 | /* We don't have fp16 LUTs, so we'll want to emit code like: |
| 2246 | * |
| 2247 | * vlut.fsinr hr0, hr0 |
| 2248 | * |
| 2249 | * where both input and output are 16-bit but the operation is carried |
| 2250 | * out in 32-bit |
| 2251 | */ |
| 2252 | |
| 2253 | switch (ins->op) { |
| 2254 | case midgard_alu_op_fsqrt: |
| 2255 | case midgard_alu_op_frcp: |
| 2256 | case midgard_alu_op_frsqrt: |
| 2257 | case midgard_alu_op_fsin: |
| 2258 | case midgard_alu_op_fcos: |
| 2259 | case midgard_alu_op_fexp2: |
| 2260 | case midgard_alu_op_flog2: |
| 2261 | max_bitsize = MAX2(max_bitsize, 32); |
| 2262 | break; |
| 2263 | |
| 2264 | default: |
| 2265 | break; |
| 2266 | } |
| 2267 | |
Alyssa Rosenzweig | 3e2cb21 | 2020-08-27 14:35:23 -0400 | [diff] [blame] | 2268 | /* High implies computing at a higher bitsize, e.g umul_high of 32-bit |
| 2269 | * requires computing at 64-bit */ |
| 2270 | if (midgard_is_integer_out_op(ins->op) && ins->outmod == midgard_outmod_int_high) { |
| 2271 | max_bitsize *= 2; |
| 2272 | assert(max_bitsize <= 64); |
| 2273 | } |
| 2274 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2275 | return max_bitsize; |
| 2276 | } |
| 2277 | |
| 2278 | midgard_reg_mode |
| 2279 | reg_mode_for_bitsize(unsigned bitsize) |
| 2280 | { |
| 2281 | switch (bitsize) { |
| 2282 | /* use 16 pipe for 8 since we don't support vec16 yet */ |
| 2283 | case 8: |
| 2284 | case 16: |
| 2285 | return midgard_reg_mode_16; |
| 2286 | case 32: |
| 2287 | return midgard_reg_mode_32; |
| 2288 | case 64: |
| 2289 | return midgard_reg_mode_64; |
| 2290 | default: |
| 2291 | unreachable("invalid bit size"); |
| 2292 | } |
| 2293 | } |
| 2294 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2295 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2296 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2297 | * constants can be demoted to inline constants, for space savings and |
| 2298 | * sometimes a performance boost */ |
| 2299 | |
| 2300 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2301 | embedded_to_inline_constant(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2302 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2303 | mir_foreach_instr_in_block(block, ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2304 | if (!ins->has_constants) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2305 | if (ins->has_inline_constant) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2306 | |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2307 | unsigned max_bitsize = max_bitsize_for_alu(ins); |
| 2308 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2309 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
Italo Nicola | 5f7e018 | 2020-07-10 09:36:58 +0000 | [diff] [blame] | 2310 | bool is_16 = max_bitsize == 16; |
| 2311 | bool is_32 = max_bitsize == 32; |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2312 | |
| 2313 | if (!(is_16 || is_32)) |
| 2314 | continue; |
| 2315 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2316 | /* src1 cannot be an inline constant due to encoding |
| 2317 | * restrictions. So, if possible we try to flip the arguments |
| 2318 | * in that case */ |
| 2319 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2320 | int op = ins->op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2321 | |
Alyssa Rosenzweig | ba9f3d1 | 2020-04-30 13:11:52 -0400 | [diff] [blame] | 2322 | if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT) && |
| 2323 | alu_opcode_props[op].props & OP_COMMUTES) { |
| 2324 | mir_flip(ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2325 | } |
| 2326 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2327 | if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2328 | /* Component is from the swizzle. Take a nonzero component */ |
| 2329 | assert(ins->mask); |
| 2330 | unsigned first_comp = ffs(ins->mask) - 1; |
| 2331 | unsigned component = ins->swizzle[1][first_comp]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2332 | |
| 2333 | /* Scale constant appropriately, if we can legally */ |
Icecream95 | d97aaad | 2020-06-05 20:17:27 +1200 | [diff] [blame] | 2334 | int16_t scaled_constant = 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2335 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2336 | if (is_16) { |
| 2337 | scaled_constant = ins->constants.u16[component]; |
| 2338 | } else if (midgard_is_integer_op(op)) { |
| 2339 | scaled_constant = ins->constants.u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2340 | |
| 2341 | /* Constant overflow after resize */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2342 | if (scaled_constant != ins->constants.u32[component]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2343 | continue; |
| 2344 | } else { |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2345 | float original = ins->constants.f32[component]; |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2346 | scaled_constant = _mesa_float_to_half(original); |
| 2347 | |
| 2348 | /* Check for loss of precision. If this is |
| 2349 | * mediump, we don't care, but for a highp |
| 2350 | * shader, we need to pay attention. NIR |
| 2351 | * doesn't yet tell us which mode we're in! |
| 2352 | * Practically this prevents most constants |
| 2353 | * from being inlined, sadly. */ |
| 2354 | |
| 2355 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2356 | |
| 2357 | if (fp32 != original) |
| 2358 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2359 | } |
| 2360 | |
Alyssa Rosenzweig | 1cd6535 | 2020-05-21 12:38:27 -0400 | [diff] [blame] | 2361 | /* Should've been const folded */ |
| 2362 | if (ins->src_abs[1] || ins->src_neg[1]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2363 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2364 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2365 | /* Make sure that the constant is not itself a vector |
| 2366 | * by checking if all accessed values are the same. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2367 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2368 | const midgard_constants *cons = &ins->constants; |
| 2369 | uint32_t value = is_16 ? cons->u16[component] : cons->u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2370 | |
| 2371 | bool is_vector = false; |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2372 | unsigned mask = effective_writemask(ins->op, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2373 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2374 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2375 | /* We only care if this component is actually used */ |
| 2376 | if (!(mask & (1 << c))) |
| 2377 | continue; |
| 2378 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2379 | uint32_t test = is_16 ? |
| 2380 | cons->u16[ins->swizzle[1][c]] : |
| 2381 | cons->u32[ins->swizzle[1][c]]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2382 | |
| 2383 | if (test != value) { |
| 2384 | is_vector = true; |
| 2385 | break; |
| 2386 | } |
| 2387 | } |
| 2388 | |
| 2389 | if (is_vector) |
| 2390 | continue; |
| 2391 | |
| 2392 | /* Get rid of the embedded constant */ |
| 2393 | ins->has_constants = false; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2394 | ins->src[1] = ~0; |
| 2395 | ins->has_inline_constant = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2396 | ins->inline_constant = scaled_constant; |
| 2397 | } |
| 2398 | } |
| 2399 | } |
| 2400 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2401 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2402 | * per block is legal semantically */ |
| 2403 | |
| 2404 | static void |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 2405 | midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2406 | { |
| 2407 | bool branched = false; |
| 2408 | |
| 2409 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2410 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2411 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2412 | if (branched) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2413 | mir_remove_instruction(ins); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2414 | |
| 2415 | branched = true; |
| 2416 | } |
| 2417 | } |
| 2418 | |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2419 | /* We want to force the invert on AND/OR to the second slot to legalize into |
| 2420 | * iandnot/iornot. The relevant patterns are for AND (and OR respectively) |
| 2421 | * |
| 2422 | * ~a & #b = ~a & ~(#~b) |
| 2423 | * ~a & b = b & ~a |
| 2424 | */ |
| 2425 | |
| 2426 | static void |
| 2427 | midgard_legalize_invert(compiler_context *ctx, midgard_block *block) |
| 2428 | { |
| 2429 | mir_foreach_instr_in_block(block, ins) { |
| 2430 | if (ins->type != TAG_ALU_4) continue; |
| 2431 | |
Italo Nicola | f4c89bf | 2020-07-09 12:02:57 +0000 | [diff] [blame] | 2432 | if (ins->op != midgard_alu_op_iand && |
| 2433 | ins->op != midgard_alu_op_ior) continue; |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2434 | |
| 2435 | if (ins->src_invert[1] || !ins->src_invert[0]) continue; |
| 2436 | |
| 2437 | if (ins->has_inline_constant) { |
| 2438 | /* ~(#~a) = ~(~#a) = a, so valid, and forces both |
| 2439 | * inverts on */ |
| 2440 | ins->inline_constant = ~ins->inline_constant; |
| 2441 | ins->src_invert[1] = true; |
| 2442 | } else { |
| 2443 | /* Flip to the right invert order. Note |
| 2444 | * has_inline_constant false by assumption on the |
| 2445 | * branch, so flipping makes sense. */ |
| 2446 | mir_flip(ins); |
| 2447 | } |
| 2448 | } |
| 2449 | } |
| 2450 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2451 | static unsigned |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2452 | emit_fragment_epilogue(compiler_context *ctx, unsigned rt) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2453 | { |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2454 | /* Loop to ourselves */ |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2455 | midgard_instruction *br = ctx->writeout_branch[rt]; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2456 | struct midgard_instruction ins = v_branch(false, false); |
Icecream95 | 92d3f1f | 2020-06-06 15:08:06 +1200 | [diff] [blame] | 2457 | ins.writeout = br->writeout; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2458 | ins.branch.target_block = ctx->block_count - 1; |
Boris Brezillon | e1ba0cd | 2020-01-31 10:05:16 +0100 | [diff] [blame] | 2459 | ins.constants.u32[0] = br->constants.u32[0]; |
Icecream95 | 2a5504f | 2020-06-06 14:42:18 +1200 | [diff] [blame] | 2460 | memcpy(&ins.src_types, &br->src_types, sizeof(ins.src_types)); |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2461 | emit_mir_instruction(ctx, ins); |
| 2462 | |
Alyssa Rosenzweig | 3448b26 | 2019-12-03 10:37:01 -0500 | [diff] [blame] | 2463 | ctx->current_block->epilogue = true; |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2464 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2465 | return ins.branch.target_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2466 | } |
| 2467 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2468 | static midgard_block * |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2469 | emit_block_init(compiler_context *ctx) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2470 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2471 | midgard_block *this_block = ctx->after_block; |
| 2472 | ctx->after_block = NULL; |
| 2473 | |
| 2474 | if (!this_block) |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2475 | this_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2476 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2477 | list_addtail(&this_block->base.link, &ctx->blocks); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2478 | |
Alyssa Rosenzweig | c5dd1d5 | 2020-03-11 08:22:08 -0400 | [diff] [blame] | 2479 | this_block->scheduled = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2480 | ++ctx->block_count; |
| 2481 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2482 | /* Set up current block */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2483 | list_inithead(&this_block->base.instructions); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2484 | ctx->current_block = this_block; |
| 2485 | |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2486 | return this_block; |
| 2487 | } |
| 2488 | |
| 2489 | static midgard_block * |
| 2490 | emit_block(compiler_context *ctx, nir_block *block) |
| 2491 | { |
| 2492 | midgard_block *this_block = emit_block_init(ctx); |
| 2493 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2494 | nir_foreach_instr(instr, block) { |
| 2495 | emit_instr(ctx, instr); |
| 2496 | ++ctx->instruction_count; |
| 2497 | } |
| 2498 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2499 | return this_block; |
| 2500 | } |
| 2501 | |
| 2502 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2503 | |
| 2504 | static void |
| 2505 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2506 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2507 | midgard_block *before_block = ctx->current_block; |
| 2508 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2509 | /* Speculatively emit the branch, but we can't fill it in until later */ |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2510 | bool inv = false; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2511 | EMIT(branch, true, true); |
| 2512 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2513 | then_branch->src[0] = mir_get_branch_cond(&nif->condition, &inv); |
Alyssa Rosenzweig | 074815c | 2020-04-29 16:29:01 -0400 | [diff] [blame] | 2514 | then_branch->src_types[0] = nir_type_uint32; |
Alyssa Rosenzweig | db7b0eb | 2020-04-30 14:17:06 -0400 | [diff] [blame] | 2515 | then_branch->branch.invert_conditional = !inv; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2516 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2517 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2518 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2519 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2520 | |
| 2521 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2522 | EMIT(branch, false, false); |
| 2523 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2524 | |
| 2525 | /* Emit second block, and check if it's empty */ |
| 2526 | |
| 2527 | int else_idx = ctx->block_count; |
| 2528 | int count_in = ctx->instruction_count; |
| 2529 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2530 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2531 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2532 | |
| 2533 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2534 | |
| 2535 | assert(then_block); |
| 2536 | assert(else_block); |
| 2537 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2538 | if (ctx->instruction_count == count_in) { |
| 2539 | /* The else block is empty, so don't emit an exit jump */ |
| 2540 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2541 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2542 | } else { |
| 2543 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2544 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2545 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2546 | |
| 2547 | /* Wire up the successors */ |
| 2548 | |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2549 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2550 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2551 | pan_block_add_successor(&before_block->base, &then_block->base); |
| 2552 | pan_block_add_successor(&before_block->base, &else_block->base); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2553 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2554 | pan_block_add_successor(&end_then_block->base, &ctx->after_block->base); |
| 2555 | pan_block_add_successor(&end_else_block->base, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2556 | } |
| 2557 | |
| 2558 | static void |
| 2559 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2560 | { |
| 2561 | /* Remember where we are */ |
| 2562 | midgard_block *start_block = ctx->current_block; |
| 2563 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2564 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2565 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2566 | |
| 2567 | /* Get index from before the body so we can loop back later */ |
| 2568 | int start_idx = ctx->block_count; |
| 2569 | |
| 2570 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2571 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2572 | |
| 2573 | /* Branch back to loop back */ |
| 2574 | struct midgard_instruction br_back = v_branch(false, false); |
| 2575 | br_back.branch.target_block = start_idx; |
| 2576 | emit_mir_instruction(ctx, br_back); |
| 2577 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2578 | /* Mark down that branch in the graph. */ |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2579 | pan_block_add_successor(&start_block->base, &loop_block->base); |
| 2580 | pan_block_add_successor(&ctx->current_block->base, &loop_block->base); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2581 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2582 | /* Find the index of the block about to follow us (note: we don't add |
| 2583 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2584 | int break_block_idx = ctx->block_count; |
| 2585 | |
| 2586 | /* Fix up the break statements we emitted to point to the right place, |
| 2587 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2588 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2589 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2590 | mir_foreach_block_from(ctx, start_block, _block) { |
| 2591 | mir_foreach_instr_in_block(((midgard_block *) _block), ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2592 | if (ins->type != TAG_ALU_4) continue; |
| 2593 | if (!ins->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2594 | |
| 2595 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2596 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2597 | |
| 2598 | /* It's a break! Check if it's our break */ |
| 2599 | if (ins->branch.target_break != loop_idx) continue; |
| 2600 | |
| 2601 | /* Okay, cool, we're breaking out of this loop. |
| 2602 | * Rewrite from a break to a goto */ |
| 2603 | |
| 2604 | ins->branch.target_type = TARGET_GOTO; |
| 2605 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2606 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2607 | pan_block_add_successor(_block, &ctx->after_block->base); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2608 | } |
| 2609 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2610 | |
| 2611 | /* Now that we've finished emitting the loop, free up the depth again |
| 2612 | * so we play nice with recursion amid nested loops */ |
| 2613 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2614 | |
| 2615 | /* Dump loop stats */ |
| 2616 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2617 | } |
| 2618 | |
| 2619 | static midgard_block * |
| 2620 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2621 | { |
| 2622 | midgard_block *start_block = NULL; |
| 2623 | |
| 2624 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2625 | switch (node->type) { |
| 2626 | case nir_cf_node_block: { |
| 2627 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2628 | |
| 2629 | if (!start_block) |
| 2630 | start_block = block; |
| 2631 | |
| 2632 | break; |
| 2633 | } |
| 2634 | |
| 2635 | case nir_cf_node_if: |
| 2636 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2637 | break; |
| 2638 | |
| 2639 | case nir_cf_node_loop: |
| 2640 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2641 | break; |
| 2642 | |
| 2643 | case nir_cf_node_function: |
| 2644 | assert(0); |
| 2645 | break; |
| 2646 | } |
| 2647 | } |
| 2648 | |
| 2649 | return start_block; |
| 2650 | } |
| 2651 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2652 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2653 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2654 | * until we find one that 'works' */ |
| 2655 | |
Italo Nicola | 8150c1d | 2020-07-29 20:14:55 +0000 | [diff] [blame] | 2656 | unsigned |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2657 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2658 | { |
| 2659 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2660 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2661 | mir_foreach_block_from(ctx, initial_block, _v) { |
| 2662 | midgard_block *v = (midgard_block *) _v; |
Alyssa Rosenzweig | 45ac8ea | 2019-11-04 10:32:49 -0500 | [diff] [blame] | 2663 | if (v->quadword_count) { |
| 2664 | midgard_bundle *initial_bundle = |
| 2665 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2666 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2667 | return initial_bundle->tag; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2668 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2669 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2670 | |
Alyssa Rosenzweig | a55a2e02 | 2020-02-04 09:28:06 -0500 | [diff] [blame] | 2671 | /* Default to a tag 1 which will break from the shader, in case we jump |
| 2672 | * to the exit block (i.e. `return` in a compute shader) */ |
| 2673 | |
| 2674 | return 1; |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2675 | } |
| 2676 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2677 | /* For each fragment writeout instruction, generate a writeout loop to |
| 2678 | * associate with it */ |
| 2679 | |
| 2680 | static void |
| 2681 | mir_add_writeout_loops(compiler_context *ctx) |
| 2682 | { |
| 2683 | for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) { |
| 2684 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 2685 | if (!br) continue; |
| 2686 | |
| 2687 | unsigned popped = br->branch.target_block; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2688 | pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base), &ctx->current_block->base); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2689 | br->branch.target_block = emit_fragment_epilogue(ctx, rt); |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2690 | br->branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2691 | |
| 2692 | /* If we have more RTs, we'll need to restore back after our |
| 2693 | * loop terminates */ |
| 2694 | |
| 2695 | if ((rt + 1) < ARRAY_SIZE(ctx->writeout_branch) && ctx->writeout_branch[rt + 1]) { |
| 2696 | midgard_instruction uncond = v_branch(false, false); |
| 2697 | uncond.branch.target_block = popped; |
Alyssa Rosenzweig | e27fd4b | 2020-04-27 20:34:36 -0400 | [diff] [blame] | 2698 | uncond.branch.target_type = TARGET_GOTO; |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2699 | emit_mir_instruction(ctx, uncond); |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2700 | pan_block_add_successor(&ctx->current_block->base, &(mir_get_block(ctx, popped)->base)); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2701 | schedule_barrier(ctx); |
| 2702 | } else { |
| 2703 | /* We're last, so we can terminate here */ |
| 2704 | br->last_writeout = true; |
| 2705 | } |
| 2706 | } |
| 2707 | } |
| 2708 | |
Boris Brezillon | 69c864b | 2020-10-17 12:08:17 +0200 | [diff] [blame] | 2709 | panfrost_program * |
| 2710 | midgard_compile_shader_nir(void *mem_ctx, nir_shader *nir, |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2711 | const struct panfrost_compile_inputs *inputs) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2712 | { |
Boris Brezillon | 69c864b | 2020-10-17 12:08:17 +0200 | [diff] [blame] | 2713 | panfrost_program *program = rzalloc(mem_ctx, panfrost_program); |
| 2714 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2715 | struct util_dynarray *compiled = &program->compiled; |
| 2716 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2717 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2718 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2719 | /* TODO: Bound against what? */ |
| 2720 | compiler_context *ctx = rzalloc(NULL, compiler_context); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2721 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2722 | ctx->nir = nir; |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2723 | ctx->stage = nir->info.stage; |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2724 | ctx->is_blend = inputs->is_blend; |
| 2725 | ctx->blend_rt = MIDGARD_COLOR_RT0 + inputs->blend.rt; |
Boris Brezillon | a5005c3 | 2020-10-08 10:58:53 +0200 | [diff] [blame] | 2726 | memcpy(ctx->blend_constants, inputs->blend.constants, sizeof(ctx->blend_constants)); |
Alyssa Rosenzweig | 277b616 | 2020-06-12 16:45:24 -0400 | [diff] [blame] | 2727 | ctx->blend_input = ~0; |
Icecream95 | 85954ec | 2020-06-25 22:21:50 +1200 | [diff] [blame] | 2728 | ctx->blend_src1 = ~0; |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2729 | ctx->quirks = midgard_get_quirks(inputs->gpu_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2730 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2731 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2732 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2733 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2734 | ctx->uniform_cutoff = 8; |
| 2735 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2736 | /* Initialize at a global (not block) level hash tables */ |
| 2737 | |
| 2738 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2739 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2740 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2741 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2742 | * messed with our I/O quite a bit already) */ |
| 2743 | |
| 2744 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2745 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2746 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2747 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | 2023716 | 2019-08-26 12:14:11 -0700 | [diff] [blame] | 2748 | NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2749 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2750 | |
| 2751 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2752 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 2753 | NIR_PASS_V(nir, nir_split_var_copies); |
| 2754 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2755 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 2756 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2757 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2758 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2759 | unsigned pan_quirks = panfrost_get_quirks(inputs->gpu_id); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 2760 | NIR_PASS_V(nir, pan_lower_framebuffer, |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2761 | inputs->rt_formats, inputs->is_blend, pan_quirks); |
Icecream95 | 1e1eee9 | 2020-07-06 19:30:37 +1200 | [diff] [blame] | 2762 | |
Jason Ekstrand | b019b22 | 2020-06-10 17:54:25 -0500 | [diff] [blame] | 2763 | NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out, |
| 2764 | glsl_type_size, 0); |
Alyssa Rosenzweig | 3148937 | 2019-11-05 08:59:49 -0500 | [diff] [blame] | 2765 | NIR_PASS_V(nir, nir_lower_ssbo); |
Alyssa Rosenzweig | 42319c5 | 2020-11-04 08:37:55 -0500 | [diff] [blame^] | 2766 | NIR_PASS_V(nir, pan_nir_lower_zs_store); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2767 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2768 | /* Optimisation passes */ |
| 2769 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2770 | optimise_nir(nir, ctx->quirks, inputs->is_blend); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2771 | |
Alyssa Rosenzweig | 42319c5 | 2020-11-04 08:37:55 -0500 | [diff] [blame^] | 2772 | NIR_PASS_V(nir, pan_nir_reorder_writeout); |
Icecream95 | 0ff6263 | 2020-07-06 23:52:40 +1200 | [diff] [blame] | 2773 | |
Icecream95 | 756441b | 2020-09-26 12:19:14 +1200 | [diff] [blame] | 2774 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !nir->info.internal) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2775 | nir_print_shader(nir, stdout); |
| 2776 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2777 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2778 | /* Assign sysvals and counts, now that we're sure |
| 2779 | * (post-optimisation) */ |
| 2780 | |
Alyssa Rosenzweig | 680fb05 | 2020-08-18 08:31:42 -0400 | [diff] [blame] | 2781 | panfrost_nir_assign_sysvals(&ctx->sysvals, ctx, nir); |
Alyssa Rosenzweig | c2ff3bb | 2020-03-10 16:00:56 -0400 | [diff] [blame] | 2782 | program->sysval_count = ctx->sysvals.sysval_count; |
| 2783 | memcpy(program->sysvals, ctx->sysvals.sysvals, sizeof(ctx->sysvals.sysvals[0]) * ctx->sysvals.sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2784 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2785 | nir_foreach_function(func, nir) { |
| 2786 | if (!func->impl) |
| 2787 | continue; |
| 2788 | |
| 2789 | list_inithead(&ctx->blocks); |
| 2790 | ctx->block_count = 0; |
| 2791 | ctx->func = func; |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 2792 | ctx->already_emitted = calloc(BITSET_WORDS(func->impl->ssa_alloc), sizeof(BITSET_WORD)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2793 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2794 | if (nir->info.outputs_read && !inputs->is_blend) { |
Icecream95 | ed4d273 | 2020-07-08 13:15:09 +1200 | [diff] [blame] | 2795 | emit_block_init(ctx); |
| 2796 | |
| 2797 | struct midgard_instruction wait = v_branch(false, false); |
| 2798 | wait.branch.target_type = TARGET_TILEBUF_WAIT; |
| 2799 | |
| 2800 | emit_mir_instruction(ctx, wait); |
| 2801 | |
| 2802 | ++ctx->instruction_count; |
| 2803 | } |
| 2804 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2805 | emit_cf_list(ctx, &func->impl->body); |
Alyssa Rosenzweig | 22bb5a9 | 2020-04-29 18:08:26 -0400 | [diff] [blame] | 2806 | free(ctx->already_emitted); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2807 | break; /* TODO: Multi-function shaders */ |
| 2808 | } |
| 2809 | |
Boris Brezillon | 69c864b | 2020-10-17 12:08:17 +0200 | [diff] [blame] | 2810 | util_dynarray_init(compiled, program); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2811 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2812 | /* Per-block lowering before opts */ |
| 2813 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2814 | mir_foreach_block(ctx, _block) { |
| 2815 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2816 | inline_alu_constants(ctx, block); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2817 | embedded_to_inline_constant(ctx, block); |
| 2818 | } |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2819 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2820 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2821 | bool progress = false; |
| 2822 | |
| 2823 | do { |
| 2824 | progress = false; |
Alyssa Rosenzweig | fc06b8b | 2020-05-06 17:34:09 -0400 | [diff] [blame] | 2825 | progress |= midgard_opt_dead_code_eliminate(ctx); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2826 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2827 | mir_foreach_block(ctx, _block) { |
| 2828 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2829 | progress |= midgard_opt_copy_prop(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 2830 | progress |= midgard_opt_combine_projection(ctx, block); |
| 2831 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2832 | } |
| 2833 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2834 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2835 | mir_foreach_block(ctx, _block) { |
| 2836 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 2837 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 622e3a8 | 2020-06-02 12:15:18 -0400 | [diff] [blame] | 2838 | midgard_legalize_invert(ctx, block); |
Alyssa Rosenzweig | 1c2d469 | 2020-04-30 13:13:24 -0400 | [diff] [blame] | 2839 | midgard_cull_dead_branch(ctx, block); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2840 | } |
| 2841 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2842 | if (ctx->stage == MESA_SHADER_FRAGMENT) |
| 2843 | mir_add_writeout_loops(ctx); |
| 2844 | |
Alyssa Rosenzweig | 9a7f0e2 | 2020-05-12 13:26:32 -0400 | [diff] [blame] | 2845 | /* Analyze now that the code is known but before scheduling creates |
| 2846 | * pipeline registers which are harder to track */ |
| 2847 | mir_analyze_helper_terminate(ctx); |
| 2848 | mir_analyze_helper_requirements(ctx); |
| 2849 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2850 | /* Schedule! */ |
Robert Foss | 62adb65 | 2020-01-15 01:14:16 +0100 | [diff] [blame] | 2851 | midgard_schedule_program(ctx); |
Alyssa Rosenzweig | 9dc3b18 | 2019-12-06 09:32:38 -0500 | [diff] [blame] | 2852 | mir_ra(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2853 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2854 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 2855 | * sequence. Save instruction boundaries such that lookahead tags can |
| 2856 | * be assigned easily */ |
| 2857 | |
| 2858 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 2859 | |
| 2860 | int bundle_count = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2861 | mir_foreach_block(ctx, _block) { |
| 2862 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2863 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 2864 | } |
| 2865 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 2866 | int bundle_idx = 0; |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2867 | mir_foreach_block(ctx, _block) { |
| 2868 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2869 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 2870 | source_order_bundles[bundle_idx++] = bundle; |
| 2871 | } |
| 2872 | } |
| 2873 | |
| 2874 | int current_bundle = 0; |
| 2875 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 2876 | /* Midgard prefetches instruction types, so during emission we |
| 2877 | * need to lookahead. Unless this is the last instruction, in |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2878 | * which we return 1. */ |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 2879 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2880 | mir_foreach_block(ctx, _block) { |
| 2881 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 2882 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2883 | int lookahead = 1; |
| 2884 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2885 | if (!bundle->last_writeout && (current_bundle + 1 < bundle_count)) |
| 2886 | lookahead = source_order_bundles[current_bundle + 1]->tag; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2887 | |
Alyssa Rosenzweig | 30a393f | 2020-05-21 19:14:23 -0400 | [diff] [blame] | 2888 | emit_binary_bundle(ctx, block, bundle, compiled, lookahead); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2889 | ++current_bundle; |
| 2890 | } |
| 2891 | |
| 2892 | /* TODO: Free deeper */ |
| 2893 | //util_dynarray_fini(&block->instructions); |
| 2894 | } |
| 2895 | |
| 2896 | free(source_order_bundles); |
| 2897 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2898 | /* Report the very first tag executed */ |
| 2899 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2900 | |
| 2901 | /* Deal with off-by-one related to the fencepost problem */ |
| 2902 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2903 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 2904 | |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 2905 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2906 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2907 | if ((midgard_debug & MIDGARD_DBG_SHADERS) && !nir->info.internal) { |
| 2908 | disassemble_midgard(stdout, |
| 2909 | program->compiled.data, |
| 2910 | program->compiled.size, |
| 2911 | inputs->gpu_id, ctx->stage); |
| 2912 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2913 | |
Boris Brezillon | 0a74a04 | 2020-10-08 10:09:56 +0200 | [diff] [blame] | 2914 | if ((midgard_debug & MIDGARD_DBG_SHADERDB || inputs->shaderdb) && |
| 2915 | !nir->info.internal) { |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 2916 | unsigned nr_bundles = 0, nr_ins = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2917 | |
| 2918 | /* Count instructions and bundles */ |
| 2919 | |
Alyssa Rosenzweig | 5aaaf7b | 2020-03-11 08:36:31 -0400 | [diff] [blame] | 2920 | mir_foreach_block(ctx, _block) { |
| 2921 | midgard_block *block = (midgard_block *) _block; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2922 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2923 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 2924 | |
Alyssa Rosenzweig | 67909c8 | 2019-08-30 13:08:16 -0700 | [diff] [blame] | 2925 | mir_foreach_bundle_in_block(block, bun) |
| 2926 | nr_ins += bun->instruction_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2927 | } |
| 2928 | |
| 2929 | /* Calculate thread count. There are certain cutoffs by |
| 2930 | * register count for thread count */ |
| 2931 | |
| 2932 | unsigned nr_registers = program->work_register_count; |
| 2933 | |
| 2934 | unsigned nr_threads = |
| 2935 | (nr_registers <= 4) ? 4 : |
| 2936 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2937 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2938 | |
| 2939 | /* Dump stats */ |
| 2940 | |
| 2941 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2942 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2943 | "%u registers, %u threads, %u loops, " |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 2944 | "%u:%u spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2945 | SHADER_DB_COUNT++, |
Alyssa Rosenzweig | 014d2e4 | 2020-05-25 13:19:43 -0400 | [diff] [blame] | 2946 | ctx->is_blend ? "PAN_SHADER_BLEND" : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2947 | gl_shader_stage_name(ctx->stage), |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 2948 | nr_ins, nr_bundles, ctx->quadword_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2949 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2950 | ctx->loop_count, |
| 2951 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2952 | } |
| 2953 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2954 | ralloc_free(ctx); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2955 | |
Boris Brezillon | 69c864b | 2020-10-17 12:08:17 +0200 | [diff] [blame] | 2956 | return program; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2957 | } |