Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
| 36 | #include "main/imports.h" |
| 37 | #include "compiler/nir/nir_builder.h" |
| 38 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 39 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 40 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 41 | #include "util/u_dynarray.h" |
| 42 | #include "util/list.h" |
| 43 | #include "main/mtypes.h" |
| 44 | |
| 45 | #include "midgard.h" |
| 46 | #include "midgard_nir.h" |
| 47 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 48 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 49 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 50 | #include "compiler.h" |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 51 | #include "midgard_quirks.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 52 | |
| 53 | #include "disassemble.h" |
| 54 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 55 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 56 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 57 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 58 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 59 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 60 | }; |
| 61 | |
| 62 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 63 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 64 | unsigned SHADER_DB_COUNT = 0; |
| 65 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 66 | int midgard_debug = 0; |
| 67 | |
| 68 | #define DBG(fmt, ...) \ |
| 69 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 70 | fprintf(stderr, "%s:%d: "fmt, \ |
| 71 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 72 | static midgard_block * |
| 73 | create_empty_block(compiler_context *ctx) |
| 74 | { |
| 75 | midgard_block *blk = rzalloc(ctx, midgard_block); |
| 76 | |
| 77 | blk->predecessors = _mesa_set_create(blk, |
| 78 | _mesa_hash_pointer, |
| 79 | _mesa_key_pointer_equal); |
| 80 | |
| 81 | blk->source_id = ctx->block_source_count++; |
| 82 | |
| 83 | return blk; |
| 84 | } |
| 85 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 86 | static void |
| 87 | midgard_block_add_successor(midgard_block *block, midgard_block *successor) |
| 88 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 89 | assert(block); |
| 90 | assert(successor); |
| 91 | |
| 92 | /* Deduplicate */ |
| 93 | for (unsigned i = 0; i < block->nr_successors; ++i) { |
| 94 | if (block->successors[i] == successor) |
| 95 | return; |
| 96 | } |
| 97 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 98 | block->successors[block->nr_successors++] = successor; |
| 99 | assert(block->nr_successors <= ARRAY_SIZE(block->successors)); |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 100 | |
| 101 | /* Note the predecessor in the other direction */ |
| 102 | _mesa_set_add(successor->predecessors, block); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 103 | } |
| 104 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 105 | static void |
| 106 | schedule_barrier(compiler_context *ctx) |
| 107 | { |
| 108 | midgard_block *temp = ctx->after_block; |
| 109 | ctx->after_block = create_empty_block(ctx); |
| 110 | ctx->block_count++; |
| 111 | list_addtail(&ctx->after_block->link, &ctx->blocks); |
| 112 | list_inithead(&ctx->after_block->instructions); |
| 113 | midgard_block_add_successor(ctx->current_block, ctx->after_block); |
| 114 | ctx->current_block = ctx->after_block; |
| 115 | ctx->after_block = temp; |
| 116 | } |
| 117 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 118 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 119 | * driver seems to do it that way */ |
| 120 | |
| 121 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 122 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 123 | #define M_LOAD_STORE(name, store) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 124 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 125 | midgard_instruction i = { \ |
| 126 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 127 | .mask = 0xF, \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 128 | .dest = ~0, \ |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 129 | .src = { ~0, ~0, ~0, ~0 }, \ |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 130 | .swizzle = SWIZZLE_IDENTITY_4, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 131 | .load_store = { \ |
| 132 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 133 | .address = address \ |
| 134 | } \ |
| 135 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 136 | \ |
| 137 | if (store) \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 138 | i.src[0] = ssa; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 139 | else \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 140 | i.dest = ssa; \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 141 | \ |
| 142 | return i; \ |
| 143 | } |
| 144 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 145 | #define M_LOAD(name) M_LOAD_STORE(name, false) |
| 146 | #define M_STORE(name) M_LOAD_STORE(name, true) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 147 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 148 | /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs |
| 149 | * the corresponding Midgard source */ |
| 150 | |
| 151 | static midgard_vector_alu_src |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 152 | vector_alu_modifiers(nir_alu_src *src, bool is_int, unsigned broadcast_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 153 | bool half, bool sext) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 154 | { |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 155 | /* Figure out how many components there are so we can adjust. |
| 156 | * Specifically we want to broadcast the last channel so things like |
| 157 | * ball2/3 work. |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 158 | */ |
| 159 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 160 | if (broadcast_count && src) { |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 161 | uint8_t last_component = src->swizzle[broadcast_count - 1]; |
| 162 | |
| 163 | for (unsigned c = broadcast_count; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 164 | src->swizzle[c] = last_component; |
| 165 | } |
| 166 | } |
| 167 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 168 | midgard_vector_alu_src alu_src = { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 169 | .rep_low = 0, |
| 170 | .rep_high = 0, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 171 | .half = half |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | }; |
| 173 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 174 | if (is_int) { |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 175 | alu_src.mod = midgard_int_normal; |
| 176 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 177 | /* Sign/zero-extend if needed */ |
| 178 | |
| 179 | if (half) { |
| 180 | alu_src.mod = sext ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 181 | midgard_int_sign_extend |
| 182 | : midgard_int_zero_extend; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 183 | } |
| 184 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 185 | /* These should have been lowered away */ |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 186 | if (src) |
| 187 | assert(!(src->abs || src->negate)); |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 188 | } else { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 189 | if (src) |
| 190 | alu_src.mod = (src->abs << 0) | (src->negate << 1); |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 191 | } |
| 192 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 193 | return alu_src; |
| 194 | } |
| 195 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 196 | /* load/store instructions have both 32-bit and 16-bit variants, depending on |
| 197 | * whether we are using vectors composed of highp or mediump. At the moment, we |
| 198 | * don't support half-floats -- this requires changes in other parts of the |
| 199 | * compiler -- therefore the 16-bit versions are commented out. */ |
| 200 | |
Alyssa Rosenzweig | 74ab80b | 2019-05-14 04:11:36 +0000 | [diff] [blame] | 201 | //M_LOAD(ld_attr_16); |
| 202 | M_LOAD(ld_attr_32); |
| 203 | //M_LOAD(ld_vary_16); |
| 204 | M_LOAD(ld_vary_32); |
Alyssa Rosenzweig | ec2f0b5 | 2019-08-13 08:51:40 -0700 | [diff] [blame] | 205 | M_LOAD(ld_ubo_int4); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 206 | M_LOAD(ld_int4); |
| 207 | M_STORE(st_int4); |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 208 | M_LOAD(ld_color_buffer_32u); |
Alyssa Rosenzweig | 74ab80b | 2019-05-14 04:11:36 +0000 | [diff] [blame] | 209 | //M_STORE(st_vary_16); |
| 210 | M_STORE(st_vary_32); |
Alyssa Rosenzweig | 9ae4d36 | 2019-08-16 07:50:12 -0700 | [diff] [blame] | 211 | M_LOAD(ld_cubemap_coords); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 212 | M_LOAD(ld_compute_id); |
Alyssa Rosenzweig | 564a782 | 2020-01-08 15:11:45 -0500 | [diff] [blame] | 213 | M_LOAD(pack_colour); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 214 | |
| 215 | static midgard_instruction |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 216 | v_branch(bool conditional, bool invert) |
| 217 | { |
| 218 | midgard_instruction ins = { |
| 219 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 220 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 221 | .compact_branch = true, |
| 222 | .branch = { |
| 223 | .conditional = conditional, |
| 224 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 225 | }, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 226 | .dest = ~0, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 227 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 228 | }; |
| 229 | |
| 230 | return ins; |
| 231 | } |
| 232 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 233 | static midgard_branch_extended |
| 234 | midgard_create_branch_extended( midgard_condition cond, |
| 235 | midgard_jmp_writeout_op op, |
| 236 | unsigned dest_tag, |
| 237 | signed quadword_offset) |
| 238 | { |
Alyssa Rosenzweig | 13ee87c | 2019-07-29 09:15:32 -0700 | [diff] [blame] | 239 | /* The condition code is actually a LUT describing a function to |
| 240 | * combine multiple condition codes. However, we only support a single |
| 241 | * condition code at the moment, so we just duplicate over a bunch of |
| 242 | * times. */ |
| 243 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 244 | uint16_t duplicated_cond = |
| 245 | (cond << 14) | |
| 246 | (cond << 12) | |
| 247 | (cond << 10) | |
| 248 | (cond << 8) | |
| 249 | (cond << 6) | |
| 250 | (cond << 4) | |
| 251 | (cond << 2) | |
| 252 | (cond << 0); |
| 253 | |
| 254 | midgard_branch_extended branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 255 | .op = op, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 256 | .dest_tag = dest_tag, |
| 257 | .offset = quadword_offset, |
| 258 | .cond = duplicated_cond |
| 259 | }; |
| 260 | |
| 261 | return branch; |
| 262 | } |
| 263 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 264 | static void |
| 265 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 266 | { |
| 267 | ins->has_constants = true; |
| 268 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 269 | } |
| 270 | |
| 271 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 272 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 273 | { |
| 274 | return glsl_count_attribute_slots(type, false); |
| 275 | } |
| 276 | |
| 277 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
| 278 | static void |
| 279 | midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu) |
| 280 | { |
| 281 | if (alu->op != nir_op_fdot2) |
| 282 | return; |
| 283 | |
| 284 | b->cursor = nir_before_instr(&alu->instr); |
| 285 | |
| 286 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 287 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 288 | |
| 289 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 290 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 291 | nir_ssa_def *sum = nir_fadd(b, |
| 292 | nir_channel(b, product, 0), |
| 293 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 294 | |
| 295 | /* Replace the fdot2 with this sum */ |
| 296 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
| 297 | } |
| 298 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 299 | static int |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 300 | midgard_sysval_for_ssbo(nir_intrinsic_instr *instr) |
| 301 | { |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 302 | /* This is way too meta */ |
| 303 | bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo; |
| 304 | unsigned idx_idx = is_store ? 1 : 0; |
| 305 | |
| 306 | nir_src index = instr->src[idx_idx]; |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 307 | assert(nir_src_is_const(index)); |
| 308 | uint32_t uindex = nir_src_as_uint(index); |
| 309 | |
| 310 | return PAN_SYSVAL(SSBO, uindex); |
| 311 | } |
| 312 | |
| 313 | static int |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 314 | midgard_sysval_for_sampler(nir_intrinsic_instr *instr) |
| 315 | { |
| 316 | /* TODO: indirect samplers !!! */ |
| 317 | nir_src index = instr->src[0]; |
| 318 | assert(nir_src_is_const(index)); |
| 319 | uint32_t uindex = nir_src_as_uint(index); |
| 320 | |
| 321 | return PAN_SYSVAL(SAMPLER, uindex); |
| 322 | } |
| 323 | |
| 324 | static int |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 325 | midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr) |
| 326 | { |
| 327 | switch (instr->intrinsic) { |
| 328 | case nir_intrinsic_load_viewport_scale: |
| 329 | return PAN_SYSVAL_VIEWPORT_SCALE; |
| 330 | case nir_intrinsic_load_viewport_offset: |
| 331 | return PAN_SYSVAL_VIEWPORT_OFFSET; |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 332 | case nir_intrinsic_load_num_work_groups: |
| 333 | return PAN_SYSVAL_NUM_WORK_GROUPS; |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 334 | case nir_intrinsic_load_ssbo: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 335 | case nir_intrinsic_store_ssbo: |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 336 | return midgard_sysval_for_ssbo(instr); |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 337 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
| 338 | return midgard_sysval_for_sampler(instr); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 339 | default: |
Alyssa Rosenzweig | 6189274 | 2019-08-21 09:15:56 -0700 | [diff] [blame] | 340 | return ~0; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 341 | } |
| 342 | } |
| 343 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 344 | static int sysval_for_instr(compiler_context *ctx, nir_instr *instr, |
| 345 | unsigned *dest) |
| 346 | { |
| 347 | nir_intrinsic_instr *intr; |
| 348 | nir_dest *dst = NULL; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 349 | nir_tex_instr *tex; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 350 | int sysval = -1; |
| 351 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 352 | bool is_store = false; |
| 353 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 354 | switch (instr->type) { |
| 355 | case nir_instr_type_intrinsic: |
| 356 | intr = nir_instr_as_intrinsic(instr); |
| 357 | sysval = midgard_nir_sysval_for_intrinsic(intr); |
| 358 | dst = &intr->dest; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 359 | is_store |= intr->intrinsic == nir_intrinsic_store_ssbo; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 360 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 361 | case nir_instr_type_tex: |
| 362 | tex = nir_instr_as_tex(instr); |
| 363 | if (tex->op != nir_texop_txs) |
| 364 | break; |
| 365 | |
| 366 | sysval = PAN_SYSVAL(TEXTURE_SIZE, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 367 | PAN_TXS_SYSVAL_ID(tex->texture_index, |
| 368 | nir_tex_instr_dest_size(tex) - |
| 369 | (tex->is_array ? 1 : 0), |
| 370 | tex->is_array)); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 371 | dst = &tex->dest; |
| 372 | break; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 373 | default: |
| 374 | break; |
| 375 | } |
| 376 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 377 | if (dest && dst && !is_store) |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 378 | *dest = nir_dest_index(ctx, dst); |
| 379 | |
| 380 | return sysval; |
| 381 | } |
| 382 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 383 | static void |
| 384 | midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr) |
| 385 | { |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 386 | int sysval; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 387 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 388 | sysval = sysval_for_instr(ctx, instr, NULL); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 389 | if (sysval < 0) |
| 390 | return; |
| 391 | |
| 392 | /* We have a sysval load; check if it's already been assigned */ |
| 393 | |
| 394 | if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval)) |
| 395 | return; |
| 396 | |
| 397 | /* It hasn't -- so assign it now! */ |
| 398 | |
| 399 | unsigned id = ctx->sysval_count++; |
| 400 | _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1)); |
| 401 | ctx->sysvals[id] = sysval; |
| 402 | } |
| 403 | |
| 404 | static void |
| 405 | midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader) |
| 406 | { |
| 407 | ctx->sysval_count = 0; |
| 408 | |
| 409 | nir_foreach_function(function, shader) { |
| 410 | if (!function->impl) continue; |
| 411 | |
| 412 | nir_foreach_block(block, function->impl) { |
| 413 | nir_foreach_instr_safe(instr, block) { |
| 414 | midgard_nir_assign_sysval_body(ctx, instr); |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 420 | static bool |
| 421 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 422 | { |
| 423 | bool progress = false; |
| 424 | |
| 425 | nir_foreach_function(function, shader) { |
| 426 | if (!function->impl) continue; |
| 427 | |
| 428 | nir_builder _b; |
| 429 | nir_builder *b = &_b; |
| 430 | nir_builder_init(b, function->impl); |
| 431 | |
| 432 | nir_foreach_block(block, function->impl) { |
| 433 | nir_foreach_instr_safe(instr, block) { |
| 434 | if (instr->type != nir_instr_type_alu) continue; |
| 435 | |
| 436 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 437 | midgard_nir_lower_fdot2_body(b, alu); |
| 438 | |
| 439 | progress |= true; |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 444 | |
| 445 | } |
| 446 | |
| 447 | return progress; |
| 448 | } |
| 449 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 450 | /* Flushes undefined values to zero */ |
| 451 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 452 | static void |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 453 | optimise_nir(nir_shader *nir, unsigned quirks) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 454 | { |
| 455 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 456 | unsigned lower_flrp = |
| 457 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 458 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 459 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 460 | |
| 461 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
Rhys Perry | 8b98d09 | 2019-02-05 15:56:24 +0000 | [diff] [blame] | 462 | NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 463 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 464 | nir_lower_tex_options lower_tex_options = { |
| 465 | .lower_txs_lod = true, |
Alyssa Rosenzweig | 4c43b35 | 2019-11-21 13:40:00 -0500 | [diff] [blame] | 466 | .lower_txp = ~0, |
| 467 | .lower_tex_without_implicit_lod = |
| 468 | (quirks & MIDGARD_EXPLICIT_LOD), |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 469 | |
| 470 | /* TODO: we have native gradient.. */ |
| 471 | .lower_txd = true, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 472 | }; |
| 473 | |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 474 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 475 | |
Alyssa Rosenzweig | c57337b | 2019-12-19 11:12:50 -0500 | [diff] [blame] | 476 | /* Must lower fdot2 after tex is lowered */ |
| 477 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
| 478 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 479 | /* T720 is broken. */ |
| 480 | |
| 481 | if (quirks & MIDGARD_BROKEN_LOD) |
| 482 | NIR_PASS_V(nir, midgard_nir_lod_errata); |
| 483 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 484 | do { |
| 485 | progress = false; |
| 486 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 487 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 488 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 489 | |
| 490 | NIR_PASS(progress, nir, nir_copy_prop); |
Boris Brezillon | 440b0d6 | 2020-01-06 14:31:38 +0100 | [diff] [blame] | 491 | NIR_PASS(progress, nir, nir_opt_remove_phis); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 492 | NIR_PASS(progress, nir, nir_opt_dce); |
| 493 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 494 | NIR_PASS(progress, nir, nir_opt_cse); |
| 495 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 496 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 497 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 498 | |
| 499 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 500 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 501 | NIR_PASS(lower_flrp_progress, |
| 502 | nir, |
| 503 | nir_lower_flrp, |
| 504 | lower_flrp, |
| 505 | false /* always_precise */, |
| 506 | nir->options->lower_ffma); |
| 507 | if (lower_flrp_progress) { |
| 508 | NIR_PASS(progress, nir, |
| 509 | nir_opt_constant_folding); |
| 510 | progress = true; |
| 511 | } |
| 512 | |
| 513 | /* Nothing should rematerialize any flrps, so we only |
| 514 | * need to do this lowering once. |
| 515 | */ |
| 516 | lower_flrp = 0; |
| 517 | } |
| 518 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 519 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 520 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 521 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 522 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 523 | nir_var_shader_in | |
| 524 | nir_var_shader_out | |
| 525 | nir_var_function_temp); |
| 526 | |
Alyssa Rosenzweig | 9402970 | 2019-06-17 11:12:51 -0700 | [diff] [blame] | 527 | NIR_PASS(progress, nir, nir_opt_vectorize); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 528 | } while (progress); |
| 529 | |
| 530 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 531 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 532 | |
| 533 | do { |
| 534 | progress = false; |
| 535 | |
| 536 | NIR_PASS(progress, nir, nir_opt_dce); |
| 537 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 538 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 539 | NIR_PASS(progress, nir, nir_copy_prop); |
| 540 | } while (progress); |
| 541 | |
| 542 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 543 | |
| 544 | /* We implement booleans as 32-bit 0/~0 */ |
| 545 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 546 | |
| 547 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 548 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 549 | |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 550 | /* Lower mods for float ops only. Integer ops don't support modifiers |
| 551 | * (saturate doesn't make sense on integers, neg/abs require dedicated |
| 552 | * instructions) */ |
| 553 | |
| 554 | NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 555 | NIR_PASS(progress, nir, nir_copy_prop); |
| 556 | NIR_PASS(progress, nir, nir_opt_dce); |
| 557 | |
| 558 | /* Take us out of SSA */ |
| 559 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 560 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 561 | |
| 562 | /* We are a vector architecture; write combine where possible */ |
| 563 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 564 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 565 | |
| 566 | NIR_PASS(progress, nir, nir_opt_dce); |
| 567 | } |
| 568 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 569 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 570 | |
| 571 | static void |
| 572 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 573 | { |
| 574 | nir_ssa_def def = instr->def; |
| 575 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 576 | midgard_constants *consts = rzalloc(NULL, midgard_constants); |
| 577 | |
| 578 | assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8); |
| 579 | |
| 580 | #define RAW_CONST_COPY(bits) \ |
| 581 | nir_const_value_to_array(consts->u##bits, instr->value, \ |
| 582 | instr->def.num_components, u##bits) |
| 583 | |
| 584 | switch (instr->def.bit_size) { |
| 585 | case 64: |
| 586 | RAW_CONST_COPY(64); |
| 587 | break; |
| 588 | case 32: |
| 589 | RAW_CONST_COPY(32); |
| 590 | break; |
| 591 | case 16: |
| 592 | RAW_CONST_COPY(16); |
| 593 | break; |
| 594 | case 8: |
| 595 | RAW_CONST_COPY(8); |
| 596 | break; |
| 597 | default: |
| 598 | unreachable("Invalid bit_size for load_const instruction\n"); |
| 599 | } |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 600 | |
| 601 | /* Shifted for SSA, +1 for off-by-one */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 602 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 603 | } |
| 604 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 605 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 606 | * explicitly emit a move with the constant source */ |
| 607 | |
| 608 | static void |
| 609 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 610 | { |
| 611 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 612 | |
| 613 | if (constant_value) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 614 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), to); |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 615 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 616 | emit_mir_instruction(ctx, ins); |
| 617 | } |
| 618 | } |
| 619 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 620 | static bool |
| 621 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 622 | { |
| 623 | unsigned comp = src->swizzle[0]; |
| 624 | |
| 625 | for (unsigned c = 1; c < nr_components; ++c) { |
| 626 | if (src->swizzle[c] != comp) |
| 627 | return true; |
| 628 | } |
| 629 | |
| 630 | return false; |
| 631 | } |
| 632 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 633 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 634 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 635 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 636 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 637 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 638 | |
| 639 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 640 | case nir_op_##nir: \ |
| 641 | op = midgard_alu_op_##_op; \ |
| 642 | broadcast_swizzle = count; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 643 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 644 | break; |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 645 | static bool |
| 646 | nir_is_fzero_constant(nir_src src) |
| 647 | { |
| 648 | if (!nir_src_is_const(src)) |
| 649 | return false; |
| 650 | |
| 651 | for (unsigned c = 0; c < nir_src_num_components(src); ++c) { |
| 652 | if (nir_src_comp_as_float(src, c) != 0.0) |
| 653 | return false; |
| 654 | } |
| 655 | |
| 656 | return true; |
| 657 | } |
| 658 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 659 | /* Analyze the sizes of the inputs to determine which reg mode. Ops needed |
| 660 | * special treatment override this anyway. */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 661 | |
| 662 | static midgard_reg_mode |
| 663 | reg_mode_for_nir(nir_alu_instr *instr) |
| 664 | { |
| 665 | unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 666 | |
| 667 | switch (src_bitsize) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 668 | case 8: |
| 669 | return midgard_reg_mode_8; |
| 670 | case 16: |
| 671 | return midgard_reg_mode_16; |
| 672 | case 32: |
| 673 | return midgard_reg_mode_32; |
| 674 | case 64: |
| 675 | return midgard_reg_mode_64; |
| 676 | default: |
| 677 | unreachable("Invalid bit size"); |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 678 | } |
| 679 | } |
| 680 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 681 | static void |
| 682 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 683 | { |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 684 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 685 | * is handled elsewhere */ |
| 686 | |
| 687 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 688 | midgard_emit_derivatives(ctx, instr); |
| 689 | return; |
| 690 | } |
| 691 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 692 | bool is_ssa = instr->dest.dest.is_ssa; |
| 693 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 694 | unsigned dest = nir_dest_index(ctx, &instr->dest.dest); |
Alyssa Rosenzweig | f42e5be | 2019-07-01 15:28:37 -0700 | [diff] [blame] | 695 | unsigned nr_components = nir_dest_num_components(instr->dest.dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 696 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 697 | |
| 698 | /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are |
| 699 | * supported. A few do not and are commented for now. Also, there are a |
| 700 | * number of NIR ops which Midgard does not support and need to be |
| 701 | * lowered, also TODO. This switch block emits the opcode and calling |
| 702 | * convention of the Midgard instruction; actual packing is done in |
| 703 | * emit_alu below */ |
| 704 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 705 | unsigned op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 706 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 707 | /* Number of components valid to check for the instruction (the rest |
| 708 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 709 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 710 | * in Midgard */ |
| 711 | |
| 712 | unsigned broadcast_swizzle = 0; |
| 713 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 714 | /* What register mode should we operate in? */ |
| 715 | midgard_reg_mode reg_mode = |
| 716 | reg_mode_for_nir(instr); |
| 717 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 718 | /* Do we need a destination override? Used for inline |
| 719 | * type conversion */ |
| 720 | |
| 721 | midgard_dest_override dest_override = |
| 722 | midgard_dest_override_none; |
| 723 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 724 | /* Should we use a smaller respective source and sign-extend? */ |
| 725 | |
| 726 | bool half_1 = false, sext_1 = false; |
| 727 | bool half_2 = false, sext_2 = false; |
| 728 | |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 729 | unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 730 | unsigned dst_bitsize = nir_dest_bit_size(instr->dest.dest); |
| 731 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 732 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 733 | ALU_CASE(fadd, fadd); |
| 734 | ALU_CASE(fmul, fmul); |
| 735 | ALU_CASE(fmin, fmin); |
| 736 | ALU_CASE(fmax, fmax); |
| 737 | ALU_CASE(imin, imin); |
| 738 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 739 | ALU_CASE(umin, umin); |
| 740 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 741 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 742 | ALU_CASE(fround_even, froundeven); |
| 743 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 744 | ALU_CASE(fceil, fceil); |
| 745 | ALU_CASE(fdot3, fdot3); |
| 746 | ALU_CASE(fdot4, fdot4); |
| 747 | ALU_CASE(iadd, iadd); |
| 748 | ALU_CASE(isub, isub); |
| 749 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 750 | |
| 751 | /* Zero shoved as second-arg */ |
| 752 | ALU_CASE(iabs, iabsdiff); |
| 753 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 754 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 755 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 756 | ALU_CASE(feq32, feq); |
| 757 | ALU_CASE(fne32, fne); |
| 758 | ALU_CASE(flt32, flt); |
| 759 | ALU_CASE(ieq32, ieq); |
| 760 | ALU_CASE(ine32, ine); |
| 761 | ALU_CASE(ilt32, ilt); |
Alyssa Rosenzweig | b8739c2 | 2019-03-26 04:00:33 +0000 | [diff] [blame] | 762 | ALU_CASE(ult32, ult); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 763 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 764 | /* We don't have a native b2f32 instruction. Instead, like many |
| 765 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 766 | * correspondingly AND |
| 767 | * by 1.0 to do the type conversion. For the moment, prime us |
| 768 | * to emit: |
| 769 | * |
| 770 | * iand [whatever], #0 |
| 771 | * |
| 772 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 773 | */ |
| 774 | |
| 775 | ALU_CASE(b2f32, iand); |
| 776 | ALU_CASE(b2i32, iand); |
| 777 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 778 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 779 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 780 | |
| 781 | ALU_CASE(f2b32, fne); |
Alyssa Rosenzweig | 5b95fef | 2019-03-25 00:56:48 +0000 | [diff] [blame] | 782 | ALU_CASE(i2b32, ine); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 783 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 784 | ALU_CASE(frcp, frcp); |
| 785 | ALU_CASE(frsq, frsqrt); |
| 786 | ALU_CASE(fsqrt, fsqrt); |
| 787 | ALU_CASE(fexp2, fexp2); |
| 788 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 789 | |
Boris Brezillon | fcceeaf | 2020-01-20 22:05:14 +0100 | [diff] [blame^] | 790 | ALU_CASE(f2i64, f2i_rtz); |
| 791 | ALU_CASE(f2u64, f2u_rtz); |
| 792 | ALU_CASE(i2f64, i2f_rtz); |
| 793 | ALU_CASE(u2f64, u2f_rtz); |
| 794 | |
Alyssa Rosenzweig | 73bf669 | 2019-06-05 15:03:02 -0700 | [diff] [blame] | 795 | ALU_CASE(f2i32, f2i_rtz); |
| 796 | ALU_CASE(f2u32, f2u_rtz); |
| 797 | ALU_CASE(i2f32, i2f_rtz); |
| 798 | ALU_CASE(u2f32, u2f_rtz); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 799 | |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 800 | ALU_CASE(f2i16, f2i_rtz); |
| 801 | ALU_CASE(f2u16, f2u_rtz); |
| 802 | ALU_CASE(i2f16, i2f_rtz); |
| 803 | ALU_CASE(u2f16, u2f_rtz); |
| 804 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 805 | ALU_CASE(fsin, fsin); |
| 806 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 807 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 808 | /* We'll set invert */ |
| 809 | ALU_CASE(inot, imov); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 810 | ALU_CASE(iand, iand); |
| 811 | ALU_CASE(ior, ior); |
| 812 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 813 | ALU_CASE(ishl, ishl); |
| 814 | ALU_CASE(ishr, iasr); |
| 815 | ALU_CASE(ushr, ilsr); |
| 816 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 817 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 818 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 819 | ALU_CASE(b32all_fequal4, fball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 820 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 821 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 822 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 823 | ALU_CASE(b32any_fnequal4, fbany_neq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 824 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 825 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 826 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 827 | ALU_CASE(b32all_iequal4, iball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 828 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 829 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 830 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 831 | ALU_CASE(b32any_inequal4, ibany_neq); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 832 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 833 | /* Source mods will be shoved in later */ |
| 834 | ALU_CASE(fabs, fmov); |
| 835 | ALU_CASE(fneg, fmov); |
| 836 | ALU_CASE(fsat, fmov); |
| 837 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 838 | /* For size conversion, we use a move. Ideally though we would squash |
| 839 | * these ops together; maybe that has to happen after in NIR as part of |
| 840 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 841 | * only / exactly one size. If stepping down, we use a dest override to |
| 842 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 843 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 844 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 845 | case nir_op_i2i8: |
| 846 | case nir_op_i2i16: |
| 847 | case nir_op_i2i32: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 848 | case nir_op_i2i64: |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 849 | /* If we end up upscale, we'll need a sign-extend on the |
| 850 | * operand (the second argument) */ |
| 851 | |
| 852 | sext_2 = true; |
Alyssa Rosenzweig | 14a2032f | 2019-08-21 09:20:17 -0700 | [diff] [blame] | 853 | /* fallthrough */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 854 | case nir_op_u2u8: |
| 855 | case nir_op_u2u16: |
Alyssa Rosenzweig | 2655a30 | 2019-11-04 22:21:20 -0500 | [diff] [blame] | 856 | case nir_op_u2u32: |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 857 | case nir_op_u2u64: |
| 858 | case nir_op_f2f16: |
Boris Brezillon | e1f9e8d | 2020-01-20 16:05:31 +0100 | [diff] [blame] | 859 | case nir_op_f2f32: |
| 860 | case nir_op_f2f64: { |
| 861 | if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 || |
| 862 | instr->op == nir_op_f2f64) |
Boris Brezillon | f53a079 | 2020-01-20 16:03:52 +0100 | [diff] [blame] | 863 | op = midgard_alu_op_fmov; |
| 864 | else |
| 865 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 866 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 867 | if (dst_bitsize == (src_bitsize * 2)) { |
| 868 | /* Converting up */ |
| 869 | half_2 = true; |
| 870 | |
| 871 | /* Use a greater register mode */ |
| 872 | reg_mode++; |
| 873 | } else if (src_bitsize == (dst_bitsize * 2)) { |
| 874 | /* Converting down */ |
| 875 | dest_override = midgard_dest_override_lower; |
| 876 | } |
| 877 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 878 | break; |
| 879 | } |
| 880 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 881 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 882 | * arguments */ |
| 883 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 884 | case nir_op_fge: |
| 885 | case nir_op_fge32: |
| 886 | case nir_op_ige32: |
| 887 | case nir_op_uge32: { |
| 888 | op = |
| 889 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 890 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 891 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 892 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 893 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 894 | |
| 895 | /* Swap via temporary */ |
| 896 | nir_alu_src temp = instr->src[1]; |
| 897 | instr->src[1] = instr->src[0]; |
| 898 | instr->src[0] = temp; |
| 899 | |
| 900 | break; |
| 901 | } |
| 902 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 903 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 904 | /* Midgard features both fcsel and icsel, depending on |
| 905 | * the type of the arguments/output. However, as long |
| 906 | * as we're careful we can _always_ use icsel and |
| 907 | * _never_ need fcsel, since the latter does additional |
| 908 | * floating-point-specific processing whereas the |
| 909 | * former just moves bits on the wire. It's not obvious |
| 910 | * why these are separate opcodes, save for the ability |
| 911 | * to do things like sat/pos/abs/neg for free */ |
Alyssa Rosenzweig | 3d7874c | 2019-05-03 01:54:16 +0000 | [diff] [blame] | 912 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 913 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
| 914 | op = mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 915 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 916 | /* The condition is the first argument; move the other |
| 917 | * arguments up one to be a binary instruction for |
Alyssa Rosenzweig | d3b3daa | 2019-09-23 08:00:51 -0400 | [diff] [blame] | 918 | * Midgard with the condition last */ |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 919 | |
Alyssa Rosenzweig | d3b3daa | 2019-09-23 08:00:51 -0400 | [diff] [blame] | 920 | nir_alu_src temp = instr->src[2]; |
| 921 | |
| 922 | instr->src[2] = instr->src[0]; |
| 923 | instr->src[0] = instr->src[1]; |
| 924 | instr->src[1] = temp; |
| 925 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 926 | break; |
| 927 | } |
| 928 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 929 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 930 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 931 | assert(0); |
| 932 | return; |
| 933 | } |
| 934 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 935 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 936 | unsigned outmod; |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 937 | |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 938 | if (midgard_is_integer_out_op(op)) { |
| 939 | outmod = midgard_outmod_int_wrap; |
| 940 | } else { |
| 941 | bool sat = instr->dest.saturate || instr->op == nir_op_fsat; |
| 942 | outmod = sat ? midgard_outmod_sat : midgard_outmod_none; |
| 943 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 944 | |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 945 | /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */ |
| 946 | |
| 947 | if (instr->op == nir_op_fmax) { |
| 948 | if (nir_is_fzero_constant(instr->src[0].src)) { |
| 949 | op = midgard_alu_op_fmov; |
| 950 | nr_inputs = 1; |
| 951 | outmod = midgard_outmod_pos; |
| 952 | instr->src[0] = instr->src[1]; |
| 953 | } else if (nir_is_fzero_constant(instr->src[1].src)) { |
| 954 | op = midgard_alu_op_fmov; |
| 955 | nr_inputs = 1; |
| 956 | outmod = midgard_outmod_pos; |
| 957 | } |
| 958 | } |
| 959 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 960 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 961 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 962 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 963 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 964 | /* src0 will always exist afaik, but src1 will not for 1-argument |
| 965 | * instructions. The latter can only be fetched if the instruction |
| 966 | * needs it, or else we may segfault. */ |
| 967 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 968 | unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 8e36996 | 2019-08-30 10:42:05 -0700 | [diff] [blame] | 969 | unsigned src1 = nr_inputs >= 2 ? nir_alu_src_index(ctx, &instr->src[1]) : ~0; |
| 970 | unsigned src2 = nr_inputs == 3 ? nir_alu_src_index(ctx, &instr->src[2]) : ~0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 971 | assert(nr_inputs <= 3); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 972 | |
| 973 | /* Rather than use the instruction generation helpers, we do it |
| 974 | * ourselves here to avoid the mess */ |
| 975 | |
| 976 | midgard_instruction ins = { |
| 977 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 978 | .src = { |
| 979 | quirk_flipped_r24 ? ~0 : src0, |
| 980 | quirk_flipped_r24 ? src0 : src1, |
Alyssa Rosenzweig | 8e36996 | 2019-08-30 10:42:05 -0700 | [diff] [blame] | 981 | src2, |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 982 | ~0 |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 983 | }, |
| 984 | .dest = dest, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 985 | }; |
| 986 | |
Alyssa Rosenzweig | 8e36996 | 2019-08-30 10:42:05 -0700 | [diff] [blame] | 987 | nir_alu_src *nirmods[3] = { NULL }; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 988 | |
Alyssa Rosenzweig | 8e36996 | 2019-08-30 10:42:05 -0700 | [diff] [blame] | 989 | if (nr_inputs >= 2) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 990 | nirmods[0] = &instr->src[0]; |
| 991 | nirmods[1] = &instr->src[1]; |
| 992 | } else if (nr_inputs == 1) { |
| 993 | nirmods[quirk_flipped_r24] = &instr->src[0]; |
| 994 | } else { |
| 995 | assert(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 996 | } |
| 997 | |
Alyssa Rosenzweig | 8e36996 | 2019-08-30 10:42:05 -0700 | [diff] [blame] | 998 | if (nr_inputs == 3) |
| 999 | nirmods[2] = &instr->src[2]; |
| 1000 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1001 | /* These were lowered to a move, so apply the corresponding mod */ |
| 1002 | |
| 1003 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
| 1004 | nir_alu_src *s = nirmods[quirk_flipped_r24]; |
| 1005 | |
| 1006 | if (instr->op == nir_op_fneg) |
| 1007 | s->negate = !s->negate; |
| 1008 | |
| 1009 | if (instr->op == nir_op_fabs) |
| 1010 | s->abs = !s->abs; |
| 1011 | } |
| 1012 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 1013 | bool is_int = midgard_is_integer_op(op); |
| 1014 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1015 | ins.mask = mask_of(nr_components); |
| 1016 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1017 | midgard_vector_alu alu = { |
| 1018 | .op = op, |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1019 | .reg_mode = reg_mode, |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1020 | .dest_override = dest_override, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1021 | .outmod = outmod, |
| 1022 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1023 | .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, sext_1)), |
| 1024 | .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int, broadcast_swizzle, half_2, sext_2)), |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1025 | }; |
| 1026 | |
| 1027 | /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */ |
| 1028 | |
| 1029 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1030 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1031 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1032 | for (unsigned m = 0; m < 3; ++m) { |
| 1033 | if (!nirmods[m]) |
| 1034 | continue; |
| 1035 | |
| 1036 | for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) |
| 1037 | ins.swizzle[m][c] = nirmods[m]->swizzle[c]; |
| 1038 | |
| 1039 | /* Replicate. TODO: remove when vec16 lands */ |
| 1040 | for (unsigned c = NIR_MAX_VEC_COMPONENTS; c < MIR_VEC_COMPONENTS; ++c) |
| 1041 | ins.swizzle[m][c] = nirmods[m]->swizzle[NIR_MAX_VEC_COMPONENTS - 1]; |
| 1042 | } |
| 1043 | |
| 1044 | if (nr_inputs == 3) { |
| 1045 | /* Conditions can't have mods */ |
| 1046 | assert(!nirmods[2]->abs); |
| 1047 | assert(!nirmods[2]->negate); |
| 1048 | } |
| 1049 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1050 | ins.alu = alu; |
| 1051 | |
| 1052 | /* Late fixup for emulated instructions */ |
| 1053 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1054 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1055 | /* Presently, our second argument is an inline #0 constant. |
| 1056 | * Switch over to an embedded 1.0 constant (that can't fit |
| 1057 | * inline, since we're 32-bit, not 16-bit like the inline |
| 1058 | * constants) */ |
| 1059 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1060 | ins.has_inline_constant = false; |
| 1061 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1062 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1063 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1064 | if (instr->op == nir_op_b2f32) |
| 1065 | ins.constants.f32[0] = 1.0f; |
| 1066 | else |
| 1067 | ins.constants.i32[0] = 1; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1068 | |
| 1069 | for (unsigned c = 0; c < 16; ++c) |
| 1070 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 1071 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 1072 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1073 | ins.has_inline_constant = false; |
| 1074 | ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1075 | ins.has_constants = true; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1076 | ins.constants.u32[0] = 0; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1077 | |
| 1078 | for (unsigned c = 0; c < 16; ++c) |
| 1079 | ins.swizzle[1][c] = 0; |
Alyssa Rosenzweig | bcabcfe | 2019-04-25 04:25:33 +0000 | [diff] [blame] | 1080 | } else if (instr->op == nir_op_inot) { |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 1081 | ins.invert = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1082 | } |
| 1083 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1084 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 1085 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 1086 | * instructions can only operate as if they were scalars. Lower |
| 1087 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1088 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1089 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1090 | |
| 1091 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1092 | /* Mask the associated component, dropping the |
| 1093 | * instruction if needed */ |
| 1094 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1095 | ins.mask = 1 << i; |
| 1096 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1097 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1098 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1099 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1100 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1101 | for (unsigned j = 0; j < MIR_VEC_COMPONENTS; ++j) |
| 1102 | ins.swizzle[0][j] = nirmods[0]->swizzle[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1103 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1104 | emit_mir_instruction(ctx, ins); |
| 1105 | } |
| 1106 | } else { |
| 1107 | emit_mir_instruction(ctx, ins); |
| 1108 | } |
| 1109 | } |
| 1110 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1111 | #undef ALU_CASE |
| 1112 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1113 | static void |
| 1114 | mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read) |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1115 | { |
| 1116 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1117 | unsigned nir_mask = 0; |
| 1118 | unsigned dsize = 0; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1119 | |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1120 | if (is_read) { |
| 1121 | nir_mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1122 | dsize = nir_dest_bit_size(intr->dest); |
| 1123 | } else { |
| 1124 | nir_mask = nir_intrinsic_write_mask(intr); |
| 1125 | dsize = 32; |
| 1126 | } |
| 1127 | |
| 1128 | /* Once we have the NIR mask, we need to normalize to work in 32-bit space */ |
| 1129 | unsigned bytemask = mir_to_bytemask(mir_mode_for_destsize(dsize), nir_mask); |
| 1130 | mir_set_bytemask(ins, bytemask); |
| 1131 | |
| 1132 | if (dsize == 64) |
| 1133 | ins->load_64 = true; |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1134 | } |
| 1135 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1136 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1137 | * optimized) versions of UBO #0 */ |
| 1138 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1139 | static midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1140 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1141 | compiler_context *ctx, |
Alyssa Rosenzweig | 65e6cb4 | 2019-08-13 09:11:48 -0700 | [diff] [blame] | 1142 | nir_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1143 | unsigned dest, |
| 1144 | unsigned offset, |
| 1145 | nir_src *indirect_offset, |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1146 | unsigned indirect_shift, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1147 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1148 | { |
| 1149 | /* TODO: half-floats */ |
| 1150 | |
Alyssa Rosenzweig | bc9a7d0 | 2019-11-15 14:19:34 -0500 | [diff] [blame] | 1151 | midgard_instruction ins = m_ld_ubo_int4(dest, 0); |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1152 | ins.constants.u32[0] = offset; |
Alyssa Rosenzweig | da73651 | 2019-12-19 11:12:25 -0500 | [diff] [blame] | 1153 | |
| 1154 | if (instr->type == nir_instr_type_intrinsic) |
| 1155 | mir_set_intr_mask(instr, &ins, true); |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1156 | |
| 1157 | if (indirect_offset) { |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1158 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1159 | ins.load_store.arg_2 = (indirect_shift << 5); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1160 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1161 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1162 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1163 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1164 | ins.load_store.arg_1 = index; |
| 1165 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1166 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1167 | } |
| 1168 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1169 | /* SSBO reads are like UBO reads if you squint */ |
| 1170 | |
| 1171 | static void |
| 1172 | emit_ssbo_access( |
| 1173 | compiler_context *ctx, |
| 1174 | nir_instr *instr, |
| 1175 | bool is_read, |
| 1176 | unsigned srcdest, |
| 1177 | unsigned offset, |
| 1178 | nir_src *indirect_offset, |
| 1179 | unsigned index) |
| 1180 | { |
| 1181 | /* TODO: types */ |
| 1182 | |
| 1183 | midgard_instruction ins; |
| 1184 | |
| 1185 | if (is_read) |
| 1186 | ins = m_ld_int4(srcdest, offset); |
| 1187 | else |
| 1188 | ins = m_st_int4(srcdest, offset); |
| 1189 | |
| 1190 | /* SSBO reads use a generic memory read interface, so we need the |
| 1191 | * address of the SSBO as the first argument. This is a sysval. */ |
| 1192 | |
| 1193 | unsigned addr = make_compiler_temp(ctx); |
| 1194 | emit_sysval_read(ctx, instr, addr, 2); |
| 1195 | |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1196 | /* The source array: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1197 | * |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1198 | * src[0] = store ? value : unused |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1199 | * src[1] = arg_1 |
| 1200 | * src[2] = arg_2 |
| 1201 | * |
| 1202 | * We would like arg_1 = the address and |
| 1203 | * arg_2 = the offset. |
| 1204 | */ |
| 1205 | |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1206 | ins.src[1] = addr; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1207 | |
| 1208 | /* TODO: What is this? It looks superficially like a shift << 5, but |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1209 | * arg_1 doesn't take a shift Should it be E0 or A0? We also need the |
| 1210 | * indirect offset. */ |
| 1211 | |
| 1212 | if (indirect_offset) { |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1213 | ins.load_store.arg_1 |= 0xE0; |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1214 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1215 | } else { |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1216 | ins.load_store.arg_2 = 0x7E; |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1217 | } |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1218 | |
| 1219 | /* TODO: Bounds check */ |
| 1220 | |
| 1221 | /* Finally, we emit the direct offset */ |
| 1222 | |
| 1223 | ins.load_store.varying_parameters = (offset & 0x1FF) << 1; |
| 1224 | ins.load_store.address = (offset >> 9); |
Alyssa Rosenzweig | 1798f6b | 2019-11-15 15:16:53 -0500 | [diff] [blame] | 1225 | mir_set_intr_mask(instr, &ins, is_read); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1226 | |
| 1227 | emit_mir_instruction(ctx, ins); |
| 1228 | } |
| 1229 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1230 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1231 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1232 | compiler_context *ctx, |
| 1233 | unsigned dest, unsigned offset, |
| 1234 | unsigned nr_comp, unsigned component, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1235 | nir_src *indirect_offset, nir_alu_type type, bool flat) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1236 | { |
| 1237 | /* XXX: Half-floats? */ |
| 1238 | /* TODO: swizzle, mask */ |
| 1239 | |
| 1240 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1241 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1242 | |
| 1243 | for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) |
| 1244 | ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1245 | |
| 1246 | midgard_varying_parameter p = { |
| 1247 | .is_varying = 1, |
| 1248 | .interpolation = midgard_interp_default, |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1249 | .flat = flat, |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1250 | }; |
| 1251 | |
| 1252 | unsigned u; |
| 1253 | memcpy(&u, &p, sizeof(p)); |
| 1254 | ins.load_store.varying_parameters = u; |
| 1255 | |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1256 | if (indirect_offset) |
Alyssa Rosenzweig | e7fd14c | 2019-10-26 15:50:38 -0400 | [diff] [blame] | 1257 | ins.src[2] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1258 | else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1259 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1260 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1261 | ins.load_store.arg_1 = 0x9E; |
| 1262 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1263 | /* Use the type appropriate load */ |
| 1264 | switch (type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1265 | case nir_type_uint: |
| 1266 | case nir_type_bool: |
| 1267 | ins.load_store.op = midgard_op_ld_vary_32u; |
| 1268 | break; |
| 1269 | case nir_type_int: |
| 1270 | ins.load_store.op = midgard_op_ld_vary_32i; |
| 1271 | break; |
| 1272 | case nir_type_float: |
| 1273 | ins.load_store.op = midgard_op_ld_vary_32; |
| 1274 | break; |
| 1275 | default: |
| 1276 | unreachable("Attempted to load unknown type"); |
| 1277 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1278 | } |
| 1279 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1280 | emit_mir_instruction(ctx, ins); |
| 1281 | } |
| 1282 | |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1283 | static void |
| 1284 | emit_attr_read( |
| 1285 | compiler_context *ctx, |
| 1286 | unsigned dest, unsigned offset, |
| 1287 | unsigned nr_comp, nir_alu_type t) |
| 1288 | { |
| 1289 | midgard_instruction ins = m_ld_attr_32(dest, offset); |
| 1290 | ins.load_store.arg_1 = 0x1E; |
| 1291 | ins.load_store.arg_2 = 0x1E; |
| 1292 | ins.mask = mask_of(nr_comp); |
| 1293 | |
| 1294 | /* Use the type appropriate load */ |
| 1295 | switch (t) { |
| 1296 | case nir_type_uint: |
| 1297 | case nir_type_bool: |
| 1298 | ins.load_store.op = midgard_op_ld_attr_32u; |
| 1299 | break; |
| 1300 | case nir_type_int: |
| 1301 | ins.load_store.op = midgard_op_ld_attr_32i; |
| 1302 | break; |
| 1303 | case nir_type_float: |
| 1304 | ins.load_store.op = midgard_op_ld_attr_32; |
| 1305 | break; |
| 1306 | default: |
| 1307 | unreachable("Attempted to load unknown type"); |
| 1308 | break; |
| 1309 | } |
| 1310 | |
| 1311 | emit_mir_instruction(ctx, ins); |
| 1312 | } |
| 1313 | |
Alyssa Rosenzweig | fa68740 | 2019-08-02 11:06:21 -0700 | [diff] [blame] | 1314 | void |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1315 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, signed dest_override, |
| 1316 | unsigned nr_components) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1317 | { |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1318 | unsigned dest = 0; |
| 1319 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1320 | /* Figure out which uniform this is */ |
| 1321 | int sysval = sysval_for_instr(ctx, instr, &dest); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1322 | void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval); |
| 1323 | |
Alyssa Rosenzweig | fa68740 | 2019-08-02 11:06:21 -0700 | [diff] [blame] | 1324 | if (dest_override >= 0) |
| 1325 | dest = dest_override; |
| 1326 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1327 | /* Sysvals are prefix uniforms */ |
| 1328 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1329 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1330 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1331 | midgard_instruction *ins = |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1332 | emit_ubo_read(ctx, instr, dest, uniform * 16, NULL, 0, 0); |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1333 | |
| 1334 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1335 | } |
| 1336 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1337 | static unsigned |
| 1338 | compute_builtin_arg(nir_op op) |
| 1339 | { |
| 1340 | switch (op) { |
| 1341 | case nir_intrinsic_load_work_group_id: |
| 1342 | return 0x14; |
| 1343 | case nir_intrinsic_load_local_invocation_id: |
| 1344 | return 0x10; |
| 1345 | default: |
| 1346 | unreachable("Invalid compute paramater loaded"); |
| 1347 | } |
| 1348 | } |
| 1349 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1350 | static void |
| 1351 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned rt) |
| 1352 | { |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 1353 | emit_explicit_constant(ctx, src, src); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1354 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1355 | struct midgard_instruction ins = |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1356 | v_branch(false, false); |
| 1357 | |
| 1358 | ins.writeout = true; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1359 | |
| 1360 | /* Add dependencies */ |
Alyssa Rosenzweig | 7652983 | 2019-08-30 11:01:15 -0700 | [diff] [blame] | 1361 | ins.src[0] = src; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 1362 | ins.constants.u32[0] = rt * 0x100; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1363 | |
| 1364 | /* Emit the branch */ |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 1365 | midgard_instruction *br = emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 281cc6f | 2019-11-23 12:43:55 -0500 | [diff] [blame] | 1366 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 1367 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 1368 | assert(rt < ARRAY_SIZE(ctx->writeout_branch)); |
| 1369 | assert(!ctx->writeout_branch[rt]); |
| 1370 | ctx->writeout_branch[rt] = br; |
| 1371 | |
| 1372 | /* Push our current location = current block count - 1 = where we'll |
| 1373 | * jump to. Maybe a bit too clever for my own good */ |
| 1374 | |
| 1375 | br->branch.target_block = ctx->block_count - 1; |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1376 | } |
| 1377 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1378 | static void |
| 1379 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1380 | { |
| 1381 | unsigned reg = nir_dest_index(ctx, &instr->dest); |
| 1382 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1383 | ins.mask = mask_of(3); |
| 1384 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1385 | emit_mir_instruction(ctx, ins); |
| 1386 | } |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1387 | |
| 1388 | static unsigned |
| 1389 | vertex_builtin_arg(nir_op op) |
| 1390 | { |
| 1391 | switch (op) { |
| 1392 | case nir_intrinsic_load_vertex_id: |
| 1393 | return PAN_VERTEX_ID; |
| 1394 | case nir_intrinsic_load_instance_id: |
| 1395 | return PAN_INSTANCE_ID; |
| 1396 | default: |
| 1397 | unreachable("Invalid vertex builtin"); |
| 1398 | } |
| 1399 | } |
| 1400 | |
| 1401 | static void |
| 1402 | emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1403 | { |
| 1404 | unsigned reg = nir_dest_index(ctx, &instr->dest); |
| 1405 | emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int); |
| 1406 | } |
| 1407 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1408 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1409 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1410 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1411 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1412 | |
| 1413 | switch (instr->intrinsic) { |
| 1414 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1415 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1416 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1417 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1418 | discard.branch.target_type = TARGET_DISCARD; |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 1419 | |
| 1420 | if (conditional) |
| 1421 | discard.src[0] = nir_src_index(ctx, &instr->src[0]); |
| 1422 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1423 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1424 | schedule_barrier(ctx); |
| 1425 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1426 | break; |
| 1427 | } |
| 1428 | |
| 1429 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1430 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1431 | case nir_intrinsic_load_ssbo: |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1432 | case nir_intrinsic_load_input: |
| 1433 | case nir_intrinsic_load_interpolated_input: { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1434 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1435 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1436 | bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo; |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1437 | bool is_flat = instr->intrinsic == nir_intrinsic_load_input; |
| 1438 | bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1439 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1440 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1441 | /* TODO: Infer type? Does it matter? */ |
| 1442 | nir_alu_type t = |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1443 | (is_ubo || is_ssbo) ? nir_type_uint : |
| 1444 | (is_interp) ? nir_type_float : |
| 1445 | nir_intrinsic_type(instr); |
| 1446 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1447 | t = nir_alu_type_get_base_type(t); |
| 1448 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1449 | if (!(is_ubo || is_ssbo)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1450 | offset = nir_intrinsic_base(instr); |
| 1451 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1452 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1453 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1454 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1455 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1456 | |
| 1457 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1458 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1459 | |
| 1460 | if (direct) |
| 1461 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1462 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1463 | /* We may need to apply a fractional offset */ |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1464 | int component = (is_flat || is_interp) ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1465 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1466 | reg = nir_dest_index(ctx, &instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1467 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1468 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1469 | emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysval_count + offset) * 16, indirect_offset, 4, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1470 | } else if (is_ubo) { |
| 1471 | nir_src index = instr->src[0]; |
| 1472 | |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1473 | /* TODO: Is indirect block number possible? */ |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1474 | assert(nir_src_is_const(index)); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1475 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1476 | uint32_t uindex = nir_src_as_uint(index) + 1; |
Alyssa Rosenzweig | 59d30fd | 2020-01-10 17:47:57 -0500 | [diff] [blame] | 1477 | emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1478 | } else if (is_ssbo) { |
| 1479 | nir_src index = instr->src[0]; |
| 1480 | assert(nir_src_is_const(index)); |
| 1481 | uint32_t uindex = nir_src_as_uint(index); |
| 1482 | |
| 1483 | emit_ssbo_access(ctx, &instr->instr, true, reg, offset, indirect_offset, uindex); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1484 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1485 | emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t, is_flat); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1486 | } else if (ctx->is_blend) { |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1487 | /* For blend shaders, load the input color, which is |
| 1488 | * preloaded to r0 */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1489 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1490 | midgard_instruction move = v_mov(SSA_FIXED_REGISTER(0), reg); |
Alyssa Rosenzweig | 005d9b1 | 2019-05-20 00:46:48 +0000 | [diff] [blame] | 1491 | emit_mir_instruction(ctx, move); |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 1492 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 6e68890 | 2019-12-19 13:24:17 -0500 | [diff] [blame] | 1493 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
| 1494 | emit_attr_read(ctx, reg, offset, nr_comp, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1495 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1496 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1497 | assert(0); |
| 1498 | } |
| 1499 | |
| 1500 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1501 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1502 | |
Alyssa Rosenzweig | c17a441 | 2019-12-27 15:32:50 -0500 | [diff] [blame] | 1503 | /* Artefact of load_interpolated_input. TODO: other barycentric modes */ |
| 1504 | case nir_intrinsic_load_barycentric_pixel: |
| 1505 | break; |
| 1506 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1507 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1508 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1509 | case nir_intrinsic_load_raw_output_pan: |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1510 | case nir_intrinsic_load_output_u8_as_fp16_pan: |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1511 | reg = nir_dest_index(ctx, &instr->dest); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1512 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1513 | |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1514 | /* T720 and below use different blend opcodes with slightly |
| 1515 | * different semantics than T760 and up */ |
| 1516 | |
Alyssa Rosenzweig | 2d1e18e | 2020-01-02 12:28:54 -0500 | [diff] [blame] | 1517 | midgard_instruction ld = m_ld_color_buffer_32u(reg, 0); |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 1518 | bool old_blend = ctx->quirks & MIDGARD_OLD_BLEND; |
Alyssa Rosenzweig | 843874c | 2019-11-06 21:50:32 -0500 | [diff] [blame] | 1519 | |
| 1520 | if (instr->intrinsic == nir_intrinsic_load_output_u8_as_fp16_pan) { |
| 1521 | ld.load_store.op = old_blend ? |
| 1522 | midgard_op_ld_color_buffer_u8_as_fp16_old : |
| 1523 | midgard_op_ld_color_buffer_u8_as_fp16; |
| 1524 | |
| 1525 | if (old_blend) { |
| 1526 | ld.load_store.address = 1; |
| 1527 | ld.load_store.arg_2 = 0x1E; |
| 1528 | } |
| 1529 | |
| 1530 | for (unsigned c = 2; c < 16; ++c) |
| 1531 | ld.swizzle[0][c] = 0; |
| 1532 | } |
| 1533 | |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 1534 | emit_mir_instruction(ctx, ld); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1535 | break; |
| 1536 | |
| 1537 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1538 | assert(ctx->is_blend); |
| 1539 | reg = nir_dest_index(ctx, &instr->dest); |
| 1540 | |
| 1541 | /* Blend constants are embedded directly in the shader and |
| 1542 | * patched in, so we use some magic routing */ |
| 1543 | |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 1544 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1545 | ins.has_constants = true; |
| 1546 | ins.has_blend_constant = true; |
| 1547 | emit_mir_instruction(ctx, ins); |
| 1548 | break; |
| 1549 | } |
| 1550 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1551 | case nir_intrinsic_store_output: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1552 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1553 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1554 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1555 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1556 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1557 | |
| 1558 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1559 | emit_fragment_store(ctx, reg, offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1560 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1561 | /* We should have been vectorized, though we don't |
| 1562 | * currently check that st_vary is emitted only once |
| 1563 | * per slot (this is relevant, since there's not a mask |
| 1564 | * parameter available on the store [set to 0 by the |
| 1565 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1566 | * swizzle. If this is a constant source, we'll need to |
| 1567 | * emit that explicitly. */ |
| 1568 | |
| 1569 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1570 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1571 | unsigned dst_component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | 2788721 | 2019-08-15 16:53:03 -0700 | [diff] [blame] | 1572 | unsigned nr_comp = nir_src_num_components(instr->src[0]); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1573 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1574 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1575 | st.load_store.arg_1 = 0x9E; |
| 1576 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1577 | |
Alyssa Rosenzweig | 66c2696 | 2019-12-27 14:25:00 -0500 | [diff] [blame] | 1578 | switch (nir_alu_type_get_base_type(nir_intrinsic_type(instr))) { |
| 1579 | case nir_type_uint: |
| 1580 | case nir_type_bool: |
| 1581 | st.load_store.op = midgard_op_st_vary_32u; |
| 1582 | break; |
| 1583 | case nir_type_int: |
| 1584 | st.load_store.op = midgard_op_st_vary_32i; |
| 1585 | break; |
| 1586 | case nir_type_float: |
| 1587 | st.load_store.op = midgard_op_st_vary_32; |
| 1588 | break; |
| 1589 | default: |
| 1590 | unreachable("Attempted to store unknown type"); |
| 1591 | break; |
| 1592 | } |
| 1593 | |
Boris Brezillon | 6af63c9 | 2020-01-16 11:20:06 +0100 | [diff] [blame] | 1594 | /* nir_intrinsic_component(store_intr) encodes the |
| 1595 | * destination component start. Source component offset |
| 1596 | * adjustment is taken care of in |
| 1597 | * install_registers_instr(), when offset_swizzle() is |
| 1598 | * called. |
| 1599 | */ |
| 1600 | unsigned src_component = COMPONENT_X; |
| 1601 | |
| 1602 | assert(nr_comp > 0); |
| 1603 | for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) { |
| 1604 | st.swizzle[0][i] = src_component; |
| 1605 | if (i >= dst_component && i < dst_component + nr_comp - 1) |
| 1606 | src_component++; |
| 1607 | } |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1608 | |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1609 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1610 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1611 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1612 | assert(0); |
| 1613 | } |
| 1614 | |
| 1615 | break; |
| 1616 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1617 | /* Special case of store_output for lowered blend shaders */ |
| 1618 | case nir_intrinsic_store_raw_output_pan: |
| 1619 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 1620 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | 8555bff | 2019-11-26 08:48:33 -0500 | [diff] [blame] | 1621 | |
| 1622 | if (ctx->quirks & MIDGARD_OLD_BLEND) { |
| 1623 | /* Suppose reg = qr0.xyzw. That means 4 8-bit ---> 1 32-bit. So |
| 1624 | * reg = r0.x. We want to splatter. So we can do a 32-bit move |
| 1625 | * of: |
| 1626 | * |
| 1627 | * imov r0.xyzw, r0.xxxx |
| 1628 | */ |
| 1629 | |
| 1630 | unsigned expanded = make_compiler_temp(ctx); |
| 1631 | |
| 1632 | midgard_instruction splatter = v_mov(reg, expanded); |
| 1633 | |
| 1634 | for (unsigned c = 0; c < 16; ++c) |
| 1635 | splatter.swizzle[1][c] = 0; |
| 1636 | |
| 1637 | emit_mir_instruction(ctx, splatter); |
Alyssa Rosenzweig | a2d5503 | 2019-11-23 21:44:16 -0500 | [diff] [blame] | 1638 | emit_fragment_store(ctx, expanded, ctx->blend_rt); |
Alyssa Rosenzweig | 8555bff | 2019-11-26 08:48:33 -0500 | [diff] [blame] | 1639 | } else |
Alyssa Rosenzweig | a2d5503 | 2019-11-23 21:44:16 -0500 | [diff] [blame] | 1640 | emit_fragment_store(ctx, reg, ctx->blend_rt); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1641 | |
| 1642 | break; |
| 1643 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1644 | case nir_intrinsic_store_ssbo: |
| 1645 | assert(nir_src_is_const(instr->src[1])); |
| 1646 | |
| 1647 | bool direct_offset = nir_src_is_const(instr->src[2]); |
| 1648 | offset = direct_offset ? nir_src_as_uint(instr->src[2]) : 0; |
| 1649 | nir_src *indirect_offset = direct_offset ? NULL : &instr->src[2]; |
| 1650 | reg = nir_src_index(ctx, &instr->src[0]); |
| 1651 | |
| 1652 | uint32_t uindex = nir_src_as_uint(instr->src[1]); |
| 1653 | |
| 1654 | emit_explicit_constant(ctx, reg, reg); |
| 1655 | emit_ssbo_access(ctx, &instr->instr, false, reg, offset, indirect_offset, uindex); |
| 1656 | break; |
| 1657 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1658 | case nir_intrinsic_load_viewport_scale: |
| 1659 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 1660 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 4e07e7b | 2019-11-21 08:42:28 -0500 | [diff] [blame] | 1661 | case nir_intrinsic_load_sampler_lod_parameters_pan: |
Alyssa Rosenzweig | 6189274 | 2019-08-21 09:15:56 -0700 | [diff] [blame] | 1662 | emit_sysval_read(ctx, &instr->instr, ~0, 3); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1663 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1664 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1665 | case nir_intrinsic_load_work_group_id: |
| 1666 | case nir_intrinsic_load_local_invocation_id: |
| 1667 | emit_compute_builtin(ctx, instr); |
| 1668 | break; |
| 1669 | |
Alyssa Rosenzweig | 306800d | 2019-12-19 13:31:21 -0500 | [diff] [blame] | 1670 | case nir_intrinsic_load_vertex_id: |
| 1671 | case nir_intrinsic_load_instance_id: |
| 1672 | emit_vertex_builtin(ctx, instr); |
| 1673 | break; |
| 1674 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1675 | default: |
| 1676 | printf ("Unhandled intrinsic\n"); |
| 1677 | assert(0); |
| 1678 | break; |
| 1679 | } |
| 1680 | } |
| 1681 | |
| 1682 | static unsigned |
| 1683 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 1684 | { |
| 1685 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1686 | case GLSL_SAMPLER_DIM_1D: |
| 1687 | case GLSL_SAMPLER_DIM_BUF: |
| 1688 | return MALI_TEX_1D; |
| 1689 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1690 | case GLSL_SAMPLER_DIM_2D: |
| 1691 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 44a6c38 | 2019-08-14 08:44:40 -0700 | [diff] [blame] | 1692 | case GLSL_SAMPLER_DIM_RECT: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1693 | return MALI_TEX_2D; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1694 | |
| 1695 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1696 | return MALI_TEX_3D; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1697 | |
| 1698 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1699 | return MALI_TEX_CUBE; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1700 | |
| 1701 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1702 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1703 | assert(0); |
| 1704 | return 0; |
| 1705 | } |
| 1706 | } |
| 1707 | |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1708 | /* Tries to attach an explicit LOD / bias as a constant. Returns whether this |
| 1709 | * was successful */ |
| 1710 | |
| 1711 | static bool |
| 1712 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1713 | compiler_context *ctx, |
| 1714 | nir_src lod, |
| 1715 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1716 | { |
| 1717 | /* To attach as constant, it has to *be* constant */ |
| 1718 | |
| 1719 | if (!nir_src_is_const(lod)) |
| 1720 | return false; |
| 1721 | |
| 1722 | float f = nir_src_as_float(lod); |
| 1723 | |
| 1724 | /* Break into fixed-point */ |
| 1725 | signed lod_int = f; |
| 1726 | float lod_frac = f - lod_int; |
| 1727 | |
| 1728 | /* Carry over negative fractions */ |
| 1729 | if (lod_frac < 0.0) { |
| 1730 | lod_int--; |
| 1731 | lod_frac += 1.0; |
| 1732 | } |
| 1733 | |
| 1734 | /* Encode */ |
| 1735 | word->bias = float_to_ubyte(lod_frac); |
| 1736 | word->bias_int = lod_int; |
| 1737 | |
| 1738 | return true; |
| 1739 | } |
| 1740 | |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1741 | static enum mali_sampler_type |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1742 | midgard_sampler_type(nir_alu_type t) { |
| 1743 | switch (nir_alu_type_get_base_type(t)) |
| 1744 | { |
| 1745 | case nir_type_float: |
| 1746 | return MALI_SAMPLER_FLOAT; |
| 1747 | case nir_type_int: |
| 1748 | return MALI_SAMPLER_SIGNED; |
| 1749 | case nir_type_uint: |
| 1750 | return MALI_SAMPLER_UNSIGNED; |
| 1751 | default: |
| 1752 | unreachable("Unknown sampler type"); |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1753 | } |
| 1754 | } |
| 1755 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1756 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1757 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1758 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1759 | { |
| 1760 | /* TODO */ |
| 1761 | //assert (!instr->sampler); |
| 1762 | //assert (!instr->texture_array_size); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1763 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1764 | int texture_index = instr->texture_index; |
| 1765 | int sampler_index = texture_index; |
| 1766 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1767 | /* No helper to build texture words -- we do it all here */ |
| 1768 | midgard_instruction ins = { |
| 1769 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1770 | .mask = 0xF, |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 1771 | .dest = nir_dest_index(ctx, &instr->dest), |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 1772 | .src = { ~0, ~0, ~0, ~0 }, |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1773 | .swizzle = SWIZZLE_IDENTITY_4, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1774 | .texture = { |
| 1775 | .op = midgard_texop, |
| 1776 | .format = midgard_tex_format(instr->sampler_dim), |
| 1777 | .texture_handle = texture_index, |
| 1778 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1779 | |
| 1780 | /* TODO: half */ |
| 1781 | .in_reg_full = 1, |
| 1782 | .out_full = 1, |
| 1783 | |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1784 | .sampler_type = midgard_sampler_type(instr->dest_type), |
Alyssa Rosenzweig | 1a53bed | 2019-12-16 17:13:46 -0500 | [diff] [blame] | 1785 | .shadow = instr->is_shadow, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1786 | } |
| 1787 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 1788 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1789 | /* We may need a temporary for the coordinate */ |
| 1790 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1791 | bool needs_temp_coord = |
| 1792 | (midgard_texop == TEXTURE_OP_TEXEL_FETCH) || |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1793 | (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) || |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1794 | (instr->is_shadow); |
| 1795 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1796 | unsigned coords = needs_temp_coord ? make_compiler_temp_reg(ctx) : 0; |
| 1797 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1798 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1799 | int index = nir_src_index(ctx, &instr->src[i].src); |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 1800 | unsigned nr_components = nir_src_num_components(instr->src[i].src); |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1801 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1802 | switch (instr->src[i].src_type) { |
| 1803 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1804 | emit_explicit_constant(ctx, index, index); |
| 1805 | |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1806 | unsigned coord_mask = mask_of(instr->coord_components); |
| 1807 | |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 1808 | bool flip_zw = (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) && (coord_mask & (1 << COMPONENT_Z)); |
| 1809 | |
| 1810 | if (flip_zw) |
| 1811 | coord_mask ^= ((1 << COMPONENT_Z) | (1 << COMPONENT_W)); |
| 1812 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1813 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
| 1814 | /* texelFetch is undefined on samplerCube */ |
| 1815 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 1816 | |
| 1817 | /* For cubemaps, we use a special ld/st op to |
| 1818 | * select the face and copy the xy into the |
| 1819 | * texture register */ |
| 1820 | |
| 1821 | midgard_instruction ld = m_ld_cubemap_coords(coords, 0); |
| 1822 | ld.src[1] = index; |
| 1823 | ld.mask = 0x3; /* xy */ |
| 1824 | ld.load_store.arg_1 = 0x20; |
| 1825 | ld.swizzle[1][3] = COMPONENT_X; |
| 1826 | emit_mir_instruction(ctx, ld); |
| 1827 | |
| 1828 | /* xyzw -> xyxx */ |
| 1829 | ins.swizzle[1][2] = instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
| 1830 | ins.swizzle[1][3] = COMPONENT_X; |
| 1831 | } else if (needs_temp_coord) { |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1832 | /* mov coord_temp, coords */ |
| 1833 | midgard_instruction mov = v_mov(index, coords); |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1834 | mov.mask = coord_mask; |
Alyssa Rosenzweig | bc4c853 | 2020-01-06 21:31:46 -0500 | [diff] [blame] | 1835 | |
| 1836 | if (flip_zw) |
| 1837 | mov.swizzle[1][COMPONENT_W] = COMPONENT_Z; |
| 1838 | |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1839 | emit_mir_instruction(ctx, mov); |
| 1840 | } else { |
| 1841 | coords = index; |
| 1842 | } |
| 1843 | |
Alyssa Rosenzweig | 6b7243f | 2019-12-20 17:25:05 -0500 | [diff] [blame] | 1844 | ins.src[1] = coords; |
| 1845 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1846 | /* Texelfetch coordinates uses all four elements |
| 1847 | * (xyz/index) regardless of texture dimensionality, |
| 1848 | * which means it's necessary to zero the unused |
| 1849 | * components to keep everything happy */ |
| 1850 | |
| 1851 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1852 | /* mov index.zw, #0, or generalized */ |
Alyssa Rosenzweig | d183f84 | 2019-12-16 17:02:36 -0500 | [diff] [blame] | 1853 | midgard_instruction mov = |
| 1854 | v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), coords); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1855 | mov.has_constants = true; |
Alyssa Rosenzweig | 9e5a141 | 2019-12-20 17:01:29 -0500 | [diff] [blame] | 1856 | mov.mask = coord_mask ^ 0xF; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1857 | emit_mir_instruction(ctx, mov); |
| 1858 | } |
| 1859 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1860 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 1861 | /* Array component in w but NIR wants it in z, |
| 1862 | * but if we have a temp coord we already fixed |
| 1863 | * that up */ |
| 1864 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1865 | if (nr_components == 3) { |
| 1866 | ins.swizzle[1][2] = COMPONENT_Z; |
Alyssa Rosenzweig | 4cd3dc9 | 2020-01-06 21:36:20 -0500 | [diff] [blame] | 1867 | ins.swizzle[1][3] = needs_temp_coord ? COMPONENT_W : COMPONENT_Z; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1868 | } else if (nr_components == 2) { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1869 | ins.swizzle[1][2] = |
| 1870 | instr->is_shadow ? COMPONENT_Z : COMPONENT_X; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1871 | ins.swizzle[1][3] = COMPONENT_X; |
| 1872 | } else |
Alyssa Rosenzweig | edc8e41 | 2019-08-15 16:41:53 -0700 | [diff] [blame] | 1873 | unreachable("Invalid texture 2D components"); |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1874 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1875 | |
Alyssa Rosenzweig | 64b2fe9 | 2019-12-20 12:38:24 -0500 | [diff] [blame] | 1876 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 1877 | /* We zeroed */ |
| 1878 | ins.swizzle[1][2] = COMPONENT_Z; |
| 1879 | ins.swizzle[1][3] = COMPONENT_W; |
| 1880 | } |
| 1881 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1882 | break; |
| 1883 | } |
| 1884 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 1885 | case nir_tex_src_bias: |
| 1886 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1887 | /* Try as a constant if we can */ |
| 1888 | |
| 1889 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 1890 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 1891 | break; |
| 1892 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1893 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 1894 | ins.src[2] = index; |
Alyssa Rosenzweig | 72e5749 | 2019-12-20 12:34:20 -0500 | [diff] [blame] | 1895 | |
| 1896 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 1897 | ins.swizzle[2][c] = COMPONENT_X; |
| 1898 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1899 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1900 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1901 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1902 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1903 | |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 1904 | case nir_tex_src_offset: { |
| 1905 | ins.texture.offset_register = true; |
| 1906 | ins.src[3] = index; |
| 1907 | |
| 1908 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) |
| 1909 | ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c; |
| 1910 | |
| 1911 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | 4ec1f95 | 2019-12-20 12:58:10 -0500 | [diff] [blame] | 1912 | break; |
Alyssa Rosenzweig | ccbc9a4 | 2019-12-19 10:35:18 -0500 | [diff] [blame] | 1913 | }; |
| 1914 | |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1915 | case nir_tex_src_comparator: { |
Alyssa Rosenzweig | 66013cb | 2019-12-16 17:14:04 -0500 | [diff] [blame] | 1916 | unsigned comp = COMPONENT_Z; |
| 1917 | |
| 1918 | /* mov coord_temp.foo, coords */ |
| 1919 | midgard_instruction mov = v_mov(index, coords); |
| 1920 | mov.mask = 1 << comp; |
| 1921 | |
| 1922 | for (unsigned i = 0; i < MIR_VEC_COMPONENTS; ++i) |
| 1923 | mov.swizzle[1][i] = COMPONENT_X; |
| 1924 | |
| 1925 | emit_mir_instruction(ctx, mov); |
| 1926 | break; |
| 1927 | } |
| 1928 | |
Alyssa Rosenzweig | 5062b61 | 2019-06-11 09:55:18 -0700 | [diff] [blame] | 1929 | default: |
| 1930 | unreachable("Unknown texture source type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1931 | } |
| 1932 | } |
| 1933 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1934 | emit_mir_instruction(ctx, ins); |
| 1935 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1936 | /* Used for .cont and .last hinting */ |
| 1937 | ctx->texture_op_count++; |
| 1938 | } |
| 1939 | |
| 1940 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1941 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 1942 | { |
| 1943 | switch (instr->op) { |
| 1944 | case nir_texop_tex: |
| 1945 | case nir_texop_txb: |
| 1946 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 1947 | break; |
| 1948 | case nir_texop_txl: |
| 1949 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 1950 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 1951 | case nir_texop_txf: |
| 1952 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 1953 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 1954 | case nir_texop_txs: |
Alyssa Rosenzweig | 6189274 | 2019-08-21 09:15:56 -0700 | [diff] [blame] | 1955 | emit_sysval_read(ctx, &instr->instr, ~0, 4); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 1956 | break; |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1957 | default: |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1958 | unreachable("Unhanlded texture op"); |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1959 | } |
| 1960 | } |
| 1961 | |
| 1962 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1963 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 1964 | { |
| 1965 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1966 | case nir_jump_break: { |
| 1967 | /* Emit a branch out of the loop */ |
| 1968 | struct midgard_instruction br = v_branch(false, false); |
| 1969 | br.branch.target_type = TARGET_BREAK; |
| 1970 | br.branch.target_break = ctx->current_loop_depth; |
| 1971 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1972 | break; |
| 1973 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1974 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1975 | default: |
| 1976 | DBG("Unknown jump type %d\n", instr->type); |
| 1977 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1978 | } |
| 1979 | } |
| 1980 | |
| 1981 | static void |
| 1982 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 1983 | { |
| 1984 | switch (instr->type) { |
| 1985 | case nir_instr_type_load_const: |
| 1986 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 1987 | break; |
| 1988 | |
| 1989 | case nir_instr_type_intrinsic: |
| 1990 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 1991 | break; |
| 1992 | |
| 1993 | case nir_instr_type_alu: |
| 1994 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 1995 | break; |
| 1996 | |
| 1997 | case nir_instr_type_tex: |
| 1998 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 1999 | break; |
| 2000 | |
| 2001 | case nir_instr_type_jump: |
| 2002 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 2003 | break; |
| 2004 | |
| 2005 | case nir_instr_type_ssa_undef: |
| 2006 | /* Spurious */ |
| 2007 | break; |
| 2008 | |
| 2009 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2010 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2011 | break; |
| 2012 | } |
| 2013 | } |
| 2014 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2015 | |
| 2016 | /* ALU instructions can inline or embed constants, which decreases register |
| 2017 | * pressure and saves space. */ |
| 2018 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2019 | #define CONDITIONAL_ATTACH(idx) { \ |
| 2020 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2021 | \ |
| 2022 | if (entry) { \ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2023 | attach_constants(ctx, alu, entry, alu->src[idx] + 1); \ |
| 2024 | alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2025 | } \ |
| 2026 | } |
| 2027 | |
| 2028 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2029 | inline_alu_constants(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2030 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2031 | mir_foreach_instr_in_block(block, alu) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2032 | /* Other instructions cannot inline constants */ |
| 2033 | if (alu->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 5e06d90 | 2019-08-30 11:06:33 -0700 | [diff] [blame] | 2034 | if (alu->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2035 | |
| 2036 | /* If there is already a constant here, we can do nothing */ |
| 2037 | if (alu->has_constants) continue; |
| 2038 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2039 | CONDITIONAL_ATTACH(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2040 | |
| 2041 | if (!alu->has_constants) { |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2042 | CONDITIONAL_ATTACH(1) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2043 | } else if (!alu->inline_constant) { |
| 2044 | /* Corner case: _two_ vec4 constants, for instance with a |
| 2045 | * csel. For this case, we can only use a constant |
| 2046 | * register for one, we'll have to emit a move for the |
| 2047 | * other. Note, if both arguments are constants, then |
| 2048 | * necessarily neither argument depends on the value of |
| 2049 | * any particular register. As the destination register |
| 2050 | * will be wiped, that means we can spill the constant |
| 2051 | * to the destination register. |
| 2052 | */ |
| 2053 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2054 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1); |
| 2055 | unsigned scratch = alu->dest; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2056 | |
| 2057 | if (entry) { |
Alyssa Rosenzweig | c3a46e7 | 2019-10-30 16:29:28 -0400 | [diff] [blame] | 2058 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch); |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2059 | attach_constants(ctx, &ins, entry, alu->src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2060 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2061 | /* Set the source */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2062 | alu->src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2063 | |
| 2064 | /* Inject us -before- the last instruction which set r31 */ |
Boris Brezillon | 938c5b0 | 2019-08-28 09:17:21 +0200 | [diff] [blame] | 2065 | mir_insert_instruction_before(ctx, mir_prev_op(alu), ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2066 | } |
| 2067 | } |
| 2068 | } |
| 2069 | } |
| 2070 | |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2071 | /* Being a little silly with the names, but returns the op that is the bitwise |
| 2072 | * inverse of the op with the argument switched. I.e. (f and g are |
| 2073 | * contrapositives): |
| 2074 | * |
| 2075 | * f(a, b) = ~g(b, a) |
| 2076 | * |
| 2077 | * Corollary: if g is the contrapositve of f, f is the contrapositive of g: |
| 2078 | * |
| 2079 | * f(a, b) = ~g(b, a) |
| 2080 | * ~f(a, b) = g(b, a) |
| 2081 | * ~f(a, b) = ~h(a, b) where h is the contrapositive of g |
| 2082 | * f(a, b) = h(a, b) |
| 2083 | * |
| 2084 | * Thus we define this function in pairs. |
| 2085 | */ |
| 2086 | |
| 2087 | static inline midgard_alu_op |
| 2088 | mir_contrapositive(midgard_alu_op op) |
| 2089 | { |
| 2090 | switch (op) { |
| 2091 | case midgard_alu_op_flt: |
| 2092 | return midgard_alu_op_fle; |
| 2093 | case midgard_alu_op_fle: |
| 2094 | return midgard_alu_op_flt; |
| 2095 | |
| 2096 | case midgard_alu_op_ilt: |
| 2097 | return midgard_alu_op_ile; |
| 2098 | case midgard_alu_op_ile: |
| 2099 | return midgard_alu_op_ilt; |
| 2100 | |
| 2101 | default: |
| 2102 | unreachable("No known contrapositive"); |
| 2103 | } |
| 2104 | } |
| 2105 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2106 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2107 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2108 | * constants can be demoted to inline constants, for space savings and |
| 2109 | * sometimes a performance boost */ |
| 2110 | |
| 2111 | static void |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2112 | embedded_to_inline_constant(compiler_context *ctx, midgard_block *block) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2113 | { |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2114 | mir_foreach_instr_in_block(block, ins) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2115 | if (!ins->has_constants) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2116 | if (ins->has_inline_constant) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2117 | |
| 2118 | /* Blend constants must not be inlined by definition */ |
| 2119 | if (ins->has_blend_constant) continue; |
| 2120 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2121 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
| 2122 | bool is_16 = ins->alu.reg_mode == midgard_reg_mode_16; |
| 2123 | bool is_32 = ins->alu.reg_mode == midgard_reg_mode_32; |
| 2124 | |
| 2125 | if (!(is_16 || is_32)) |
| 2126 | continue; |
| 2127 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2128 | /* src1 cannot be an inline constant due to encoding |
| 2129 | * restrictions. So, if possible we try to flip the arguments |
| 2130 | * in that case */ |
| 2131 | |
| 2132 | int op = ins->alu.op; |
| 2133 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2134 | if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2135 | bool flip = alu_opcode_props[op].props & OP_COMMUTES; |
| 2136 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2137 | switch (op) { |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2138 | /* Conditionals can be inverted */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2139 | case midgard_alu_op_flt: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2140 | case midgard_alu_op_ilt: |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2141 | case midgard_alu_op_fle: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2142 | case midgard_alu_op_ile: |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2143 | ins->alu.op = mir_contrapositive(ins->alu.op); |
| 2144 | ins->invert = true; |
| 2145 | flip = true; |
| 2146 | break; |
| 2147 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2148 | case midgard_alu_op_fcsel: |
| 2149 | case midgard_alu_op_icsel: |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 2150 | DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name); |
Alyssa Rosenzweig | bb1aff3 | 2019-04-24 02:18:28 +0000 | [diff] [blame] | 2151 | default: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2152 | break; |
Alyssa Rosenzweig | bb1aff3 | 2019-04-24 02:18:28 +0000 | [diff] [blame] | 2153 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2154 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2155 | if (flip) |
| 2156 | mir_flip(ins); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2157 | } |
| 2158 | |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2159 | if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2160 | /* Extract the source information */ |
| 2161 | |
| 2162 | midgard_vector_alu_src *src; |
| 2163 | int q = ins->alu.src2; |
| 2164 | midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q; |
| 2165 | src = m; |
| 2166 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2167 | /* Component is from the swizzle. Take a nonzero component */ |
| 2168 | assert(ins->mask); |
| 2169 | unsigned first_comp = ffs(ins->mask) - 1; |
| 2170 | unsigned component = ins->swizzle[1][first_comp]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2171 | |
| 2172 | /* Scale constant appropriately, if we can legally */ |
| 2173 | uint16_t scaled_constant = 0; |
| 2174 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2175 | if (is_16) { |
| 2176 | scaled_constant = ins->constants.u16[component]; |
| 2177 | } else if (midgard_is_integer_op(op)) { |
| 2178 | scaled_constant = ins->constants.u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2179 | |
| 2180 | /* Constant overflow after resize */ |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2181 | if (scaled_constant != ins->constants.u32[component]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2182 | continue; |
| 2183 | } else { |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2184 | float original = ins->constants.f32[component]; |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2185 | scaled_constant = _mesa_float_to_half(original); |
| 2186 | |
| 2187 | /* Check for loss of precision. If this is |
| 2188 | * mediump, we don't care, but for a highp |
| 2189 | * shader, we need to pay attention. NIR |
| 2190 | * doesn't yet tell us which mode we're in! |
| 2191 | * Practically this prevents most constants |
| 2192 | * from being inlined, sadly. */ |
| 2193 | |
| 2194 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2195 | |
| 2196 | if (fp32 != original) |
| 2197 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2198 | } |
| 2199 | |
| 2200 | /* We don't know how to handle these with a constant */ |
| 2201 | |
Alyssa Rosenzweig | c45487b | 2019-07-26 11:52:30 -0700 | [diff] [blame] | 2202 | if (mir_nontrivial_source2_mod_simple(ins) || src->rep_low || src->rep_high) { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2203 | DBG("Bailing inline constant...\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2204 | continue; |
| 2205 | } |
| 2206 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2207 | /* Make sure that the constant is not itself a vector |
| 2208 | * by checking if all accessed values are the same. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2209 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2210 | const midgard_constants *cons = &ins->constants; |
| 2211 | uint32_t value = is_16 ? cons->u16[component] : cons->u32[component]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2212 | |
| 2213 | bool is_vector = false; |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 2214 | unsigned mask = effective_writemask(&ins->alu, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2215 | |
Alyssa Rosenzweig | 70072a2 | 2019-10-26 14:06:17 -0400 | [diff] [blame] | 2216 | for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2217 | /* We only care if this component is actually used */ |
| 2218 | if (!(mask & (1 << c))) |
| 2219 | continue; |
| 2220 | |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2221 | uint32_t test = is_16 ? |
| 2222 | cons->u16[ins->swizzle[1][c]] : |
| 2223 | cons->u32[ins->swizzle[1][c]]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2224 | |
| 2225 | if (test != value) { |
| 2226 | is_vector = true; |
| 2227 | break; |
| 2228 | } |
| 2229 | } |
| 2230 | |
| 2231 | if (is_vector) |
| 2232 | continue; |
| 2233 | |
| 2234 | /* Get rid of the embedded constant */ |
| 2235 | ins->has_constants = false; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2236 | ins->src[1] = ~0; |
| 2237 | ins->has_inline_constant = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2238 | ins->inline_constant = scaled_constant; |
| 2239 | } |
| 2240 | } |
| 2241 | } |
| 2242 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2243 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2244 | * per block is legal semantically */ |
| 2245 | |
| 2246 | static void |
| 2247 | midgard_opt_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
| 2248 | { |
| 2249 | bool branched = false; |
| 2250 | |
| 2251 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2252 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2253 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2254 | if (branched) |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2255 | mir_remove_instruction(ins); |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2256 | |
| 2257 | branched = true; |
| 2258 | } |
| 2259 | } |
| 2260 | |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2261 | /* fmov.pos is an idiom for fpos. Propoagate the .pos up to the source, so then |
| 2262 | * the move can be propagated away entirely */ |
| 2263 | |
| 2264 | static bool |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 2265 | mir_compose_float_outmod(midgard_outmod_float *outmod, midgard_outmod_float comp) |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2266 | { |
| 2267 | /* Nothing to do */ |
| 2268 | if (comp == midgard_outmod_none) |
| 2269 | return true; |
| 2270 | |
| 2271 | if (*outmod == midgard_outmod_none) { |
| 2272 | *outmod = comp; |
| 2273 | return true; |
| 2274 | } |
| 2275 | |
| 2276 | /* TODO: Compose rules */ |
| 2277 | return false; |
| 2278 | } |
| 2279 | |
| 2280 | static bool |
| 2281 | midgard_opt_pos_propagate(compiler_context *ctx, midgard_block *block) |
| 2282 | { |
| 2283 | bool progress = false; |
| 2284 | |
| 2285 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2286 | if (ins->type != TAG_ALU_4) continue; |
| 2287 | if (ins->alu.op != midgard_alu_op_fmov) continue; |
| 2288 | if (ins->alu.outmod != midgard_outmod_pos) continue; |
| 2289 | |
| 2290 | /* TODO: Registers? */ |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2291 | unsigned src = ins->src[1]; |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 2292 | if (src & IS_REG) continue; |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2293 | |
| 2294 | /* There might be a source modifier, too */ |
| 2295 | if (mir_nontrivial_source2_mod(ins)) continue; |
| 2296 | |
| 2297 | /* Backpropagate the modifier */ |
| 2298 | mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) { |
| 2299 | if (v->type != TAG_ALU_4) continue; |
Alyssa Rosenzweig | 75b6be2 | 2019-08-26 11:58:27 -0700 | [diff] [blame] | 2300 | if (v->dest != src) continue; |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2301 | |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 2302 | /* Can we even take a float outmod? */ |
| 2303 | if (midgard_is_integer_out_op(v->alu.op)) continue; |
| 2304 | |
| 2305 | midgard_outmod_float temp = v->alu.outmod; |
| 2306 | progress |= mir_compose_float_outmod(&temp, ins->alu.outmod); |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2307 | |
| 2308 | /* Throw in the towel.. */ |
| 2309 | if (!progress) break; |
| 2310 | |
| 2311 | /* Otherwise, transfer the modifier */ |
| 2312 | v->alu.outmod = temp; |
| 2313 | ins->alu.outmod = midgard_outmod_none; |
| 2314 | |
| 2315 | break; |
| 2316 | } |
| 2317 | } |
| 2318 | |
| 2319 | return progress; |
| 2320 | } |
| 2321 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2322 | static unsigned |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2323 | emit_fragment_epilogue(compiler_context *ctx, unsigned rt) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2324 | { |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2325 | /* Loop to ourselves */ |
| 2326 | |
| 2327 | struct midgard_instruction ins = v_branch(false, false); |
| 2328 | ins.writeout = true; |
| 2329 | ins.branch.target_block = ctx->block_count - 1; |
Boris Brezillon | 15c92d1 | 2020-01-20 15:00:57 +0100 | [diff] [blame] | 2330 | ins.constants.u32[0] = rt * 0x100; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2331 | emit_mir_instruction(ctx, ins); |
| 2332 | |
Alyssa Rosenzweig | 3448b26 | 2019-12-03 10:37:01 -0500 | [diff] [blame] | 2333 | ctx->current_block->epilogue = true; |
Alyssa Rosenzweig | 6039634 | 2019-11-23 16:08:02 -0500 | [diff] [blame] | 2334 | schedule_barrier(ctx); |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2335 | return ins.branch.target_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2336 | } |
| 2337 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2338 | static midgard_block * |
| 2339 | emit_block(compiler_context *ctx, nir_block *block) |
| 2340 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2341 | midgard_block *this_block = ctx->after_block; |
| 2342 | ctx->after_block = NULL; |
| 2343 | |
| 2344 | if (!this_block) |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2345 | this_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2346 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2347 | list_addtail(&this_block->link, &ctx->blocks); |
| 2348 | |
| 2349 | this_block->is_scheduled = false; |
| 2350 | ++ctx->block_count; |
| 2351 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2352 | /* Set up current block */ |
| 2353 | list_inithead(&this_block->instructions); |
| 2354 | ctx->current_block = this_block; |
| 2355 | |
| 2356 | nir_foreach_instr(instr, block) { |
| 2357 | emit_instr(ctx, instr); |
| 2358 | ++ctx->instruction_count; |
| 2359 | } |
| 2360 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2361 | return this_block; |
| 2362 | } |
| 2363 | |
| 2364 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2365 | |
| 2366 | static void |
| 2367 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2368 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2369 | midgard_block *before_block = ctx->current_block; |
| 2370 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2371 | /* Speculatively emit the branch, but we can't fill it in until later */ |
| 2372 | EMIT(branch, true, true); |
| 2373 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
Alyssa Rosenzweig | d6e4e36 | 2019-08-26 13:59:29 -0700 | [diff] [blame] | 2374 | then_branch->src[0] = nir_src_index(ctx, &nif->condition); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2375 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2376 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2377 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2378 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2379 | |
| 2380 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2381 | EMIT(branch, false, false); |
| 2382 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2383 | |
| 2384 | /* Emit second block, and check if it's empty */ |
| 2385 | |
| 2386 | int else_idx = ctx->block_count; |
| 2387 | int count_in = ctx->instruction_count; |
| 2388 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2389 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2390 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2391 | |
| 2392 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2393 | |
| 2394 | assert(then_block); |
| 2395 | assert(else_block); |
| 2396 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2397 | if (ctx->instruction_count == count_in) { |
| 2398 | /* The else block is empty, so don't emit an exit jump */ |
| 2399 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2400 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2401 | } else { |
| 2402 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2403 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2404 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2405 | |
| 2406 | /* Wire up the successors */ |
| 2407 | |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2408 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2409 | |
| 2410 | midgard_block_add_successor(before_block, then_block); |
| 2411 | midgard_block_add_successor(before_block, else_block); |
| 2412 | |
| 2413 | midgard_block_add_successor(end_then_block, ctx->after_block); |
| 2414 | midgard_block_add_successor(end_else_block, ctx->after_block); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2415 | } |
| 2416 | |
| 2417 | static void |
| 2418 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2419 | { |
| 2420 | /* Remember where we are */ |
| 2421 | midgard_block *start_block = ctx->current_block; |
| 2422 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2423 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2424 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2425 | |
| 2426 | /* Get index from before the body so we can loop back later */ |
| 2427 | int start_idx = ctx->block_count; |
| 2428 | |
| 2429 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2430 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2431 | |
| 2432 | /* Branch back to loop back */ |
| 2433 | struct midgard_instruction br_back = v_branch(false, false); |
| 2434 | br_back.branch.target_block = start_idx; |
| 2435 | emit_mir_instruction(ctx, br_back); |
| 2436 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2437 | /* Mark down that branch in the graph. */ |
| 2438 | midgard_block_add_successor(start_block, loop_block); |
| 2439 | midgard_block_add_successor(ctx->current_block, loop_block); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2440 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2441 | /* Find the index of the block about to follow us (note: we don't add |
| 2442 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2443 | int break_block_idx = ctx->block_count; |
| 2444 | |
| 2445 | /* Fix up the break statements we emitted to point to the right place, |
| 2446 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | aeeeef1 | 2019-08-15 08:11:10 -0700 | [diff] [blame] | 2447 | ctx->after_block = create_empty_block(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2448 | |
| 2449 | list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2450 | mir_foreach_instr_in_block(block, ins) { |
| 2451 | if (ins->type != TAG_ALU_4) continue; |
| 2452 | if (!ins->compact_branch) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2453 | |
| 2454 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2455 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2456 | |
| 2457 | /* It's a break! Check if it's our break */ |
| 2458 | if (ins->branch.target_break != loop_idx) continue; |
| 2459 | |
| 2460 | /* Okay, cool, we're breaking out of this loop. |
| 2461 | * Rewrite from a break to a goto */ |
| 2462 | |
| 2463 | ins->branch.target_type = TARGET_GOTO; |
| 2464 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2465 | |
| 2466 | midgard_block_add_successor(block, ctx->after_block); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2467 | } |
| 2468 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2469 | |
| 2470 | /* Now that we've finished emitting the loop, free up the depth again |
| 2471 | * so we play nice with recursion amid nested loops */ |
| 2472 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2473 | |
| 2474 | /* Dump loop stats */ |
| 2475 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2476 | } |
| 2477 | |
| 2478 | static midgard_block * |
| 2479 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2480 | { |
| 2481 | midgard_block *start_block = NULL; |
| 2482 | |
| 2483 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2484 | switch (node->type) { |
| 2485 | case nir_cf_node_block: { |
| 2486 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2487 | |
| 2488 | if (!start_block) |
| 2489 | start_block = block; |
| 2490 | |
| 2491 | break; |
| 2492 | } |
| 2493 | |
| 2494 | case nir_cf_node_if: |
| 2495 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2496 | break; |
| 2497 | |
| 2498 | case nir_cf_node_loop: |
| 2499 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2500 | break; |
| 2501 | |
| 2502 | case nir_cf_node_function: |
| 2503 | assert(0); |
| 2504 | break; |
| 2505 | } |
| 2506 | } |
| 2507 | |
| 2508 | return start_block; |
| 2509 | } |
| 2510 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2511 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2512 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2513 | * until we find one that 'works' */ |
| 2514 | |
| 2515 | static unsigned |
| 2516 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2517 | { |
| 2518 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2519 | |
| 2520 | unsigned first_tag = 0; |
| 2521 | |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2522 | mir_foreach_block_from(ctx, initial_block, v) { |
Alyssa Rosenzweig | 45ac8ea | 2019-11-04 10:32:49 -0500 | [diff] [blame] | 2523 | if (v->quadword_count) { |
| 2524 | midgard_bundle *initial_bundle = |
| 2525 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2526 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2527 | first_tag = initial_bundle->tag; |
| 2528 | break; |
| 2529 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2530 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2531 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2532 | return first_tag; |
| 2533 | } |
| 2534 | |
Alyssa Rosenzweig | 67fe2af | 2019-12-27 16:01:34 -0500 | [diff] [blame] | 2535 | static unsigned |
| 2536 | pan_format_from_nir_base(nir_alu_type base) |
| 2537 | { |
| 2538 | switch (base) { |
| 2539 | case nir_type_int: |
| 2540 | return MALI_FORMAT_SINT; |
| 2541 | case nir_type_uint: |
| 2542 | case nir_type_bool: |
| 2543 | return MALI_FORMAT_UINT; |
| 2544 | case nir_type_float: |
| 2545 | return MALI_CHANNEL_FLOAT; |
| 2546 | default: |
| 2547 | unreachable("Invalid base"); |
| 2548 | } |
| 2549 | } |
| 2550 | |
| 2551 | static unsigned |
| 2552 | pan_format_from_nir_size(nir_alu_type base, unsigned size) |
| 2553 | { |
| 2554 | if (base == nir_type_float) { |
| 2555 | switch (size) { |
| 2556 | case 16: return MALI_FORMAT_SINT; |
| 2557 | case 32: return MALI_FORMAT_UNORM; |
| 2558 | default: |
| 2559 | unreachable("Invalid float size for format"); |
| 2560 | } |
| 2561 | } else { |
| 2562 | switch (size) { |
| 2563 | case 1: |
| 2564 | case 8: return MALI_CHANNEL_8; |
| 2565 | case 16: return MALI_CHANNEL_16; |
| 2566 | case 32: return MALI_CHANNEL_32; |
| 2567 | default: |
| 2568 | unreachable("Invalid int size for format"); |
| 2569 | } |
| 2570 | } |
| 2571 | } |
| 2572 | |
| 2573 | static enum mali_format |
| 2574 | pan_format_from_glsl(const struct glsl_type *type) |
| 2575 | { |
| 2576 | enum glsl_base_type glsl_base = glsl_get_base_type(glsl_without_array(type)); |
| 2577 | nir_alu_type t = nir_get_nir_type_for_glsl_base_type(glsl_base); |
| 2578 | |
| 2579 | unsigned base = nir_alu_type_get_base_type(t); |
| 2580 | unsigned size = nir_alu_type_get_type_size(t); |
| 2581 | |
| 2582 | return pan_format_from_nir_base(base) | |
| 2583 | pan_format_from_nir_size(base, size) | |
| 2584 | MALI_NR_CHANNELS(4); |
| 2585 | } |
| 2586 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2587 | /* For each fragment writeout instruction, generate a writeout loop to |
| 2588 | * associate with it */ |
| 2589 | |
| 2590 | static void |
| 2591 | mir_add_writeout_loops(compiler_context *ctx) |
| 2592 | { |
| 2593 | for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) { |
| 2594 | midgard_instruction *br = ctx->writeout_branch[rt]; |
| 2595 | if (!br) continue; |
| 2596 | |
| 2597 | unsigned popped = br->branch.target_block; |
| 2598 | midgard_block_add_successor(mir_get_block(ctx, popped - 1), ctx->current_block); |
| 2599 | br->branch.target_block = emit_fragment_epilogue(ctx, rt); |
| 2600 | |
| 2601 | /* If we have more RTs, we'll need to restore back after our |
| 2602 | * loop terminates */ |
| 2603 | |
| 2604 | if ((rt + 1) < ARRAY_SIZE(ctx->writeout_branch) && ctx->writeout_branch[rt + 1]) { |
| 2605 | midgard_instruction uncond = v_branch(false, false); |
| 2606 | uncond.branch.target_block = popped; |
| 2607 | emit_mir_instruction(ctx, uncond); |
| 2608 | midgard_block_add_successor(ctx->current_block, mir_get_block(ctx, popped)); |
| 2609 | schedule_barrier(ctx); |
| 2610 | } else { |
| 2611 | /* We're last, so we can terminate here */ |
| 2612 | br->last_writeout = true; |
| 2613 | } |
| 2614 | } |
| 2615 | } |
| 2616 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2617 | int |
Alyssa Rosenzweig | 35418f6 | 2019-12-16 18:05:21 -0500 | [diff] [blame] | 2618 | midgard_compile_shader_nir(nir_shader *nir, midgard_program *program, bool is_blend, unsigned blend_rt, unsigned gpu_id, bool shaderdb) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2619 | { |
| 2620 | struct util_dynarray *compiled = &program->compiled; |
| 2621 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2622 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2623 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2624 | /* TODO: Bound against what? */ |
| 2625 | compiler_context *ctx = rzalloc(NULL, compiler_context); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2626 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2627 | ctx->nir = nir; |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2628 | ctx->stage = nir->info.stage; |
| 2629 | ctx->is_blend = is_blend; |
| 2630 | ctx->alpha_ref = program->alpha_ref; |
Alyssa Rosenzweig | a2d5503 | 2019-11-23 21:44:16 -0500 | [diff] [blame] | 2631 | ctx->blend_rt = blend_rt; |
Alyssa Rosenzweig | fcf144d | 2019-11-19 20:55:42 -0500 | [diff] [blame] | 2632 | ctx->quirks = midgard_get_quirks(gpu_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2633 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2634 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2635 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2636 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2637 | ctx->uniform_cutoff = 8; |
| 2638 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2639 | /* Initialize at a global (not block) level hash tables */ |
| 2640 | |
| 2641 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2642 | ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2643 | ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2644 | |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2645 | /* Record the varying mapping for the command stream's bookkeeping */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2646 | |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2647 | struct exec_list *varyings = |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2648 | ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2649 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2650 | unsigned max_varying = 0; |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2651 | nir_foreach_variable(var, varyings) { |
| 2652 | unsigned loc = var->data.driver_location; |
Alyssa Rosenzweig | 1f7b388 | 2019-04-20 23:39:29 +0000 | [diff] [blame] | 2653 | unsigned sz = glsl_type_size(var->type, FALSE); |
| 2654 | |
Boris Brezillon | 749c544 | 2019-06-13 14:56:02 +0200 | [diff] [blame] | 2655 | for (int c = 0; c < sz; ++c) { |
| 2656 | program->varyings[loc + c] = var->data.location + c; |
Alyssa Rosenzweig | 67fe2af | 2019-12-27 16:01:34 -0500 | [diff] [blame] | 2657 | program->varying_type[loc + c] = pan_format_from_glsl(var->type); |
Boris Brezillon | 749c544 | 2019-06-13 14:56:02 +0200 | [diff] [blame] | 2658 | max_varying = MAX2(max_varying, loc + c); |
Alyssa Rosenzweig | 1f7b388 | 2019-04-20 23:39:29 +0000 | [diff] [blame] | 2659 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2660 | } |
| 2661 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2662 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2663 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2664 | * messed with our I/O quite a bit already) */ |
| 2665 | |
| 2666 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2667 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2668 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2669 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | 2023716 | 2019-08-26 12:14:11 -0700 | [diff] [blame] | 2670 | NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2671 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2672 | |
| 2673 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2674 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 2675 | NIR_PASS_V(nir, nir_split_var_copies); |
| 2676 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2677 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 2678 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2679 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2680 | |
Eric Anholt | 771adff | 2019-04-08 16:32:01 -0700 | [diff] [blame] | 2681 | NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2682 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2683 | /* Optimisation passes */ |
| 2684 | |
Alyssa Rosenzweig | bda2bb3 | 2019-11-21 08:45:27 -0500 | [diff] [blame] | 2685 | optimise_nir(nir, ctx->quirks); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2686 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2687 | if (midgard_debug & MIDGARD_DBG_SHADERS) { |
| 2688 | nir_print_shader(nir, stdout); |
| 2689 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2690 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2691 | /* Assign sysvals and counts, now that we're sure |
| 2692 | * (post-optimisation) */ |
| 2693 | |
| 2694 | midgard_nir_assign_sysvals(ctx, nir); |
| 2695 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2696 | program->uniform_count = nir->num_uniforms; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2697 | program->sysval_count = ctx->sysval_count; |
| 2698 | memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2699 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2700 | nir_foreach_function(func, nir) { |
| 2701 | if (!func->impl) |
| 2702 | continue; |
| 2703 | |
| 2704 | list_inithead(&ctx->blocks); |
| 2705 | ctx->block_count = 0; |
| 2706 | ctx->func = func; |
| 2707 | |
| 2708 | emit_cf_list(ctx, &func->impl->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2709 | break; /* TODO: Multi-function shaders */ |
| 2710 | } |
| 2711 | |
| 2712 | util_dynarray_init(compiled, NULL); |
| 2713 | |
Alyssa Rosenzweig | cc2ba8e | 2019-08-30 10:53:13 -0700 | [diff] [blame] | 2714 | /* Per-block lowering before opts */ |
| 2715 | |
| 2716 | mir_foreach_block(ctx, block) { |
| 2717 | inline_alu_constants(ctx, block); |
| 2718 | midgard_opt_promote_fmov(ctx, block); |
| 2719 | embedded_to_inline_constant(ctx, block); |
| 2720 | } |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2721 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2722 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2723 | bool progress = false; |
| 2724 | |
| 2725 | do { |
| 2726 | progress = false; |
| 2727 | |
| 2728 | mir_foreach_block(ctx, block) { |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2729 | progress |= midgard_opt_pos_propagate(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2730 | progress |= midgard_opt_copy_prop(ctx, block); |
| 2731 | progress |= midgard_opt_dead_code_eliminate(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 2732 | progress |= midgard_opt_combine_projection(ctx, block); |
| 2733 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 620c271 | 2019-07-26 13:14:55 -0700 | [diff] [blame] | 2734 | progress |= midgard_opt_not_propagate(ctx, block); |
Alyssa Rosenzweig | d066ca35 | 2019-07-26 13:32:54 -0700 | [diff] [blame] | 2735 | progress |= midgard_opt_fuse_src_invert(ctx, block); |
Alyssa Rosenzweig | b821e1b | 2019-07-26 13:08:54 -0700 | [diff] [blame] | 2736 | progress |= midgard_opt_fuse_dest_invert(ctx, block); |
Alyssa Rosenzweig | c20063a | 2019-09-28 12:39:15 -0400 | [diff] [blame] | 2737 | progress |= midgard_opt_csel_invert(ctx, block); |
Afonso Bordado | 3e1e4ad | 2019-12-10 13:18:00 +0000 | [diff] [blame] | 2738 | progress |= midgard_opt_drop_cmp_invert(ctx, block); |
Afonso Bordado | 525cbe8 | 2019-12-27 17:09:51 +0000 | [diff] [blame] | 2739 | progress |= midgard_opt_invert_branch(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2740 | } |
| 2741 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2742 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2743 | mir_foreach_block(ctx, block) { |
| 2744 | midgard_lower_invert(ctx, block); |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 2745 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2746 | } |
| 2747 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2748 | /* Nested control-flow can result in dead branches at the end of the |
| 2749 | * block. This messes with our analysis and is just dead code, so cull |
| 2750 | * them */ |
| 2751 | mir_foreach_block(ctx, block) { |
| 2752 | midgard_opt_cull_dead_branch(ctx, block); |
| 2753 | } |
| 2754 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2755 | /* Ensure we were lowered */ |
| 2756 | mir_foreach_instr_global(ctx, ins) { |
| 2757 | assert(!ins->invert); |
| 2758 | } |
| 2759 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2760 | if (ctx->stage == MESA_SHADER_FRAGMENT) |
| 2761 | mir_add_writeout_loops(ctx); |
| 2762 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2763 | /* Schedule! */ |
Robert Foss | 62adb65 | 2020-01-15 01:14:16 +0100 | [diff] [blame] | 2764 | midgard_schedule_program(ctx); |
Alyssa Rosenzweig | 9dc3b18 | 2019-12-06 09:32:38 -0500 | [diff] [blame] | 2765 | mir_ra(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2766 | |
| 2767 | /* Now that all the bundles are scheduled and we can calculate block |
| 2768 | * sizes, emit actual branch instructions rather than placeholders */ |
| 2769 | |
| 2770 | int br_block_idx = 0; |
| 2771 | |
| 2772 | mir_foreach_block(ctx, block) { |
| 2773 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 2774 | for (int c = 0; c < bundle->instruction_count; ++c) { |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2775 | midgard_instruction *ins = bundle->instructions[c]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2776 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2777 | if (!midgard_is_branch_unit(ins->unit)) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2778 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2779 | /* Parse some basic branch info */ |
| 2780 | bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT; |
| 2781 | bool is_conditional = ins->branch.conditional; |
| 2782 | bool is_inverted = ins->branch.invert_conditional; |
| 2783 | bool is_discard = ins->branch.target_type == TARGET_DISCARD; |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2784 | bool is_writeout = ins->writeout; |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2785 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2786 | /* Determine the block we're jumping to */ |
| 2787 | int target_number = ins->branch.target_block; |
| 2788 | |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2789 | /* Report the destination tag */ |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2790 | int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2791 | |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2792 | /* Count up the number of quadwords we're |
| 2793 | * jumping over = number of quadwords until |
| 2794 | * (br_block_idx, target_number) */ |
| 2795 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2796 | int quadword_offset = 0; |
| 2797 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2798 | if (is_discard) { |
Alyssa Rosenzweig | 7f75b2b | 2019-07-30 17:07:25 -0700 | [diff] [blame] | 2799 | /* Ignored */ |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2800 | } else if (target_number > br_block_idx) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2801 | /* Jump forward */ |
| 2802 | |
| 2803 | for (int idx = br_block_idx + 1; idx < target_number; ++idx) { |
| 2804 | midgard_block *blk = mir_get_block(ctx, idx); |
| 2805 | assert(blk); |
| 2806 | |
| 2807 | quadword_offset += blk->quadword_count; |
| 2808 | } |
| 2809 | } else { |
| 2810 | /* Jump backwards */ |
| 2811 | |
| 2812 | for (int idx = br_block_idx; idx >= target_number; --idx) { |
| 2813 | midgard_block *blk = mir_get_block(ctx, idx); |
| 2814 | assert(blk); |
| 2815 | |
| 2816 | quadword_offset -= blk->quadword_count; |
| 2817 | } |
| 2818 | } |
| 2819 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2820 | /* Unconditional extended branches (far jumps) |
| 2821 | * have issues, so we always use a conditional |
| 2822 | * branch, setting the condition to always for |
| 2823 | * unconditional. For compact unconditional |
| 2824 | * branches, cond isn't used so it doesn't |
| 2825 | * matter what we pick. */ |
| 2826 | |
| 2827 | midgard_condition cond = |
| 2828 | !is_conditional ? midgard_condition_always : |
| 2829 | is_inverted ? midgard_condition_false : |
| 2830 | midgard_condition_true; |
| 2831 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2832 | midgard_jmp_writeout_op op = |
| 2833 | is_discard ? midgard_jmp_writeout_op_discard : |
Alyssa Rosenzweig | 02f503e | 2019-12-30 18:53:04 -0500 | [diff] [blame] | 2834 | is_writeout ? midgard_jmp_writeout_op_writeout : |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2835 | (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond : |
| 2836 | midgard_jmp_writeout_op_branch_cond; |
| 2837 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2838 | if (!is_compact) { |
| 2839 | midgard_branch_extended branch = |
| 2840 | midgard_create_branch_extended( |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2841 | cond, op, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2842 | dest_tag, |
| 2843 | quadword_offset); |
| 2844 | |
| 2845 | memcpy(&ins->branch_extended, &branch, sizeof(branch)); |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2846 | } else if (is_conditional || is_discard) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2847 | midgard_branch_cond branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2848 | .op = op, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2849 | .dest_tag = dest_tag, |
| 2850 | .offset = quadword_offset, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2851 | .cond = cond |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2852 | }; |
| 2853 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2854 | assert(branch.offset == quadword_offset); |
| 2855 | |
| 2856 | memcpy(&ins->br_compact, &branch, sizeof(branch)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2857 | } else { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2858 | assert(op == midgard_jmp_writeout_op_branch_uncond); |
| 2859 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2860 | midgard_branch_uncond branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2861 | .op = op, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2862 | .dest_tag = dest_tag, |
| 2863 | .offset = quadword_offset, |
| 2864 | .unknown = 1 |
| 2865 | }; |
| 2866 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2867 | assert(branch.offset == quadword_offset); |
| 2868 | |
| 2869 | memcpy(&ins->br_compact, &branch, sizeof(branch)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2870 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2871 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2872 | } |
| 2873 | |
| 2874 | ++br_block_idx; |
| 2875 | } |
| 2876 | |
| 2877 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 2878 | * sequence. Save instruction boundaries such that lookahead tags can |
| 2879 | * be assigned easily */ |
| 2880 | |
| 2881 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 2882 | |
| 2883 | int bundle_count = 0; |
| 2884 | mir_foreach_block(ctx, block) { |
| 2885 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 2886 | } |
| 2887 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 2888 | int bundle_idx = 0; |
| 2889 | mir_foreach_block(ctx, block) { |
| 2890 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 2891 | source_order_bundles[bundle_idx++] = bundle; |
| 2892 | } |
| 2893 | } |
| 2894 | |
| 2895 | int current_bundle = 0; |
| 2896 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 2897 | /* Midgard prefetches instruction types, so during emission we |
| 2898 | * need to lookahead. Unless this is the last instruction, in |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2899 | * which we return 1. */ |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 2900 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2901 | mir_foreach_block(ctx, block) { |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 2902 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2903 | int lookahead = 1; |
| 2904 | |
Alyssa Rosenzweig | 5bc62af | 2020-01-02 12:27:59 -0500 | [diff] [blame] | 2905 | if (!bundle->last_writeout && (current_bundle + 1 < bundle_count)) |
| 2906 | lookahead = source_order_bundles[current_bundle + 1]->tag; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2907 | |
| 2908 | emit_binary_bundle(ctx, bundle, compiled, lookahead); |
| 2909 | ++current_bundle; |
| 2910 | } |
| 2911 | |
| 2912 | /* TODO: Free deeper */ |
| 2913 | //util_dynarray_fini(&block->instructions); |
| 2914 | } |
| 2915 | |
| 2916 | free(source_order_bundles); |
| 2917 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2918 | /* Report the very first tag executed */ |
| 2919 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2920 | |
| 2921 | /* Deal with off-by-one related to the fencepost problem */ |
| 2922 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2923 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 2924 | |
| 2925 | program->blend_patch_offset = ctx->blend_constant_offset; |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 2926 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2927 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2928 | if (midgard_debug & MIDGARD_DBG_SHADERS) |
Alyssa Rosenzweig | ac14fac | 2019-11-07 09:31:02 -0500 | [diff] [blame] | 2929 | disassemble_midgard(program->compiled.data, program->compiled.size, gpu_id, ctx->stage); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2930 | |
Alyssa Rosenzweig | 35418f6 | 2019-12-16 18:05:21 -0500 | [diff] [blame] | 2931 | if (midgard_debug & MIDGARD_DBG_SHADERDB || shaderdb) { |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 2932 | unsigned nr_bundles = 0, nr_ins = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2933 | |
| 2934 | /* Count instructions and bundles */ |
| 2935 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2936 | mir_foreach_block(ctx, block) { |
| 2937 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2938 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 2939 | |
Alyssa Rosenzweig | 67909c8 | 2019-08-30 13:08:16 -0700 | [diff] [blame] | 2940 | mir_foreach_bundle_in_block(block, bun) |
| 2941 | nr_ins += bun->instruction_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2942 | } |
| 2943 | |
| 2944 | /* Calculate thread count. There are certain cutoffs by |
| 2945 | * register count for thread count */ |
| 2946 | |
| 2947 | unsigned nr_registers = program->work_register_count; |
| 2948 | |
| 2949 | unsigned nr_threads = |
| 2950 | (nr_registers <= 4) ? 4 : |
| 2951 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2952 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2953 | |
| 2954 | /* Dump stats */ |
| 2955 | |
| 2956 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2957 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2958 | "%u registers, %u threads, %u loops, " |
Alyssa Rosenzweig | 1a4153b | 2019-08-30 17:29:17 -0700 | [diff] [blame] | 2959 | "%u:%u spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2960 | SHADER_DB_COUNT++, |
| 2961 | gl_shader_stage_name(ctx->stage), |
Alyssa Rosenzweig | 19bceb5 | 2019-08-30 13:57:20 -0700 | [diff] [blame] | 2962 | nr_ins, nr_bundles, ctx->quadword_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2963 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2964 | ctx->loop_count, |
| 2965 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2966 | } |
| 2967 | |
Alyssa Rosenzweig | 4fa0932 | 2019-08-15 08:10:46 -0700 | [diff] [blame] | 2968 | ralloc_free(ctx); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2969 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2970 | return 0; |
| 2971 | } |