Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1 | /* |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 2 | * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io> |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 | * SOFTWARE. |
| 22 | */ |
| 23 | |
| 24 | #include <sys/types.h> |
| 25 | #include <sys/stat.h> |
| 26 | #include <sys/mman.h> |
| 27 | #include <fcntl.h> |
| 28 | #include <stdint.h> |
| 29 | #include <stdlib.h> |
| 30 | #include <stdio.h> |
| 31 | #include <err.h> |
| 32 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 33 | #include "main/mtypes.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 34 | #include "compiler/glsl/glsl_to_nir.h" |
| 35 | #include "compiler/nir_types.h" |
| 36 | #include "main/imports.h" |
| 37 | #include "compiler/nir/nir_builder.h" |
| 38 | #include "util/half_float.h" |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 39 | #include "util/u_math.h" |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 40 | #include "util/u_debug.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 41 | #include "util/u_dynarray.h" |
| 42 | #include "util/list.h" |
| 43 | #include "main/mtypes.h" |
| 44 | |
| 45 | #include "midgard.h" |
| 46 | #include "midgard_nir.h" |
| 47 | #include "midgard_compile.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 48 | #include "midgard_ops.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 49 | #include "helpers.h" |
Alyssa Rosenzweig | 1155446 | 2019-05-19 23:20:34 +0000 | [diff] [blame] | 50 | #include "compiler.h" |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 51 | |
| 52 | #include "disassemble.h" |
| 53 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 54 | static const struct debug_named_value debug_options[] = { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 55 | {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"}, |
| 56 | {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"}, |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 57 | {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"}, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 58 | DEBUG_NAMED_VALUE_END |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 59 | }; |
| 60 | |
| 61 | DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0) |
| 62 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 63 | unsigned SHADER_DB_COUNT = 0; |
| 64 | |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 65 | int midgard_debug = 0; |
| 66 | |
| 67 | #define DBG(fmt, ...) \ |
| 68 | do { if (midgard_debug & MIDGARD_DBG_MSGS) \ |
| 69 | fprintf(stderr, "%s:%d: "fmt, \ |
| 70 | __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0) |
| 71 | |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 72 | static bool |
| 73 | midgard_is_branch_unit(unsigned unit) |
| 74 | { |
| 75 | return (unit == ALU_ENAB_BRANCH) || (unit == ALU_ENAB_BR_COMPACT); |
| 76 | } |
| 77 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 78 | static void |
| 79 | midgard_block_add_successor(midgard_block *block, midgard_block *successor) |
| 80 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 81 | assert(block); |
| 82 | assert(successor); |
| 83 | |
| 84 | /* Deduplicate */ |
| 85 | for (unsigned i = 0; i < block->nr_successors; ++i) { |
| 86 | if (block->successors[i] == successor) |
| 87 | return; |
| 88 | } |
| 89 | |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 90 | block->successors[block->nr_successors++] = successor; |
| 91 | assert(block->nr_successors <= ARRAY_SIZE(block->successors)); |
| 92 | } |
| 93 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 94 | /* Helpers to generate midgard_instruction's using macro magic, since every |
| 95 | * driver seems to do it that way */ |
| 96 | |
| 97 | #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__)); |
Alyssa Rosenzweig | 56f9b47 | 2019-06-14 16:03:01 -0700 | [diff] [blame] | 98 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 99 | #define M_LOAD_STORE(name, store) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 100 | static midgard_instruction m_##name(unsigned ssa, unsigned address) { \ |
| 101 | midgard_instruction i = { \ |
| 102 | .type = TAG_LOAD_STORE_4, \ |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 103 | .mask = 0xF, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 104 | .ssa_args = { \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 105 | .dest = -1, \ |
| 106 | .src = { -1, -1, -1 }, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 107 | }, \ |
| 108 | .load_store = { \ |
| 109 | .op = midgard_op_##name, \ |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 110 | .swizzle = SWIZZLE_XYZW, \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 111 | .address = address \ |
| 112 | } \ |
| 113 | }; \ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 114 | \ |
| 115 | if (store) \ |
| 116 | i.ssa_args.src[0] = ssa; \ |
| 117 | else \ |
| 118 | i.ssa_args.dest = ssa; \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 119 | \ |
| 120 | return i; \ |
| 121 | } |
| 122 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 123 | #define M_LOAD(name) M_LOAD_STORE(name, false) |
| 124 | #define M_STORE(name) M_LOAD_STORE(name, true) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 125 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 126 | /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs |
| 127 | * the corresponding Midgard source */ |
| 128 | |
| 129 | static midgard_vector_alu_src |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 130 | vector_alu_modifiers(nir_alu_src *src, bool is_int, unsigned broadcast_count, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 131 | bool half, bool sext) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 132 | { |
| 133 | if (!src) return blank_alu_src; |
| 134 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 135 | /* Figure out how many components there are so we can adjust the |
| 136 | * swizzle. Specifically we want to broadcast the last channel so |
| 137 | * things like ball2/3 work |
| 138 | */ |
| 139 | |
| 140 | if (broadcast_count) { |
| 141 | uint8_t last_component = src->swizzle[broadcast_count - 1]; |
| 142 | |
| 143 | for (unsigned c = broadcast_count; c < NIR_MAX_VEC_COMPONENTS; ++c) { |
| 144 | src->swizzle[c] = last_component; |
| 145 | } |
| 146 | } |
| 147 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 148 | midgard_vector_alu_src alu_src = { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 149 | .rep_low = 0, |
| 150 | .rep_high = 0, |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 151 | .half = half, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 152 | .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle) |
| 153 | }; |
| 154 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 155 | if (is_int) { |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 156 | alu_src.mod = midgard_int_normal; |
| 157 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 158 | /* Sign/zero-extend if needed */ |
| 159 | |
| 160 | if (half) { |
| 161 | alu_src.mod = sext ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 162 | midgard_int_sign_extend |
| 163 | : midgard_int_zero_extend; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 164 | } |
| 165 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 166 | /* These should have been lowered away */ |
| 167 | assert(!(src->abs || src->negate)); |
| 168 | } else { |
| 169 | alu_src.mod = (src->abs << 0) | (src->negate << 1); |
| 170 | } |
| 171 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 172 | return alu_src; |
| 173 | } |
| 174 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 175 | /* load/store instructions have both 32-bit and 16-bit variants, depending on |
| 176 | * whether we are using vectors composed of highp or mediump. At the moment, we |
| 177 | * don't support half-floats -- this requires changes in other parts of the |
| 178 | * compiler -- therefore the 16-bit versions are commented out. */ |
| 179 | |
Alyssa Rosenzweig | 74ab80b | 2019-05-14 04:11:36 +0000 | [diff] [blame] | 180 | //M_LOAD(ld_attr_16); |
| 181 | M_LOAD(ld_attr_32); |
| 182 | //M_LOAD(ld_vary_16); |
| 183 | M_LOAD(ld_vary_32); |
Alyssa Rosenzweig | ec2f0b5 | 2019-08-13 08:51:40 -0700 | [diff] [blame^] | 184 | M_LOAD(ld_ubo_int4); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 185 | M_LOAD(ld_int4); |
| 186 | M_STORE(st_int4); |
Alyssa Rosenzweig | 74ab80b | 2019-05-14 04:11:36 +0000 | [diff] [blame] | 187 | M_LOAD(ld_color_buffer_8); |
| 188 | //M_STORE(st_vary_16); |
| 189 | M_STORE(st_vary_32); |
Alyssa Rosenzweig | be56840 | 2019-07-25 07:09:40 -0700 | [diff] [blame] | 190 | M_LOAD(st_cubemap_coords); |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 191 | M_LOAD(ld_compute_id); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 192 | |
| 193 | static midgard_instruction |
| 194 | v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond) |
| 195 | { |
| 196 | midgard_branch_cond branch = { |
| 197 | .op = op, |
| 198 | .dest_tag = tag, |
| 199 | .offset = offset, |
| 200 | .cond = cond |
| 201 | }; |
| 202 | |
| 203 | uint16_t compact; |
| 204 | memcpy(&compact, &branch, sizeof(branch)); |
| 205 | |
| 206 | midgard_instruction ins = { |
| 207 | .type = TAG_ALU_4, |
| 208 | .unit = ALU_ENAB_BR_COMPACT, |
| 209 | .prepacked_branch = true, |
| 210 | .compact_branch = true, |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 211 | .br_compact = compact, |
| 212 | .ssa_args = { |
| 213 | .dest = -1, |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 214 | .src = { -1, -1, -1 }, |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 215 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 216 | }; |
| 217 | |
| 218 | if (op == midgard_jmp_writeout_op_writeout) |
| 219 | ins.writeout = true; |
| 220 | |
| 221 | return ins; |
| 222 | } |
| 223 | |
| 224 | static midgard_instruction |
| 225 | v_branch(bool conditional, bool invert) |
| 226 | { |
| 227 | midgard_instruction ins = { |
| 228 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 229 | .unit = ALU_ENAB_BRANCH, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 230 | .compact_branch = true, |
| 231 | .branch = { |
| 232 | .conditional = conditional, |
| 233 | .invert_conditional = invert |
Alyssa Rosenzweig | 29416a8 | 2019-07-30 12:20:24 -0700 | [diff] [blame] | 234 | }, |
| 235 | .ssa_args = { |
| 236 | .dest = -1, |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 237 | .src = { -1, -1, -1 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 238 | } |
| 239 | }; |
| 240 | |
| 241 | return ins; |
| 242 | } |
| 243 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 244 | static midgard_branch_extended |
| 245 | midgard_create_branch_extended( midgard_condition cond, |
| 246 | midgard_jmp_writeout_op op, |
| 247 | unsigned dest_tag, |
| 248 | signed quadword_offset) |
| 249 | { |
Alyssa Rosenzweig | 13ee87c | 2019-07-29 09:15:32 -0700 | [diff] [blame] | 250 | /* The condition code is actually a LUT describing a function to |
| 251 | * combine multiple condition codes. However, we only support a single |
| 252 | * condition code at the moment, so we just duplicate over a bunch of |
| 253 | * times. */ |
| 254 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 255 | uint16_t duplicated_cond = |
| 256 | (cond << 14) | |
| 257 | (cond << 12) | |
| 258 | (cond << 10) | |
| 259 | (cond << 8) | |
| 260 | (cond << 6) | |
| 261 | (cond << 4) | |
| 262 | (cond << 2) | |
| 263 | (cond << 0); |
| 264 | |
| 265 | midgard_branch_extended branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 266 | .op = op, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 267 | .dest_tag = dest_tag, |
| 268 | .offset = quadword_offset, |
| 269 | .cond = duplicated_cond |
| 270 | }; |
| 271 | |
| 272 | return branch; |
| 273 | } |
| 274 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 275 | static void |
| 276 | attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name) |
| 277 | { |
| 278 | ins->has_constants = true; |
| 279 | memcpy(&ins->constants, constants, 16); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 280 | } |
| 281 | |
| 282 | static int |
Timothy Arceri | 035759b | 2019-03-29 12:39:48 +1100 | [diff] [blame] | 283 | glsl_type_size(const struct glsl_type *type, bool bindless) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 284 | { |
| 285 | return glsl_count_attribute_slots(type, false); |
| 286 | } |
| 287 | |
| 288 | /* Lower fdot2 to a vector multiplication followed by channel addition */ |
| 289 | static void |
| 290 | midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu) |
| 291 | { |
| 292 | if (alu->op != nir_op_fdot2) |
| 293 | return; |
| 294 | |
| 295 | b->cursor = nir_before_instr(&alu->instr); |
| 296 | |
| 297 | nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0); |
| 298 | nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1); |
| 299 | |
| 300 | nir_ssa_def *product = nir_fmul(b, src0, src1); |
| 301 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 302 | nir_ssa_def *sum = nir_fadd(b, |
| 303 | nir_channel(b, product, 0), |
| 304 | nir_channel(b, product, 1)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 305 | |
| 306 | /* Replace the fdot2 with this sum */ |
| 307 | nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum)); |
| 308 | } |
| 309 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 310 | static int |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 311 | midgard_sysval_for_ssbo(nir_intrinsic_instr *instr) |
| 312 | { |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 313 | /* This is way too meta */ |
| 314 | bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo; |
| 315 | unsigned idx_idx = is_store ? 1 : 0; |
| 316 | |
| 317 | nir_src index = instr->src[idx_idx]; |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 318 | assert(nir_src_is_const(index)); |
| 319 | uint32_t uindex = nir_src_as_uint(index); |
| 320 | |
| 321 | return PAN_SYSVAL(SSBO, uindex); |
| 322 | } |
| 323 | |
| 324 | static int |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 325 | midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr) |
| 326 | { |
| 327 | switch (instr->intrinsic) { |
| 328 | case nir_intrinsic_load_viewport_scale: |
| 329 | return PAN_SYSVAL_VIEWPORT_SCALE; |
| 330 | case nir_intrinsic_load_viewport_offset: |
| 331 | return PAN_SYSVAL_VIEWPORT_OFFSET; |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 332 | case nir_intrinsic_load_num_work_groups: |
| 333 | return PAN_SYSVAL_NUM_WORK_GROUPS; |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 334 | case nir_intrinsic_load_ssbo: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 335 | case nir_intrinsic_store_ssbo: |
Alyssa Rosenzweig | 2efa025 | 2019-08-01 11:03:15 -0700 | [diff] [blame] | 336 | return midgard_sysval_for_ssbo(instr); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 337 | default: |
| 338 | return -1; |
| 339 | } |
| 340 | } |
| 341 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 342 | static int sysval_for_instr(compiler_context *ctx, nir_instr *instr, |
| 343 | unsigned *dest) |
| 344 | { |
| 345 | nir_intrinsic_instr *intr; |
| 346 | nir_dest *dst = NULL; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 347 | nir_tex_instr *tex; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 348 | int sysval = -1; |
| 349 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 350 | bool is_store = false; |
| 351 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 352 | switch (instr->type) { |
| 353 | case nir_instr_type_intrinsic: |
| 354 | intr = nir_instr_as_intrinsic(instr); |
| 355 | sysval = midgard_nir_sysval_for_intrinsic(intr); |
| 356 | dst = &intr->dest; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 357 | is_store |= intr->intrinsic == nir_intrinsic_store_ssbo; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 358 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 359 | case nir_instr_type_tex: |
| 360 | tex = nir_instr_as_tex(instr); |
| 361 | if (tex->op != nir_texop_txs) |
| 362 | break; |
| 363 | |
| 364 | sysval = PAN_SYSVAL(TEXTURE_SIZE, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 365 | PAN_TXS_SYSVAL_ID(tex->texture_index, |
| 366 | nir_tex_instr_dest_size(tex) - |
| 367 | (tex->is_array ? 1 : 0), |
| 368 | tex->is_array)); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 369 | dst = &tex->dest; |
| 370 | break; |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 371 | default: |
| 372 | break; |
| 373 | } |
| 374 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 375 | if (dest && dst && !is_store) |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 376 | *dest = nir_dest_index(ctx, dst); |
| 377 | |
| 378 | return sysval; |
| 379 | } |
| 380 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 381 | static void |
| 382 | midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr) |
| 383 | { |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 384 | int sysval; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 385 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 386 | sysval = sysval_for_instr(ctx, instr, NULL); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 387 | if (sysval < 0) |
| 388 | return; |
| 389 | |
| 390 | /* We have a sysval load; check if it's already been assigned */ |
| 391 | |
| 392 | if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval)) |
| 393 | return; |
| 394 | |
| 395 | /* It hasn't -- so assign it now! */ |
| 396 | |
| 397 | unsigned id = ctx->sysval_count++; |
| 398 | _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1)); |
| 399 | ctx->sysvals[id] = sysval; |
| 400 | } |
| 401 | |
| 402 | static void |
| 403 | midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader) |
| 404 | { |
| 405 | ctx->sysval_count = 0; |
| 406 | |
| 407 | nir_foreach_function(function, shader) { |
| 408 | if (!function->impl) continue; |
| 409 | |
| 410 | nir_foreach_block(block, function->impl) { |
| 411 | nir_foreach_instr_safe(instr, block) { |
| 412 | midgard_nir_assign_sysval_body(ctx, instr); |
| 413 | } |
| 414 | } |
| 415 | } |
| 416 | } |
| 417 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 418 | static bool |
| 419 | midgard_nir_lower_fdot2(nir_shader *shader) |
| 420 | { |
| 421 | bool progress = false; |
| 422 | |
| 423 | nir_foreach_function(function, shader) { |
| 424 | if (!function->impl) continue; |
| 425 | |
| 426 | nir_builder _b; |
| 427 | nir_builder *b = &_b; |
| 428 | nir_builder_init(b, function->impl); |
| 429 | |
| 430 | nir_foreach_block(block, function->impl) { |
| 431 | nir_foreach_instr_safe(instr, block) { |
| 432 | if (instr->type != nir_instr_type_alu) continue; |
| 433 | |
| 434 | nir_alu_instr *alu = nir_instr_as_alu(instr); |
| 435 | midgard_nir_lower_fdot2_body(b, alu); |
| 436 | |
| 437 | progress |= true; |
| 438 | } |
| 439 | } |
| 440 | |
| 441 | nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); |
| 442 | |
| 443 | } |
| 444 | |
| 445 | return progress; |
| 446 | } |
| 447 | |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 448 | /* Flushes undefined values to zero */ |
| 449 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 450 | static void |
| 451 | optimise_nir(nir_shader *nir) |
| 452 | { |
| 453 | bool progress; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 454 | unsigned lower_flrp = |
| 455 | (nir->options->lower_flrp16 ? 16 : 0) | |
| 456 | (nir->options->lower_flrp32 ? 32 : 0) | |
| 457 | (nir->options->lower_flrp64 ? 64 : 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 458 | |
| 459 | NIR_PASS(progress, nir, nir_lower_regs_to_ssa); |
| 460 | NIR_PASS(progress, nir, midgard_nir_lower_fdot2); |
Alyssa Rosenzweig | c51312b | 2019-06-05 15:12:58 +0000 | [diff] [blame] | 461 | NIR_PASS(progress, nir, nir_lower_idiv); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 462 | |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 463 | nir_lower_tex_options lower_tex_1st_pass_options = { |
Alyssa Rosenzweig | 6ae4f9c | 2019-06-11 09:51:29 -0700 | [diff] [blame] | 464 | .lower_rect = true, |
| 465 | .lower_txp = ~0 |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 466 | }; |
| 467 | |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 468 | nir_lower_tex_options lower_tex_2nd_pass_options = { |
| 469 | .lower_txs_lod = true, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 470 | }; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 471 | |
| 472 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_1st_pass_options); |
| 473 | NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_2nd_pass_options); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 474 | |
| 475 | do { |
| 476 | progress = false; |
| 477 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 478 | NIR_PASS(progress, nir, nir_lower_var_copies); |
| 479 | NIR_PASS(progress, nir, nir_lower_vars_to_ssa); |
| 480 | |
| 481 | NIR_PASS(progress, nir, nir_copy_prop); |
| 482 | NIR_PASS(progress, nir, nir_opt_dce); |
| 483 | NIR_PASS(progress, nir, nir_opt_dead_cf); |
| 484 | NIR_PASS(progress, nir, nir_opt_cse); |
| 485 | NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true); |
| 486 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 487 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 488 | |
| 489 | if (lower_flrp != 0) { |
Ian Romanick | 1f1007a | 2019-05-08 07:32:43 -0700 | [diff] [blame] | 490 | bool lower_flrp_progress = false; |
Ian Romanick | d41cdef | 2018-08-18 16:42:04 -0700 | [diff] [blame] | 491 | NIR_PASS(lower_flrp_progress, |
| 492 | nir, |
| 493 | nir_lower_flrp, |
| 494 | lower_flrp, |
| 495 | false /* always_precise */, |
| 496 | nir->options->lower_ffma); |
| 497 | if (lower_flrp_progress) { |
| 498 | NIR_PASS(progress, nir, |
| 499 | nir_opt_constant_folding); |
| 500 | progress = true; |
| 501 | } |
| 502 | |
| 503 | /* Nothing should rematerialize any flrps, so we only |
| 504 | * need to do this lowering once. |
| 505 | */ |
| 506 | lower_flrp = 0; |
| 507 | } |
| 508 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 509 | NIR_PASS(progress, nir, nir_opt_undef); |
Alyssa Rosenzweig | a2f1a06 | 2019-07-08 12:40:34 -0700 | [diff] [blame] | 510 | NIR_PASS(progress, nir, nir_undef_to_zero); |
| 511 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 512 | NIR_PASS(progress, nir, nir_opt_loop_unroll, |
| 513 | nir_var_shader_in | |
| 514 | nir_var_shader_out | |
| 515 | nir_var_function_temp); |
| 516 | |
Alyssa Rosenzweig | 9402970 | 2019-06-17 11:12:51 -0700 | [diff] [blame] | 517 | NIR_PASS(progress, nir, nir_opt_vectorize); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 518 | } while (progress); |
| 519 | |
| 520 | /* Must be run at the end to prevent creation of fsin/fcos ops */ |
| 521 | NIR_PASS(progress, nir, midgard_nir_scale_trig); |
| 522 | |
| 523 | do { |
| 524 | progress = false; |
| 525 | |
| 526 | NIR_PASS(progress, nir, nir_opt_dce); |
| 527 | NIR_PASS(progress, nir, nir_opt_algebraic); |
| 528 | NIR_PASS(progress, nir, nir_opt_constant_folding); |
| 529 | NIR_PASS(progress, nir, nir_copy_prop); |
| 530 | } while (progress); |
| 531 | |
| 532 | NIR_PASS(progress, nir, nir_opt_algebraic_late); |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 533 | |
| 534 | /* We implement booleans as 32-bit 0/~0 */ |
| 535 | NIR_PASS(progress, nir, nir_lower_bool_to_int32); |
| 536 | |
| 537 | /* Now that booleans are lowered, we can run out late opts */ |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 538 | NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 539 | |
Alyssa Rosenzweig | effe6fb0 | 2019-03-25 02:49:04 +0000 | [diff] [blame] | 540 | /* Lower mods for float ops only. Integer ops don't support modifiers |
| 541 | * (saturate doesn't make sense on integers, neg/abs require dedicated |
| 542 | * instructions) */ |
| 543 | |
| 544 | NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 545 | NIR_PASS(progress, nir, nir_copy_prop); |
| 546 | NIR_PASS(progress, nir, nir_opt_dce); |
| 547 | |
| 548 | /* Take us out of SSA */ |
| 549 | NIR_PASS(progress, nir, nir_lower_locals_to_regs); |
| 550 | NIR_PASS(progress, nir, nir_convert_from_ssa, true); |
| 551 | |
| 552 | /* We are a vector architecture; write combine where possible */ |
| 553 | NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest); |
| 554 | NIR_PASS(progress, nir, nir_lower_vec_to_movs); |
| 555 | |
| 556 | NIR_PASS(progress, nir, nir_opt_dce); |
| 557 | } |
| 558 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 559 | /* Do not actually emit a load; instead, cache the constant for inlining */ |
| 560 | |
| 561 | static void |
| 562 | emit_load_const(compiler_context *ctx, nir_load_const_instr *instr) |
| 563 | { |
| 564 | nir_ssa_def def = instr->def; |
| 565 | |
Tomeu Vizoso | 554975b | 2019-05-07 17:28:36 +0200 | [diff] [blame] | 566 | float *v = rzalloc_array(NULL, float, 4); |
Karol Herbst | 14531d6 | 2019-03-27 00:59:03 +0100 | [diff] [blame] | 567 | nir_const_load_to_arr(v, instr, f32); |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 568 | |
| 569 | /* Shifted for SSA, +1 for off-by-one */ |
| 570 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, v); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 571 | } |
| 572 | |
Alyssa Rosenzweig | e169301 | 2019-07-24 12:52:27 -0700 | [diff] [blame] | 573 | /* Normally constants are embedded implicitly, but for I/O and such we have to |
| 574 | * explicitly emit a move with the constant source */ |
| 575 | |
| 576 | static void |
| 577 | emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to) |
| 578 | { |
| 579 | void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1); |
| 580 | |
| 581 | if (constant_value) { |
| 582 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, to); |
| 583 | attach_constants(ctx, &ins, constant_value, node + 1); |
| 584 | emit_mir_instruction(ctx, ins); |
| 585 | } |
| 586 | } |
| 587 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 588 | static bool |
| 589 | nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components) |
| 590 | { |
| 591 | unsigned comp = src->swizzle[0]; |
| 592 | |
| 593 | for (unsigned c = 1; c < nr_components; ++c) { |
| 594 | if (src->swizzle[c] != comp) |
| 595 | return true; |
| 596 | } |
| 597 | |
| 598 | return false; |
| 599 | } |
| 600 | |
| 601 | /* Midgard puts scalar conditionals in r31.w; move an arbitrary source (the |
| 602 | * output of a conditional test) into that register */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 603 | |
| 604 | static void |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 605 | emit_condition(compiler_context *ctx, nir_src *src, bool for_branch, unsigned component) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 606 | { |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 607 | int condition = nir_src_index(ctx, src); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 608 | |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 609 | /* Source to swizzle the desired component into w */ |
| 610 | |
| 611 | const midgard_vector_alu_src alu_src = { |
| 612 | .swizzle = SWIZZLE(component, component, component, component), |
| 613 | }; |
| 614 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 615 | /* There is no boolean move instruction. Instead, we simulate a move by |
| 616 | * ANDing the condition with itself to get it into r31.w */ |
| 617 | |
| 618 | midgard_instruction ins = { |
| 619 | .type = TAG_ALU_4, |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 620 | |
| 621 | /* We need to set the conditional as close as possible */ |
| 622 | .precede_break = true, |
| 623 | .unit = for_branch ? UNIT_SMUL : UNIT_SADD, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 624 | .mask = 1 << COMPONENT_W, |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 625 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 626 | .ssa_args = { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 627 | .src = { condition, condition, -1 }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 628 | .dest = SSA_FIXED_REGISTER(31), |
| 629 | }, |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 630 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 631 | .alu = { |
| 632 | .op = midgard_alu_op_iand, |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 633 | .outmod = midgard_outmod_int_wrap, |
Alyssa Rosenzweig | 576a27f | 2019-04-30 02:19:26 +0000 | [diff] [blame] | 634 | .reg_mode = midgard_reg_mode_32, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 635 | .dest_override = midgard_dest_override_none, |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 636 | .src1 = vector_alu_srco_unsigned(alu_src), |
| 637 | .src2 = vector_alu_srco_unsigned(alu_src) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 638 | }, |
| 639 | }; |
| 640 | |
| 641 | emit_mir_instruction(ctx, ins); |
| 642 | } |
| 643 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 644 | /* Or, for mixed conditions (with csel_v), here's a vector version using all of |
| 645 | * r31 instead */ |
| 646 | |
| 647 | static void |
| 648 | emit_condition_mixed(compiler_context *ctx, nir_alu_src *src, unsigned nr_comp) |
| 649 | { |
| 650 | int condition = nir_src_index(ctx, &src->src); |
| 651 | |
| 652 | /* Source to swizzle the desired component into w */ |
| 653 | |
| 654 | const midgard_vector_alu_src alu_src = { |
| 655 | .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle), |
| 656 | }; |
| 657 | |
| 658 | /* There is no boolean move instruction. Instead, we simulate a move by |
| 659 | * ANDing the condition with itself to get it into r31.w */ |
| 660 | |
| 661 | midgard_instruction ins = { |
| 662 | .type = TAG_ALU_4, |
| 663 | .precede_break = true, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 664 | .mask = mask_of(nr_comp), |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 665 | .ssa_args = { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 666 | .src = { condition, condition, -1 }, |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 667 | .dest = SSA_FIXED_REGISTER(31), |
| 668 | }, |
| 669 | .alu = { |
| 670 | .op = midgard_alu_op_iand, |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 671 | .outmod = midgard_outmod_int_wrap, |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 672 | .reg_mode = midgard_reg_mode_32, |
| 673 | .dest_override = midgard_dest_override_none, |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 674 | .src1 = vector_alu_srco_unsigned(alu_src), |
| 675 | .src2 = vector_alu_srco_unsigned(alu_src) |
| 676 | }, |
| 677 | }; |
| 678 | |
| 679 | emit_mir_instruction(ctx, ins); |
| 680 | } |
| 681 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 682 | #define ALU_CASE(nir, _op) \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 683 | case nir_op_##nir: \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 684 | op = midgard_alu_op_##_op; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 685 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 686 | break; |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 687 | |
| 688 | #define ALU_CASE_BCAST(nir, _op, count) \ |
| 689 | case nir_op_##nir: \ |
| 690 | op = midgard_alu_op_##_op; \ |
| 691 | broadcast_swizzle = count; \ |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 692 | assert(src_bitsize == dst_bitsize); \ |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 693 | break; |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 694 | static bool |
| 695 | nir_is_fzero_constant(nir_src src) |
| 696 | { |
| 697 | if (!nir_src_is_const(src)) |
| 698 | return false; |
| 699 | |
| 700 | for (unsigned c = 0; c < nir_src_num_components(src); ++c) { |
| 701 | if (nir_src_comp_as_float(src, c) != 0.0) |
| 702 | return false; |
| 703 | } |
| 704 | |
| 705 | return true; |
| 706 | } |
| 707 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 708 | /* Analyze the sizes of the inputs to determine which reg mode. Ops needed |
| 709 | * special treatment override this anyway. */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 710 | |
| 711 | static midgard_reg_mode |
| 712 | reg_mode_for_nir(nir_alu_instr *instr) |
| 713 | { |
| 714 | unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 715 | |
| 716 | switch (src_bitsize) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 717 | case 8: |
| 718 | return midgard_reg_mode_8; |
| 719 | case 16: |
| 720 | return midgard_reg_mode_16; |
| 721 | case 32: |
| 722 | return midgard_reg_mode_32; |
| 723 | case 64: |
| 724 | return midgard_reg_mode_64; |
| 725 | default: |
| 726 | unreachable("Invalid bit size"); |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 727 | } |
| 728 | } |
| 729 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 730 | static void |
| 731 | emit_alu(compiler_context *ctx, nir_alu_instr *instr) |
| 732 | { |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 733 | /* Derivatives end up emitted on the texture pipe, not the ALUs. This |
| 734 | * is handled elsewhere */ |
| 735 | |
| 736 | if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) { |
| 737 | midgard_emit_derivatives(ctx, instr); |
| 738 | return; |
| 739 | } |
| 740 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 741 | bool is_ssa = instr->dest.dest.is_ssa; |
| 742 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 743 | unsigned dest = nir_dest_index(ctx, &instr->dest.dest); |
Alyssa Rosenzweig | f42e5be | 2019-07-01 15:28:37 -0700 | [diff] [blame] | 744 | unsigned nr_components = nir_dest_num_components(instr->dest.dest); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 745 | unsigned nr_inputs = nir_op_infos[instr->op].num_inputs; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 746 | |
| 747 | /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are |
| 748 | * supported. A few do not and are commented for now. Also, there are a |
| 749 | * number of NIR ops which Midgard does not support and need to be |
| 750 | * lowered, also TODO. This switch block emits the opcode and calling |
| 751 | * convention of the Midgard instruction; actual packing is done in |
| 752 | * emit_alu below */ |
| 753 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 754 | unsigned op; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 755 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 756 | /* Number of components valid to check for the instruction (the rest |
| 757 | * will be forced to the last), or 0 to use as-is. Relevant as |
| 758 | * ball-type instructions have a channel count in NIR but are all vec4 |
| 759 | * in Midgard */ |
| 760 | |
| 761 | unsigned broadcast_swizzle = 0; |
| 762 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 763 | /* What register mode should we operate in? */ |
| 764 | midgard_reg_mode reg_mode = |
| 765 | reg_mode_for_nir(instr); |
| 766 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 767 | /* Do we need a destination override? Used for inline |
| 768 | * type conversion */ |
| 769 | |
| 770 | midgard_dest_override dest_override = |
| 771 | midgard_dest_override_none; |
| 772 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 773 | /* Should we use a smaller respective source and sign-extend? */ |
| 774 | |
| 775 | bool half_1 = false, sext_1 = false; |
| 776 | bool half_2 = false, sext_2 = false; |
| 777 | |
Alyssa Rosenzweig | 0ed8cca | 2019-07-01 17:35:25 -0700 | [diff] [blame] | 778 | unsigned src_bitsize = nir_src_bit_size(instr->src[0].src); |
| 779 | unsigned dst_bitsize = nir_dest_bit_size(instr->dest.dest); |
| 780 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 781 | switch (instr->op) { |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 782 | ALU_CASE(fadd, fadd); |
| 783 | ALU_CASE(fmul, fmul); |
| 784 | ALU_CASE(fmin, fmin); |
| 785 | ALU_CASE(fmax, fmax); |
| 786 | ALU_CASE(imin, imin); |
| 787 | ALU_CASE(imax, imax); |
Alyssa Rosenzweig | 2e7555b | 2019-04-05 05:16:54 +0000 | [diff] [blame] | 788 | ALU_CASE(umin, umin); |
| 789 | ALU_CASE(umax, umax); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 790 | ALU_CASE(ffloor, ffloor); |
Alyssa Rosenzweig | c6be996 | 2019-02-23 01:12:10 +0000 | [diff] [blame] | 791 | ALU_CASE(fround_even, froundeven); |
| 792 | ALU_CASE(ftrunc, ftrunc); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 793 | ALU_CASE(fceil, fceil); |
| 794 | ALU_CASE(fdot3, fdot3); |
| 795 | ALU_CASE(fdot4, fdot4); |
| 796 | ALU_CASE(iadd, iadd); |
| 797 | ALU_CASE(isub, isub); |
| 798 | ALU_CASE(imul, imul); |
Alyssa Rosenzweig | 9f14e20 | 2019-06-05 15:18:35 +0000 | [diff] [blame] | 799 | |
| 800 | /* Zero shoved as second-arg */ |
| 801 | ALU_CASE(iabs, iabsdiff); |
| 802 | |
Jason Ekstrand | f2dc0f2 | 2019-05-06 11:45:46 -0500 | [diff] [blame] | 803 | ALU_CASE(mov, imov); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 804 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 805 | ALU_CASE(feq32, feq); |
| 806 | ALU_CASE(fne32, fne); |
| 807 | ALU_CASE(flt32, flt); |
| 808 | ALU_CASE(ieq32, ieq); |
| 809 | ALU_CASE(ine32, ine); |
| 810 | ALU_CASE(ilt32, ilt); |
Alyssa Rosenzweig | b8739c2 | 2019-03-26 04:00:33 +0000 | [diff] [blame] | 811 | ALU_CASE(ult32, ult); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 812 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 813 | /* We don't have a native b2f32 instruction. Instead, like many |
| 814 | * GPUs, we exploit booleans as 0/~0 for false/true, and |
| 815 | * correspondingly AND |
| 816 | * by 1.0 to do the type conversion. For the moment, prime us |
| 817 | * to emit: |
| 818 | * |
| 819 | * iand [whatever], #0 |
| 820 | * |
| 821 | * At the end of emit_alu (as MIR), we'll fix-up the constant |
| 822 | */ |
| 823 | |
| 824 | ALU_CASE(b2f32, iand); |
| 825 | ALU_CASE(b2i32, iand); |
| 826 | |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 827 | /* Likewise, we don't have a dedicated f2b32 instruction, but |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 828 | * we can do a "not equal to 0.0" test. */ |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 829 | |
| 830 | ALU_CASE(f2b32, fne); |
Alyssa Rosenzweig | 5b95fef | 2019-03-25 00:56:48 +0000 | [diff] [blame] | 831 | ALU_CASE(i2b32, ine); |
Alyssa Rosenzweig | ae43b8f | 2019-03-25 00:53:46 +0000 | [diff] [blame] | 832 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 833 | ALU_CASE(frcp, frcp); |
| 834 | ALU_CASE(frsq, frsqrt); |
| 835 | ALU_CASE(fsqrt, fsqrt); |
| 836 | ALU_CASE(fexp2, fexp2); |
| 837 | ALU_CASE(flog2, flog2); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 838 | |
Alyssa Rosenzweig | 73bf669 | 2019-06-05 15:03:02 -0700 | [diff] [blame] | 839 | ALU_CASE(f2i32, f2i_rtz); |
| 840 | ALU_CASE(f2u32, f2u_rtz); |
| 841 | ALU_CASE(i2f32, i2f_rtz); |
| 842 | ALU_CASE(u2f32, u2f_rtz); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 843 | |
Alyssa Rosenzweig | d8c084d | 2019-07-01 17:41:20 -0700 | [diff] [blame] | 844 | ALU_CASE(f2i16, f2i_rtz); |
| 845 | ALU_CASE(f2u16, f2u_rtz); |
| 846 | ALU_CASE(i2f16, i2f_rtz); |
| 847 | ALU_CASE(u2f16, u2f_rtz); |
| 848 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 849 | ALU_CASE(fsin, fsin); |
| 850 | ALU_CASE(fcos, fcos); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 851 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 852 | /* We'll set invert */ |
| 853 | ALU_CASE(inot, imov); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 854 | ALU_CASE(iand, iand); |
| 855 | ALU_CASE(ior, ior); |
| 856 | ALU_CASE(ixor, ixor); |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 857 | ALU_CASE(ishl, ishl); |
| 858 | ALU_CASE(ishr, iasr); |
| 859 | ALU_CASE(ushr, ilsr); |
| 860 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 861 | ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2); |
| 862 | ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 863 | ALU_CASE(b32all_fequal4, fball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 864 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 865 | ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2); |
| 866 | ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 867 | ALU_CASE(b32any_fnequal4, fbany_neq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 868 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 869 | ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2); |
| 870 | ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 871 | ALU_CASE(b32all_iequal4, iball_eq); |
Alyssa Rosenzweig | 5366410 | 2019-03-25 00:12:06 +0000 | [diff] [blame] | 872 | |
Alyssa Rosenzweig | 195e297 | 2019-06-19 07:23:27 -0700 | [diff] [blame] | 873 | ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2); |
| 874 | ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3); |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 875 | ALU_CASE(b32any_inequal4, ibany_neq); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 876 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 877 | /* Source mods will be shoved in later */ |
| 878 | ALU_CASE(fabs, fmov); |
| 879 | ALU_CASE(fneg, fmov); |
| 880 | ALU_CASE(fsat, fmov); |
| 881 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 882 | /* For size conversion, we use a move. Ideally though we would squash |
| 883 | * these ops together; maybe that has to happen after in NIR as part of |
| 884 | * propagation...? An earlier algebraic pass ensured we step down by |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 885 | * only / exactly one size. If stepping down, we use a dest override to |
| 886 | * reduce the size; if stepping up, we use a larger-sized move with a |
| 887 | * half source and a sign/zero-extension modifier */ |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 888 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 889 | case nir_op_i2i8: |
| 890 | case nir_op_i2i16: |
| 891 | case nir_op_i2i32: |
| 892 | /* If we end up upscale, we'll need a sign-extend on the |
| 893 | * operand (the second argument) */ |
| 894 | |
| 895 | sext_2 = true; |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 896 | case nir_op_u2u8: |
| 897 | case nir_op_u2u16: |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 898 | case nir_op_u2u32: { |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 899 | op = midgard_alu_op_imov; |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 900 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 901 | if (dst_bitsize == (src_bitsize * 2)) { |
| 902 | /* Converting up */ |
| 903 | half_2 = true; |
| 904 | |
| 905 | /* Use a greater register mode */ |
| 906 | reg_mode++; |
| 907 | } else if (src_bitsize == (dst_bitsize * 2)) { |
| 908 | /* Converting down */ |
| 909 | dest_override = midgard_dest_override_lower; |
| 910 | } |
| 911 | |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 912 | break; |
| 913 | } |
| 914 | |
Alyssa Rosenzweig | 954c6af | 2019-07-01 17:38:26 -0700 | [diff] [blame] | 915 | case nir_op_f2f16: { |
| 916 | assert(src_bitsize == 32); |
| 917 | |
| 918 | op = midgard_alu_op_fmov; |
| 919 | dest_override = midgard_dest_override_lower; |
| 920 | break; |
| 921 | } |
| 922 | |
| 923 | case nir_op_f2f32: { |
| 924 | assert(src_bitsize == 16); |
| 925 | |
| 926 | op = midgard_alu_op_fmov; |
| 927 | half_2 = true; |
| 928 | reg_mode++; |
| 929 | break; |
| 930 | } |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 931 | |
Alyssa Rosenzweig | 954c6af | 2019-07-01 17:38:26 -0700 | [diff] [blame] | 932 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 933 | /* For greater-or-equal, we lower to less-or-equal and flip the |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 934 | * arguments */ |
| 935 | |
Alyssa Rosenzweig | 7b78af8 | 2019-03-26 04:01:33 +0000 | [diff] [blame] | 936 | case nir_op_fge: |
| 937 | case nir_op_fge32: |
| 938 | case nir_op_ige32: |
| 939 | case nir_op_uge32: { |
| 940 | op = |
| 941 | instr->op == nir_op_fge ? midgard_alu_op_fle : |
| 942 | instr->op == nir_op_fge32 ? midgard_alu_op_fle : |
| 943 | instr->op == nir_op_ige32 ? midgard_alu_op_ile : |
| 944 | instr->op == nir_op_uge32 ? midgard_alu_op_ule : |
| 945 | 0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 946 | |
| 947 | /* Swap via temporary */ |
| 948 | nir_alu_src temp = instr->src[1]; |
| 949 | instr->src[1] = instr->src[0]; |
| 950 | instr->src[0] = temp; |
| 951 | |
| 952 | break; |
| 953 | } |
| 954 | |
Alyssa Rosenzweig | 3fb8842 | 2019-03-25 00:25:01 +0000 | [diff] [blame] | 955 | case nir_op_b32csel: { |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 956 | /* Midgard features both fcsel and icsel, depending on |
| 957 | * the type of the arguments/output. However, as long |
| 958 | * as we're careful we can _always_ use icsel and |
| 959 | * _never_ need fcsel, since the latter does additional |
| 960 | * floating-point-specific processing whereas the |
| 961 | * former just moves bits on the wire. It's not obvious |
| 962 | * why these are separate opcodes, save for the ability |
| 963 | * to do things like sat/pos/abs/neg for free */ |
Alyssa Rosenzweig | 3d7874c | 2019-05-03 01:54:16 +0000 | [diff] [blame] | 964 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 965 | bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components); |
| 966 | op = mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 967 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 968 | /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */ |
| 969 | nr_inputs = 2; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 970 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 971 | /* Emit the condition into r31 */ |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 972 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 973 | if (mixed) |
| 974 | emit_condition_mixed(ctx, &instr->src[0], nr_components); |
| 975 | else |
| 976 | emit_condition(ctx, &instr->src[0].src, false, instr->src[0].swizzle[0]); |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 977 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 978 | /* The condition is the first argument; move the other |
| 979 | * arguments up one to be a binary instruction for |
| 980 | * Midgard */ |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 981 | |
Alyssa Rosenzweig | 726f026 | 2019-05-07 02:52:08 +0000 | [diff] [blame] | 982 | memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 983 | break; |
| 984 | } |
| 985 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 986 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 987 | DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 988 | assert(0); |
| 989 | return; |
| 990 | } |
| 991 | |
Alyssa Rosenzweig | 0a13bab | 2019-05-15 01:16:51 +0000 | [diff] [blame] | 992 | /* Midgard can perform certain modifiers on output of an ALU op */ |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 993 | unsigned outmod; |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 994 | |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 995 | if (midgard_is_integer_out_op(op)) { |
| 996 | outmod = midgard_outmod_int_wrap; |
| 997 | } else { |
| 998 | bool sat = instr->dest.saturate || instr->op == nir_op_fsat; |
| 999 | outmod = sat ? midgard_outmod_sat : midgard_outmod_none; |
| 1000 | } |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1001 | |
Alyssa Rosenzweig | 7bc91b4 | 2019-04-24 23:42:30 +0000 | [diff] [blame] | 1002 | /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */ |
| 1003 | |
| 1004 | if (instr->op == nir_op_fmax) { |
| 1005 | if (nir_is_fzero_constant(instr->src[0].src)) { |
| 1006 | op = midgard_alu_op_fmov; |
| 1007 | nr_inputs = 1; |
| 1008 | outmod = midgard_outmod_pos; |
| 1009 | instr->src[0] = instr->src[1]; |
| 1010 | } else if (nir_is_fzero_constant(instr->src[1].src)) { |
| 1011 | op = midgard_alu_op_fmov; |
| 1012 | nr_inputs = 1; |
| 1013 | outmod = midgard_outmod_pos; |
| 1014 | } |
| 1015 | } |
| 1016 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1017 | /* Fetch unit, quirks, etc information */ |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 1018 | unsigned opcode_props = alu_opcode_props[op].props; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1019 | bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1020 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1021 | /* src0 will always exist afaik, but src1 will not for 1-argument |
| 1022 | * instructions. The latter can only be fetched if the instruction |
| 1023 | * needs it, or else we may segfault. */ |
| 1024 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1025 | unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]); |
| 1026 | unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : SSA_UNUSED_0; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1027 | |
| 1028 | /* Rather than use the instruction generation helpers, we do it |
| 1029 | * ourselves here to avoid the mess */ |
| 1030 | |
| 1031 | midgard_instruction ins = { |
| 1032 | .type = TAG_ALU_4, |
| 1033 | .ssa_args = { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1034 | .src = { |
| 1035 | quirk_flipped_r24 ? SSA_UNUSED_1 : src0, |
| 1036 | quirk_flipped_r24 ? src0 : src1, |
| 1037 | -1 |
| 1038 | }, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1039 | .dest = dest, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1040 | } |
| 1041 | }; |
| 1042 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1043 | nir_alu_src *nirmods[2] = { NULL }; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1044 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1045 | if (nr_inputs == 2) { |
| 1046 | nirmods[0] = &instr->src[0]; |
| 1047 | nirmods[1] = &instr->src[1]; |
| 1048 | } else if (nr_inputs == 1) { |
| 1049 | nirmods[quirk_flipped_r24] = &instr->src[0]; |
| 1050 | } else { |
| 1051 | assert(0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1052 | } |
| 1053 | |
Alyssa Rosenzweig | 659aa3d | 2019-05-26 03:16:37 +0000 | [diff] [blame] | 1054 | /* These were lowered to a move, so apply the corresponding mod */ |
| 1055 | |
| 1056 | if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) { |
| 1057 | nir_alu_src *s = nirmods[quirk_flipped_r24]; |
| 1058 | |
| 1059 | if (instr->op == nir_op_fneg) |
| 1060 | s->negate = !s->negate; |
| 1061 | |
| 1062 | if (instr->op == nir_op_fabs) |
| 1063 | s->abs = !s->abs; |
| 1064 | } |
| 1065 | |
Alyssa Rosenzweig | fcdfb67 | 2019-04-22 03:25:42 +0000 | [diff] [blame] | 1066 | bool is_int = midgard_is_integer_op(op); |
| 1067 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1068 | ins.mask = mask_of(nr_components); |
| 1069 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1070 | midgard_vector_alu alu = { |
| 1071 | .op = op, |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1072 | .reg_mode = reg_mode, |
Alyssa Rosenzweig | 4df80ca | 2019-07-01 15:26:22 -0700 | [diff] [blame] | 1073 | .dest_override = dest_override, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1074 | .outmod = outmod, |
| 1075 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1076 | .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, sext_1)), |
| 1077 | .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int, broadcast_swizzle, half_2, sext_2)), |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1078 | }; |
| 1079 | |
| 1080 | /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */ |
| 1081 | |
| 1082 | if (!is_ssa) |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1083 | ins.mask &= instr->dest.write_mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1084 | |
| 1085 | ins.alu = alu; |
| 1086 | |
| 1087 | /* Late fixup for emulated instructions */ |
| 1088 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1089 | if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1090 | /* Presently, our second argument is an inline #0 constant. |
| 1091 | * Switch over to an embedded 1.0 constant (that can't fit |
| 1092 | * inline, since we're 32-bit, not 16-bit like the inline |
| 1093 | * constants) */ |
| 1094 | |
| 1095 | ins.ssa_args.inline_constant = false; |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1096 | ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1097 | ins.has_constants = true; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1098 | |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1099 | if (instr->op == nir_op_b2f32) { |
| 1100 | ins.constants[0] = 1.0f; |
| 1101 | } else { |
| 1102 | /* Type pun it into place */ |
| 1103 | uint32_t one = 0x1; |
| 1104 | memcpy(&ins.constants[0], &one, sizeof(uint32_t)); |
| 1105 | } |
| 1106 | |
| 1107 | ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx); |
Alyssa Rosenzweig | 88c5979 | 2019-06-05 15:24:51 +0000 | [diff] [blame] | 1108 | } else if (nr_inputs == 1 && !quirk_flipped_r24) { |
| 1109 | /* Lots of instructions need a 0 plonked in */ |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1110 | ins.ssa_args.inline_constant = false; |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1111 | ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | 3208c9d | 2019-03-25 01:13:12 +0000 | [diff] [blame] | 1112 | ins.has_constants = true; |
| 1113 | ins.constants[0] = 0.0f; |
Alyssa Rosenzweig | 9da4603 | 2019-03-24 16:07:31 +0000 | [diff] [blame] | 1114 | ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx); |
Alyssa Rosenzweig | bcabcfe | 2019-04-25 04:25:33 +0000 | [diff] [blame] | 1115 | } else if (instr->op == nir_op_inot) { |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 1116 | ins.invert = true; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1117 | } |
| 1118 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1119 | if ((opcode_props & UNITS_ALL) == UNIT_VLUT) { |
| 1120 | /* To avoid duplicating the lookup tables (probably), true LUT |
| 1121 | * instructions can only operate as if they were scalars. Lower |
| 1122 | * them here by changing the component. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1123 | |
| 1124 | uint8_t original_swizzle[4]; |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1125 | memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle)); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1126 | unsigned orig_mask = ins.mask; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1127 | |
| 1128 | for (int i = 0; i < nr_components; ++i) { |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1129 | /* Mask the associated component, dropping the |
| 1130 | * instruction if needed */ |
| 1131 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1132 | ins.mask = 1 << i; |
| 1133 | ins.mask &= orig_mask; |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1134 | |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1135 | if (!ins.mask) |
Alyssa Rosenzweig | 2c9e124 | 2019-06-17 11:49:44 -0700 | [diff] [blame] | 1136 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1137 | |
| 1138 | for (int j = 0; j < 4; ++j) |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1139 | nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1140 | |
Alyssa Rosenzweig | 7f807ef | 2019-07-01 16:44:00 -0700 | [diff] [blame] | 1141 | ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, false)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1142 | emit_mir_instruction(ctx, ins); |
| 1143 | } |
| 1144 | } else { |
| 1145 | emit_mir_instruction(ctx, ins); |
| 1146 | } |
| 1147 | } |
| 1148 | |
Alyssa Rosenzweig | 97dcad8 | 2019-02-07 03:39:25 +0000 | [diff] [blame] | 1149 | #undef ALU_CASE |
| 1150 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1151 | /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly |
| 1152 | * optimized) versions of UBO #0 */ |
| 1153 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1154 | midgard_instruction * |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1155 | emit_ubo_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1156 | compiler_context *ctx, |
| 1157 | unsigned dest, |
| 1158 | unsigned offset, |
| 1159 | nir_src *indirect_offset, |
| 1160 | unsigned index) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1161 | { |
| 1162 | /* TODO: half-floats */ |
| 1163 | |
Alyssa Rosenzweig | ec2f0b5 | 2019-08-13 08:51:40 -0700 | [diff] [blame^] | 1164 | midgard_instruction ins = m_ld_ubo_int4(dest, offset); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1165 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1166 | /* TODO: Don't split */ |
| 1167 | ins.load_store.varying_parameters = (offset & 7) << 7; |
| 1168 | ins.load_store.address = offset >> 3; |
| 1169 | |
| 1170 | if (indirect_offset) { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1171 | ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1172 | ins.load_store.arg_2 = 0x80; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1173 | } else { |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1174 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1175 | } |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 1176 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1177 | ins.load_store.arg_1 = index; |
| 1178 | |
Alyssa Rosenzweig | e7ac46b | 2019-08-02 17:09:54 -0700 | [diff] [blame] | 1179 | return emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1180 | } |
| 1181 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1182 | /* SSBO reads are like UBO reads if you squint */ |
| 1183 | |
| 1184 | static void |
| 1185 | emit_ssbo_access( |
| 1186 | compiler_context *ctx, |
| 1187 | nir_instr *instr, |
| 1188 | bool is_read, |
| 1189 | unsigned srcdest, |
| 1190 | unsigned offset, |
| 1191 | nir_src *indirect_offset, |
| 1192 | unsigned index) |
| 1193 | { |
| 1194 | /* TODO: types */ |
| 1195 | |
| 1196 | midgard_instruction ins; |
| 1197 | |
| 1198 | if (is_read) |
| 1199 | ins = m_ld_int4(srcdest, offset); |
| 1200 | else |
| 1201 | ins = m_st_int4(srcdest, offset); |
| 1202 | |
| 1203 | /* SSBO reads use a generic memory read interface, so we need the |
| 1204 | * address of the SSBO as the first argument. This is a sysval. */ |
| 1205 | |
| 1206 | unsigned addr = make_compiler_temp(ctx); |
| 1207 | emit_sysval_read(ctx, instr, addr, 2); |
| 1208 | |
| 1209 | /* The source array is a bit of a leaky abstraction for SSBOs. |
| 1210 | * Nevertheless, for loads: |
| 1211 | * |
| 1212 | * src[0] = arg_1 |
| 1213 | * src[1] = arg_2 |
| 1214 | * src[2] = unused |
| 1215 | * |
| 1216 | * Whereas for stores: |
| 1217 | * |
| 1218 | * src[0] = value |
| 1219 | * src[1] = arg_1 |
| 1220 | * src[2] = arg_2 |
| 1221 | * |
| 1222 | * We would like arg_1 = the address and |
| 1223 | * arg_2 = the offset. |
| 1224 | */ |
| 1225 | |
| 1226 | ins.ssa_args.src[is_read ? 0 : 1] = addr; |
| 1227 | |
| 1228 | /* TODO: What is this? It looks superficially like a shift << 5, but |
| 1229 | * arg_1 doesn't take a shift Should it be E0 or A0? */ |
| 1230 | if (indirect_offset) |
| 1231 | ins.load_store.arg_1 |= 0xE0; |
| 1232 | |
| 1233 | /* We also need to emit the indirect offset */ |
| 1234 | |
| 1235 | if (indirect_offset) |
| 1236 | ins.ssa_args.src[is_read ? 1 : 2] = nir_src_index(ctx, indirect_offset); |
| 1237 | else |
| 1238 | ins.load_store.arg_2 = 0x7E; |
| 1239 | |
| 1240 | /* TODO: Bounds check */ |
| 1241 | |
| 1242 | /* Finally, we emit the direct offset */ |
| 1243 | |
| 1244 | ins.load_store.varying_parameters = (offset & 0x1FF) << 1; |
| 1245 | ins.load_store.address = (offset >> 9); |
| 1246 | |
| 1247 | nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 1248 | |
| 1249 | if (is_read) |
| 1250 | ins.mask = mask_of(nir_intrinsic_dest_components(intr)); |
| 1251 | else |
| 1252 | ins.mask = nir_intrinsic_write_mask(intr); |
| 1253 | |
| 1254 | emit_mir_instruction(ctx, ins); |
| 1255 | } |
| 1256 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1257 | static void |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1258 | emit_varying_read( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1259 | compiler_context *ctx, |
| 1260 | unsigned dest, unsigned offset, |
| 1261 | unsigned nr_comp, unsigned component, |
| 1262 | nir_src *indirect_offset, nir_alu_type type) |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1263 | { |
| 1264 | /* XXX: Half-floats? */ |
| 1265 | /* TODO: swizzle, mask */ |
| 1266 | |
| 1267 | midgard_instruction ins = m_ld_vary_32(dest, offset); |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1268 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1269 | ins.load_store.swizzle = SWIZZLE_XYZW >> (2 * component); |
| 1270 | |
| 1271 | midgard_varying_parameter p = { |
| 1272 | .is_varying = 1, |
| 1273 | .interpolation = midgard_interp_default, |
| 1274 | .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0 |
| 1275 | }; |
| 1276 | |
| 1277 | unsigned u; |
| 1278 | memcpy(&u, &p, sizeof(p)); |
| 1279 | ins.load_store.varying_parameters = u; |
| 1280 | |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1281 | if (indirect_offset) |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1282 | ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset); |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1283 | else |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1284 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1285 | |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1286 | ins.load_store.arg_1 = 0x9E; |
| 1287 | |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1288 | /* Use the type appropriate load */ |
| 1289 | switch (type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1290 | case nir_type_uint: |
| 1291 | case nir_type_bool: |
| 1292 | ins.load_store.op = midgard_op_ld_vary_32u; |
| 1293 | break; |
| 1294 | case nir_type_int: |
| 1295 | ins.load_store.op = midgard_op_ld_vary_32i; |
| 1296 | break; |
| 1297 | case nir_type_float: |
| 1298 | ins.load_store.op = midgard_op_ld_vary_32; |
| 1299 | break; |
| 1300 | default: |
| 1301 | unreachable("Attempted to load unknown type"); |
| 1302 | break; |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1303 | } |
| 1304 | |
Alyssa Rosenzweig | 15fae1e | 2019-06-04 23:26:09 +0000 | [diff] [blame] | 1305 | emit_mir_instruction(ctx, ins); |
| 1306 | } |
| 1307 | |
Alyssa Rosenzweig | fa68740 | 2019-08-02 11:06:21 -0700 | [diff] [blame] | 1308 | void |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1309 | emit_sysval_read(compiler_context *ctx, nir_instr *instr, signed dest_override, |
| 1310 | unsigned nr_components) |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1311 | { |
Alyssa Rosenzweig | 6d8490f | 2019-07-11 15:34:56 -0700 | [diff] [blame] | 1312 | unsigned dest = 0; |
| 1313 | |
Boris Brezillon | bd49c8f | 2019-06-14 09:59:20 +0200 | [diff] [blame] | 1314 | /* Figure out which uniform this is */ |
| 1315 | int sysval = sysval_for_instr(ctx, instr, &dest); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1316 | void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval); |
| 1317 | |
Alyssa Rosenzweig | fa68740 | 2019-08-02 11:06:21 -0700 | [diff] [blame] | 1318 | if (dest_override >= 0) |
| 1319 | dest = dest_override; |
| 1320 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1321 | /* Sysvals are prefix uniforms */ |
| 1322 | unsigned uniform = ((uintptr_t) val) - 1; |
| 1323 | |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1324 | /* Emit the read itself -- this is never indirect */ |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1325 | midgard_instruction *ins = |
| 1326 | emit_ubo_read(ctx, dest, uniform, NULL, 0); |
| 1327 | |
| 1328 | ins->mask = mask_of(nr_components); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1329 | } |
| 1330 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1331 | static unsigned |
| 1332 | compute_builtin_arg(nir_op op) |
| 1333 | { |
| 1334 | switch (op) { |
| 1335 | case nir_intrinsic_load_work_group_id: |
| 1336 | return 0x14; |
| 1337 | case nir_intrinsic_load_local_invocation_id: |
| 1338 | return 0x10; |
| 1339 | default: |
| 1340 | unreachable("Invalid compute paramater loaded"); |
| 1341 | } |
| 1342 | } |
| 1343 | |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1344 | /* Emit store for a fragment shader, which is encoded via a fancy branch. TODO: |
| 1345 | * Handle MRT here */ |
| 1346 | |
| 1347 | static void |
| 1348 | emit_fragment_store(compiler_context *ctx, unsigned src, unsigned rt) |
| 1349 | { |
| 1350 | /* First, move in whatever we're outputting */ |
| 1351 | midgard_instruction move = v_mov(src, blank_alu_src, SSA_FIXED_REGISTER(0)); |
| 1352 | if (rt != 0) { |
| 1353 | /* Force a tight schedule. TODO: Make the scheduler MRT aware */ |
| 1354 | move.unit = UNIT_VMUL; |
| 1355 | move.precede_break = true; |
| 1356 | move.dont_eliminate = true; |
| 1357 | } |
| 1358 | |
| 1359 | emit_mir_instruction(ctx, move); |
| 1360 | |
| 1361 | /* If we're doing MRT, we need to specify the render target */ |
| 1362 | |
| 1363 | midgard_instruction rt_move = { |
| 1364 | .ssa_args = { |
| 1365 | .dest = -1 |
| 1366 | } |
| 1367 | }; |
| 1368 | |
| 1369 | if (rt != 0) { |
| 1370 | /* We'll write to r1.z */ |
| 1371 | rt_move = v_mov(-1, blank_alu_src, SSA_FIXED_REGISTER(1)); |
| 1372 | rt_move.mask = 1 << COMPONENT_Z; |
| 1373 | rt_move.unit = UNIT_SADD; |
| 1374 | |
| 1375 | /* r1.z = (rt * 0x100) */ |
| 1376 | rt_move.ssa_args.inline_constant = true; |
| 1377 | rt_move.inline_constant = (rt * 0x100); |
| 1378 | |
| 1379 | /* r1 */ |
| 1380 | ctx->work_registers = MAX2(ctx->work_registers, 1); |
| 1381 | |
| 1382 | /* Do the write */ |
| 1383 | emit_mir_instruction(ctx, rt_move); |
| 1384 | } |
| 1385 | |
| 1386 | /* Next, generate the branch. For R render targets in the writeout, the |
| 1387 | * i'th render target jumps to pseudo-offset [2(R-1) + i] */ |
| 1388 | |
| 1389 | unsigned offset = (2 * (ctx->nir->num_outputs - 1)) + rt; |
| 1390 | |
| 1391 | struct midgard_instruction ins = |
| 1392 | v_alu_br_compact_cond(midgard_jmp_writeout_op_writeout, TAG_ALU_4, offset, midgard_condition_always); |
| 1393 | |
| 1394 | /* Add dependencies */ |
| 1395 | ins.ssa_args.src[0] = move.ssa_args.dest; |
| 1396 | ins.ssa_args.src[1] = rt_move.ssa_args.dest; |
| 1397 | |
| 1398 | /* Emit the branch */ |
| 1399 | emit_mir_instruction(ctx, ins); |
| 1400 | } |
| 1401 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1402 | static void |
| 1403 | emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1404 | { |
| 1405 | unsigned reg = nir_dest_index(ctx, &instr->dest); |
| 1406 | midgard_instruction ins = m_ld_compute_id(reg, 0); |
| 1407 | ins.mask = mask_of(3); |
| 1408 | ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic); |
| 1409 | emit_mir_instruction(ctx, ins); |
| 1410 | } |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1411 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1412 | emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr) |
| 1413 | { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1414 | unsigned offset = 0, reg; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1415 | |
| 1416 | switch (instr->intrinsic) { |
| 1417 | case nir_intrinsic_discard_if: |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 1418 | emit_condition(ctx, &instr->src[0], true, COMPONENT_X); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1419 | |
| 1420 | /* fallthrough */ |
| 1421 | |
| 1422 | case nir_intrinsic_discard: { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 1423 | bool conditional = instr->intrinsic == nir_intrinsic_discard_if; |
| 1424 | struct midgard_instruction discard = v_branch(conditional, false); |
| 1425 | discard.branch.target_type = TARGET_DISCARD; |
| 1426 | emit_mir_instruction(ctx, discard); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1427 | break; |
| 1428 | } |
| 1429 | |
| 1430 | case nir_intrinsic_load_uniform: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1431 | case nir_intrinsic_load_ubo: |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1432 | case nir_intrinsic_load_ssbo: |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1433 | case nir_intrinsic_load_input: { |
| 1434 | bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform; |
| 1435 | bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo; |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1436 | bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1437 | |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1438 | /* Get the base type of the intrinsic */ |
Alyssa Rosenzweig | 8d74749 | 2019-06-27 14:13:10 -0700 | [diff] [blame] | 1439 | /* TODO: Infer type? Does it matter? */ |
| 1440 | nir_alu_type t = |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1441 | (is_ubo || is_ssbo) ? nir_type_uint : nir_intrinsic_type(instr); |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1442 | t = nir_alu_type_get_base_type(t); |
| 1443 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1444 | if (!(is_ubo || is_ssbo)) { |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1445 | offset = nir_intrinsic_base(instr); |
| 1446 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1447 | |
Alyssa Rosenzweig | c1715b5 | 2019-05-22 02:44:12 +0000 | [diff] [blame] | 1448 | unsigned nr_comp = nir_intrinsic_dest_components(instr); |
Alyssa Rosenzweig | 6a466c0 | 2019-04-20 23:52:42 +0000 | [diff] [blame] | 1449 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1450 | nir_src *src_offset = nir_get_io_offset_src(instr); |
| 1451 | |
| 1452 | bool direct = nir_src_is_const(*src_offset); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1453 | nir_src *indirect_offset = direct ? NULL : src_offset; |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1454 | |
| 1455 | if (direct) |
| 1456 | offset += nir_src_as_uint(*src_offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1457 | |
Alyssa Rosenzweig | 43568f2 | 2019-06-06 08:16:04 -0700 | [diff] [blame] | 1458 | /* We may need to apply a fractional offset */ |
| 1459 | int component = instr->intrinsic == nir_intrinsic_load_input ? |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1460 | nir_intrinsic_component(instr) : 0; |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1461 | reg = nir_dest_index(ctx, &instr->dest); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1462 | |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1463 | if (is_uniform && !ctx->is_blend) { |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1464 | emit_ubo_read(ctx, reg, ctx->sysval_count + offset, indirect_offset, 0); |
Alyssa Rosenzweig | 5e2c3d4 | 2019-06-20 15:51:31 -0700 | [diff] [blame] | 1465 | } else if (is_ubo) { |
| 1466 | nir_src index = instr->src[0]; |
| 1467 | |
| 1468 | /* We don't yet support indirect UBOs. For indirect |
| 1469 | * block numbers (if that's possible), we don't know |
| 1470 | * enough about the hardware yet. For indirect sources, |
| 1471 | * we know what we need but we need to add some NIR |
| 1472 | * support for lowering correctly with respect to |
| 1473 | * 128-bit reads */ |
| 1474 | |
| 1475 | assert(nir_src_is_const(index)); |
| 1476 | assert(nir_src_is_const(*src_offset)); |
| 1477 | |
| 1478 | /* TODO: Alignment */ |
| 1479 | assert((offset & 0xF) == 0); |
| 1480 | |
| 1481 | uint32_t uindex = nir_src_as_uint(index) + 1; |
| 1482 | emit_ubo_read(ctx, reg, offset / 16, NULL, uindex); |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1483 | } else if (is_ssbo) { |
| 1484 | nir_src index = instr->src[0]; |
| 1485 | assert(nir_src_is_const(index)); |
| 1486 | uint32_t uindex = nir_src_as_uint(index); |
| 1487 | |
| 1488 | emit_ssbo_access(ctx, &instr->instr, true, reg, offset, indirect_offset, uindex); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1489 | } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) { |
Alyssa Rosenzweig | 9b97ed1 | 2019-06-28 09:30:59 -0700 | [diff] [blame] | 1490 | emit_varying_read(ctx, reg, offset, nr_comp, component, !direct ? &instr->src[0] : NULL, t); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1491 | } else if (ctx->is_blend) { |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1492 | /* For blend shaders, load the input color, which is |
| 1493 | * preloaded to r0 */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1494 | |
Alyssa Rosenzweig | 13f61f2 | 2019-07-26 08:15:50 -0700 | [diff] [blame] | 1495 | midgard_instruction move = v_mov(SSA_FIXED_REGISTER(0), blank_alu_src, reg); |
Alyssa Rosenzweig | 005d9b1 | 2019-05-20 00:46:48 +0000 | [diff] [blame] | 1496 | emit_mir_instruction(ctx, move); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1497 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 74ab80b | 2019-05-14 04:11:36 +0000 | [diff] [blame] | 1498 | midgard_instruction ins = m_ld_attr_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1499 | ins.load_store.arg_1 = 0x1E; |
| 1500 | ins.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1501 | ins.mask = mask_of(nr_comp); |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1502 | |
| 1503 | /* Use the type appropriate load */ |
| 1504 | switch (t) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1505 | case nir_type_uint: |
| 1506 | case nir_type_bool: |
| 1507 | ins.load_store.op = midgard_op_ld_attr_32u; |
| 1508 | break; |
| 1509 | case nir_type_int: |
| 1510 | ins.load_store.op = midgard_op_ld_attr_32i; |
| 1511 | break; |
| 1512 | case nir_type_float: |
| 1513 | ins.load_store.op = midgard_op_ld_attr_32; |
| 1514 | break; |
| 1515 | default: |
| 1516 | unreachable("Attempted to load unknown type"); |
| 1517 | break; |
Alyssa Rosenzweig | bbc050b | 2019-06-27 15:33:07 -0700 | [diff] [blame] | 1518 | } |
| 1519 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1520 | emit_mir_instruction(ctx, ins); |
| 1521 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1522 | DBG("Unknown load\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1523 | assert(0); |
| 1524 | } |
| 1525 | |
| 1526 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1527 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1528 | |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1529 | /* Reads 128-bit value raw off the tilebuffer during blending, tasty */ |
| 1530 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1531 | case nir_intrinsic_load_raw_output_pan: |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1532 | reg = nir_dest_index(ctx, &instr->dest); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1533 | assert(ctx->is_blend); |
Alyssa Rosenzweig | 1686ef8 | 2019-07-01 17:23:58 -0700 | [diff] [blame] | 1534 | |
| 1535 | midgard_instruction ins = m_ld_color_buffer_8(reg, 0); |
| 1536 | emit_mir_instruction(ctx, ins); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1537 | break; |
| 1538 | |
| 1539 | case nir_intrinsic_load_blend_const_color_rgba: { |
| 1540 | assert(ctx->is_blend); |
| 1541 | reg = nir_dest_index(ctx, &instr->dest); |
| 1542 | |
| 1543 | /* Blend constants are embedded directly in the shader and |
| 1544 | * patched in, so we use some magic routing */ |
| 1545 | |
Alyssa Rosenzweig | 565c446 | 2019-06-17 09:40:14 -0700 | [diff] [blame] | 1546 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg); |
Alyssa Rosenzweig | 3910422 | 2019-05-06 02:12:41 +0000 | [diff] [blame] | 1547 | ins.has_constants = true; |
| 1548 | ins.has_blend_constant = true; |
| 1549 | emit_mir_instruction(ctx, ins); |
| 1550 | break; |
| 1551 | } |
| 1552 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1553 | case nir_intrinsic_store_output: |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1554 | assert(nir_src_is_const(instr->src[1]) && "no indirect outputs"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1555 | |
Karol Herbst | 1aabb79 | 2019-03-29 21:40:45 +0100 | [diff] [blame] | 1556 | offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1557 | |
Alyssa Rosenzweig | 4ed23b1 | 2019-02-07 04:56:13 +0000 | [diff] [blame] | 1558 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1559 | |
| 1560 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1561 | /* Determine number of render targets */ |
| 1562 | emit_fragment_store(ctx, reg, offset); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1563 | } else if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1564 | /* We should have been vectorized, though we don't |
| 1565 | * currently check that st_vary is emitted only once |
| 1566 | * per slot (this is relevant, since there's not a mask |
| 1567 | * parameter available on the store [set to 0 by the |
| 1568 | * blob]). We do respect the component by adjusting the |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1569 | * swizzle. If this is a constant source, we'll need to |
| 1570 | * emit that explicitly. */ |
| 1571 | |
| 1572 | emit_explicit_constant(ctx, reg, reg); |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1573 | |
| 1574 | unsigned component = nir_intrinsic_component(instr); |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 1575 | |
Alyssa Rosenzweig | 233c0fa | 2019-07-24 12:54:59 -0700 | [diff] [blame] | 1576 | midgard_instruction st = m_st_vary_32(reg, offset); |
Alyssa Rosenzweig | c908772 | 2019-08-01 13:29:01 -0700 | [diff] [blame] | 1577 | st.load_store.arg_1 = 0x9E; |
| 1578 | st.load_store.arg_2 = 0x1E; |
Alyssa Rosenzweig | a3ae3cb | 2019-06-17 12:35:57 -0700 | [diff] [blame] | 1579 | st.load_store.swizzle = SWIZZLE_XYZW << (2*component); |
Alyssa Rosenzweig | 4aced18 | 2019-06-06 08:21:27 -0700 | [diff] [blame] | 1580 | emit_mir_instruction(ctx, st); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1581 | } else { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1582 | DBG("Unknown store\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1583 | assert(0); |
| 1584 | } |
| 1585 | |
| 1586 | break; |
| 1587 | |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1588 | /* Special case of store_output for lowered blend shaders */ |
| 1589 | case nir_intrinsic_store_raw_output_pan: |
| 1590 | assert (ctx->stage == MESA_SHADER_FRAGMENT); |
| 1591 | reg = nir_src_index(ctx, &instr->src[0]); |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 1592 | emit_fragment_store(ctx, reg, 0); |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 1593 | |
| 1594 | break; |
| 1595 | |
Alyssa Rosenzweig | 419ddd6 | 2019-08-01 10:03:02 -0700 | [diff] [blame] | 1596 | case nir_intrinsic_store_ssbo: |
| 1597 | assert(nir_src_is_const(instr->src[1])); |
| 1598 | |
| 1599 | bool direct_offset = nir_src_is_const(instr->src[2]); |
| 1600 | offset = direct_offset ? nir_src_as_uint(instr->src[2]) : 0; |
| 1601 | nir_src *indirect_offset = direct_offset ? NULL : &instr->src[2]; |
| 1602 | reg = nir_src_index(ctx, &instr->src[0]); |
| 1603 | |
| 1604 | uint32_t uindex = nir_src_as_uint(instr->src[1]); |
| 1605 | |
| 1606 | emit_explicit_constant(ctx, reg, reg); |
| 1607 | emit_ssbo_access(ctx, &instr->instr, false, reg, offset, indirect_offset, uindex); |
| 1608 | break; |
| 1609 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1610 | case nir_intrinsic_load_alpha_ref_float: |
| 1611 | assert(instr->dest.is_ssa); |
| 1612 | |
| 1613 | float ref_value = ctx->alpha_ref; |
| 1614 | |
Alyssa Rosenzweig | 463164b | 2019-07-29 08:31:03 -0700 | [diff] [blame] | 1615 | /* See emit_load_const */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1616 | float *v = ralloc_array(NULL, float, 4); |
| 1617 | memcpy(v, &ref_value, sizeof(float)); |
Alyssa Rosenzweig | 463164b | 2019-07-29 08:31:03 -0700 | [diff] [blame] | 1618 | _mesa_hash_table_u64_insert(ctx->ssa_constants, (instr->dest.ssa.index << 1) + 1, v); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1619 | break; |
| 1620 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1621 | case nir_intrinsic_load_viewport_scale: |
| 1622 | case nir_intrinsic_load_viewport_offset: |
Alyssa Rosenzweig | 15954ab | 2019-08-06 14:07:10 -0700 | [diff] [blame] | 1623 | case nir_intrinsic_load_num_work_groups: |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1624 | emit_sysval_read(ctx, &instr->instr, -1, 3); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 1625 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1626 | |
Alyssa Rosenzweig | 7229af7 | 2019-08-06 13:47:17 -0700 | [diff] [blame] | 1627 | case nir_intrinsic_load_work_group_id: |
| 1628 | case nir_intrinsic_load_local_invocation_id: |
| 1629 | emit_compute_builtin(ctx, instr); |
| 1630 | break; |
| 1631 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1632 | default: |
| 1633 | printf ("Unhandled intrinsic\n"); |
| 1634 | assert(0); |
| 1635 | break; |
| 1636 | } |
| 1637 | } |
| 1638 | |
| 1639 | static unsigned |
| 1640 | midgard_tex_format(enum glsl_sampler_dim dim) |
| 1641 | { |
| 1642 | switch (dim) { |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1643 | case GLSL_SAMPLER_DIM_1D: |
| 1644 | case GLSL_SAMPLER_DIM_BUF: |
| 1645 | return MALI_TEX_1D; |
| 1646 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1647 | case GLSL_SAMPLER_DIM_2D: |
| 1648 | case GLSL_SAMPLER_DIM_EXTERNAL: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1649 | return MALI_TEX_2D; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1650 | |
| 1651 | case GLSL_SAMPLER_DIM_3D: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1652 | return MALI_TEX_3D; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1653 | |
| 1654 | case GLSL_SAMPLER_DIM_CUBE: |
Alyssa Rosenzweig | 83c02a5 | 2019-06-17 14:26:08 -0700 | [diff] [blame] | 1655 | return MALI_TEX_CUBE; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1656 | |
| 1657 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1658 | DBG("Unknown sampler dim type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1659 | assert(0); |
| 1660 | return 0; |
| 1661 | } |
| 1662 | } |
| 1663 | |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1664 | /* Tries to attach an explicit LOD / bias as a constant. Returns whether this |
| 1665 | * was successful */ |
| 1666 | |
| 1667 | static bool |
| 1668 | pan_attach_constant_bias( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1669 | compiler_context *ctx, |
| 1670 | nir_src lod, |
| 1671 | midgard_texture_word *word) |
Alyssa Rosenzweig | 213b628 | 2019-06-18 09:02:20 -0700 | [diff] [blame] | 1672 | { |
| 1673 | /* To attach as constant, it has to *be* constant */ |
| 1674 | |
| 1675 | if (!nir_src_is_const(lod)) |
| 1676 | return false; |
| 1677 | |
| 1678 | float f = nir_src_as_float(lod); |
| 1679 | |
| 1680 | /* Break into fixed-point */ |
| 1681 | signed lod_int = f; |
| 1682 | float lod_frac = f - lod_int; |
| 1683 | |
| 1684 | /* Carry over negative fractions */ |
| 1685 | if (lod_frac < 0.0) { |
| 1686 | lod_int--; |
| 1687 | lod_frac += 1.0; |
| 1688 | } |
| 1689 | |
| 1690 | /* Encode */ |
| 1691 | word->bias = float_to_ubyte(lod_frac); |
| 1692 | word->bias_int = lod_int; |
| 1693 | |
| 1694 | return true; |
| 1695 | } |
| 1696 | |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1697 | static enum mali_sampler_type |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1698 | midgard_sampler_type(nir_alu_type t) { |
| 1699 | switch (nir_alu_type_get_base_type(t)) |
| 1700 | { |
| 1701 | case nir_type_float: |
| 1702 | return MALI_SAMPLER_FLOAT; |
| 1703 | case nir_type_int: |
| 1704 | return MALI_SAMPLER_SIGNED; |
| 1705 | case nir_type_uint: |
| 1706 | return MALI_SAMPLER_UNSIGNED; |
| 1707 | default: |
| 1708 | unreachable("Unknown sampler type"); |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1709 | } |
| 1710 | } |
| 1711 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1712 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1713 | emit_texop_native(compiler_context *ctx, nir_tex_instr *instr, |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1714 | unsigned midgard_texop) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1715 | { |
| 1716 | /* TODO */ |
| 1717 | //assert (!instr->sampler); |
| 1718 | //assert (!instr->texture_array_size); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1719 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1720 | int texture_index = instr->texture_index; |
| 1721 | int sampler_index = texture_index; |
| 1722 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1723 | /* No helper to build texture words -- we do it all here */ |
| 1724 | midgard_instruction ins = { |
| 1725 | .type = TAG_TEXTURE_4, |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1726 | .mask = 0xF, |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1727 | .ssa_args = { |
| 1728 | .dest = nir_dest_index(ctx, &instr->dest), |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1729 | .src = { -1, -1, -1 }, |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1730 | }, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1731 | .texture = { |
| 1732 | .op = midgard_texop, |
| 1733 | .format = midgard_tex_format(instr->sampler_dim), |
| 1734 | .texture_handle = texture_index, |
| 1735 | .sampler_handle = sampler_index, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1736 | .swizzle = SWIZZLE_XYZW, |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1737 | .in_reg_swizzle = SWIZZLE_XYZW, |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1738 | |
| 1739 | /* TODO: half */ |
| 1740 | .in_reg_full = 1, |
| 1741 | .out_full = 1, |
| 1742 | |
Alyssa Rosenzweig | e32af4b | 2019-06-26 16:12:28 -0700 | [diff] [blame] | 1743 | .sampler_type = midgard_sampler_type(instr->dest_type), |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1744 | } |
| 1745 | }; |
Alyssa Rosenzweig | 8429bee | 2019-06-14 16:03:39 -0700 | [diff] [blame] | 1746 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1747 | for (unsigned i = 0; i < instr->num_srcs; ++i) { |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1748 | int index = nir_src_index(ctx, &instr->src[i].src); |
| 1749 | midgard_vector_alu_src alu_src = blank_alu_src; |
| 1750 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1751 | switch (instr->src[i].src_type) { |
| 1752 | case nir_tex_src_coord: { |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1753 | emit_explicit_constant(ctx, index, index); |
| 1754 | |
| 1755 | /* Texelfetch coordinates uses all four elements |
| 1756 | * (xyz/index) regardless of texture dimensionality, |
| 1757 | * which means it's necessary to zero the unused |
| 1758 | * components to keep everything happy */ |
| 1759 | |
| 1760 | if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) { |
| 1761 | unsigned old_index = index; |
| 1762 | |
| 1763 | index = make_compiler_temp(ctx); |
| 1764 | |
| 1765 | /* mov index, old_index */ |
| 1766 | midgard_instruction mov = v_mov(old_index, blank_alu_src, index); |
| 1767 | mov.mask = 0x3; |
| 1768 | emit_mir_instruction(ctx, mov); |
| 1769 | |
| 1770 | /* mov index.zw, #0 */ |
| 1771 | mov = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), |
| 1772 | blank_alu_src, index); |
| 1773 | mov.has_constants = true; |
| 1774 | mov.mask = (1 << COMPONENT_Z) | (1 << COMPONENT_W); |
| 1775 | emit_mir_instruction(ctx, mov); |
| 1776 | } |
| 1777 | |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1778 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) { |
Alyssa Rosenzweig | faf8ad4 | 2019-06-24 14:39:25 -0700 | [diff] [blame] | 1779 | /* texelFetch is undefined on samplerCube */ |
| 1780 | assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH); |
| 1781 | |
Alyssa Rosenzweig | be56840 | 2019-07-25 07:09:40 -0700 | [diff] [blame] | 1782 | /* For cubemaps, we use a special ld/st op to |
| 1783 | * select the face and copy the xy into the |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1784 | * texture register */ |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1785 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1786 | unsigned temp = make_compiler_temp(ctx); |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1787 | midgard_instruction st = m_st_cubemap_coords(temp, 0); |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1788 | st.ssa_args.src[0] = index; |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 1789 | st.mask = 0x3; /* xy */ |
Alyssa Rosenzweig | 513d02c | 2019-08-01 14:28:34 -0700 | [diff] [blame] | 1790 | st.load_store.arg_1 = 0x20; |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1791 | st.load_store.swizzle = alu_src.swizzle; |
| 1792 | emit_mir_instruction(ctx, st); |
| 1793 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1794 | ins.ssa_args.src[0] = temp; |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1795 | } else { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1796 | ins.ssa_args.src[0] = index; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1797 | } |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1798 | |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1799 | if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) { |
| 1800 | /* Array component in w but NIR wants it in z */ |
| 1801 | ins.texture.in_reg_swizzle = SWIZZLE_XYZZ; |
Alyssa Rosenzweig | 70b3e5d | 2019-03-28 04:27:13 +0000 | [diff] [blame] | 1802 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1803 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1804 | break; |
| 1805 | } |
| 1806 | |
Alyssa Rosenzweig | 4012e06 | 2019-06-11 09:43:08 -0700 | [diff] [blame] | 1807 | case nir_tex_src_bias: |
| 1808 | case nir_tex_src_lod: { |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1809 | /* Try as a constant if we can */ |
| 1810 | |
| 1811 | bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH; |
| 1812 | if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture)) |
| 1813 | break; |
| 1814 | |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1815 | ins.texture.lod_register = true; |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1816 | ins.ssa_args.src[1] = index; |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 1817 | emit_explicit_constant(ctx, index, index); |
Alyssa Rosenzweig | b0e8941 | 2019-06-18 09:02:35 -0700 | [diff] [blame] | 1818 | |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1819 | break; |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1820 | }; |
Alyssa Rosenzweig | a19ca34 | 2019-06-11 09:23:05 -0700 | [diff] [blame] | 1821 | |
Alyssa Rosenzweig | 5062b61 | 2019-06-11 09:55:18 -0700 | [diff] [blame] | 1822 | default: |
| 1823 | unreachable("Unknown texture source type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1824 | } |
| 1825 | } |
| 1826 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1827 | emit_mir_instruction(ctx, ins); |
| 1828 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1829 | /* Used for .cont and .last hinting */ |
| 1830 | ctx->texture_op_count++; |
| 1831 | } |
| 1832 | |
| 1833 | static void |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1834 | emit_tex(compiler_context *ctx, nir_tex_instr *instr) |
| 1835 | { |
Alyssa Rosenzweig | 6729912 | 2019-06-24 10:35:03 -0700 | [diff] [blame] | 1836 | /* Fixup op, since only textureLod is permitted in VS but NIR can give |
| 1837 | * generic tex in some cases (which confuses the hardware) */ |
| 1838 | |
| 1839 | bool is_vertex = ctx->stage == MESA_SHADER_VERTEX; |
| 1840 | |
| 1841 | if (is_vertex && instr->op == nir_texop_tex) |
| 1842 | instr->op = nir_texop_txl; |
| 1843 | |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1844 | switch (instr->op) { |
| 1845 | case nir_texop_tex: |
| 1846 | case nir_texop_txb: |
| 1847 | emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL); |
| 1848 | break; |
| 1849 | case nir_texop_txl: |
| 1850 | emit_texop_native(ctx, instr, TEXTURE_OP_LOD); |
| 1851 | break; |
Alyssa Rosenzweig | f4bb7f0 | 2019-06-21 16:17:34 -0700 | [diff] [blame] | 1852 | case nir_texop_txf: |
| 1853 | emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH); |
| 1854 | break; |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 1855 | case nir_texop_txs: |
Alyssa Rosenzweig | 63e240d | 2019-08-02 17:10:18 -0700 | [diff] [blame] | 1856 | emit_sysval_read(ctx, &instr->instr, -1, 4); |
Boris Brezillon | c355886 | 2019-06-17 22:13:04 +0200 | [diff] [blame] | 1857 | break; |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1858 | default: |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1859 | unreachable("Unhanlded texture op"); |
Boris Brezillon | 5c17f84 | 2019-06-17 21:47:46 +0200 | [diff] [blame] | 1860 | } |
| 1861 | } |
| 1862 | |
| 1863 | static void |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1864 | emit_jump(compiler_context *ctx, nir_jump_instr *instr) |
| 1865 | { |
| 1866 | switch (instr->type) { |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1867 | case nir_jump_break: { |
| 1868 | /* Emit a branch out of the loop */ |
| 1869 | struct midgard_instruction br = v_branch(false, false); |
| 1870 | br.branch.target_type = TARGET_BREAK; |
| 1871 | br.branch.target_break = ctx->current_loop_depth; |
| 1872 | emit_mir_instruction(ctx, br); |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1873 | break; |
| 1874 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1875 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 1876 | default: |
| 1877 | DBG("Unknown jump type %d\n", instr->type); |
| 1878 | break; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1879 | } |
| 1880 | } |
| 1881 | |
| 1882 | static void |
| 1883 | emit_instr(compiler_context *ctx, struct nir_instr *instr) |
| 1884 | { |
| 1885 | switch (instr->type) { |
| 1886 | case nir_instr_type_load_const: |
| 1887 | emit_load_const(ctx, nir_instr_as_load_const(instr)); |
| 1888 | break; |
| 1889 | |
| 1890 | case nir_instr_type_intrinsic: |
| 1891 | emit_intrinsic(ctx, nir_instr_as_intrinsic(instr)); |
| 1892 | break; |
| 1893 | |
| 1894 | case nir_instr_type_alu: |
| 1895 | emit_alu(ctx, nir_instr_as_alu(instr)); |
| 1896 | break; |
| 1897 | |
| 1898 | case nir_instr_type_tex: |
| 1899 | emit_tex(ctx, nir_instr_as_tex(instr)); |
| 1900 | break; |
| 1901 | |
| 1902 | case nir_instr_type_jump: |
| 1903 | emit_jump(ctx, nir_instr_as_jump(instr)); |
| 1904 | break; |
| 1905 | |
| 1906 | case nir_instr_type_ssa_undef: |
| 1907 | /* Spurious */ |
| 1908 | break; |
| 1909 | |
| 1910 | default: |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 1911 | DBG("Unhandled instruction type\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1912 | break; |
| 1913 | } |
| 1914 | } |
| 1915 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1916 | |
| 1917 | /* ALU instructions can inline or embed constants, which decreases register |
| 1918 | * pressure and saves space. */ |
| 1919 | |
| 1920 | #define CONDITIONAL_ATTACH(src) { \ |
| 1921 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \ |
| 1922 | \ |
| 1923 | if (entry) { \ |
| 1924 | attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \ |
| 1925 | alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \ |
| 1926 | } \ |
| 1927 | } |
| 1928 | |
| 1929 | static void |
| 1930 | inline_alu_constants(compiler_context *ctx) |
| 1931 | { |
| 1932 | mir_foreach_instr(ctx, alu) { |
| 1933 | /* Other instructions cannot inline constants */ |
| 1934 | if (alu->type != TAG_ALU_4) continue; |
| 1935 | |
| 1936 | /* If there is already a constant here, we can do nothing */ |
| 1937 | if (alu->has_constants) continue; |
| 1938 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1939 | CONDITIONAL_ATTACH(src[0]); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1940 | |
| 1941 | if (!alu->has_constants) { |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1942 | CONDITIONAL_ATTACH(src[1]) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1943 | } else if (!alu->inline_constant) { |
| 1944 | /* Corner case: _two_ vec4 constants, for instance with a |
| 1945 | * csel. For this case, we can only use a constant |
| 1946 | * register for one, we'll have to emit a move for the |
| 1947 | * other. Note, if both arguments are constants, then |
| 1948 | * necessarily neither argument depends on the value of |
| 1949 | * any particular register. As the destination register |
| 1950 | * will be wiped, that means we can spill the constant |
| 1951 | * to the destination register. |
| 1952 | */ |
| 1953 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1954 | void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1955 | unsigned scratch = alu->ssa_args.dest; |
| 1956 | |
| 1957 | if (entry) { |
Alyssa Rosenzweig | 565c446 | 2019-06-17 09:40:14 -0700 | [diff] [blame] | 1958 | midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch); |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1959 | attach_constants(ctx, &ins, entry, alu->ssa_args.src[1] + 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1960 | |
| 1961 | /* Force a break XXX Defer r31 writes */ |
| 1962 | ins.unit = UNIT_VLUT; |
| 1963 | |
| 1964 | /* Set the source */ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 1965 | alu->ssa_args.src[1] = scratch; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 1966 | |
| 1967 | /* Inject us -before- the last instruction which set r31 */ |
| 1968 | mir_insert_instruction_before(mir_prev_op(alu), ins); |
| 1969 | } |
| 1970 | } |
| 1971 | } |
| 1972 | } |
| 1973 | |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 1974 | /* Being a little silly with the names, but returns the op that is the bitwise |
| 1975 | * inverse of the op with the argument switched. I.e. (f and g are |
| 1976 | * contrapositives): |
| 1977 | * |
| 1978 | * f(a, b) = ~g(b, a) |
| 1979 | * |
| 1980 | * Corollary: if g is the contrapositve of f, f is the contrapositive of g: |
| 1981 | * |
| 1982 | * f(a, b) = ~g(b, a) |
| 1983 | * ~f(a, b) = g(b, a) |
| 1984 | * ~f(a, b) = ~h(a, b) where h is the contrapositive of g |
| 1985 | * f(a, b) = h(a, b) |
| 1986 | * |
| 1987 | * Thus we define this function in pairs. |
| 1988 | */ |
| 1989 | |
| 1990 | static inline midgard_alu_op |
| 1991 | mir_contrapositive(midgard_alu_op op) |
| 1992 | { |
| 1993 | switch (op) { |
| 1994 | case midgard_alu_op_flt: |
| 1995 | return midgard_alu_op_fle; |
| 1996 | case midgard_alu_op_fle: |
| 1997 | return midgard_alu_op_flt; |
| 1998 | |
| 1999 | case midgard_alu_op_ilt: |
| 2000 | return midgard_alu_op_ile; |
| 2001 | case midgard_alu_op_ile: |
| 2002 | return midgard_alu_op_ilt; |
| 2003 | |
| 2004 | default: |
| 2005 | unreachable("No known contrapositive"); |
| 2006 | } |
| 2007 | } |
| 2008 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2009 | /* Midgard supports two types of constants, embedded constants (128-bit) and |
| 2010 | * inline constants (16-bit). Sometimes, especially with scalar ops, embedded |
| 2011 | * constants can be demoted to inline constants, for space savings and |
| 2012 | * sometimes a performance boost */ |
| 2013 | |
| 2014 | static void |
| 2015 | embedded_to_inline_constant(compiler_context *ctx) |
| 2016 | { |
| 2017 | mir_foreach_instr(ctx, ins) { |
| 2018 | if (!ins->has_constants) continue; |
| 2019 | |
| 2020 | if (ins->ssa_args.inline_constant) continue; |
| 2021 | |
| 2022 | /* Blend constants must not be inlined by definition */ |
| 2023 | if (ins->has_blend_constant) continue; |
| 2024 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2025 | /* We can inline 32-bit (sometimes) or 16-bit (usually) */ |
| 2026 | bool is_16 = ins->alu.reg_mode == midgard_reg_mode_16; |
| 2027 | bool is_32 = ins->alu.reg_mode == midgard_reg_mode_32; |
| 2028 | |
| 2029 | if (!(is_16 || is_32)) |
| 2030 | continue; |
| 2031 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2032 | /* src1 cannot be an inline constant due to encoding |
| 2033 | * restrictions. So, if possible we try to flip the arguments |
| 2034 | * in that case */ |
| 2035 | |
| 2036 | int op = ins->alu.op; |
| 2037 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 2038 | if (ins->ssa_args.src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2039 | bool flip = alu_opcode_props[op].props & OP_COMMUTES; |
| 2040 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2041 | switch (op) { |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2042 | /* Conditionals can be inverted */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2043 | case midgard_alu_op_flt: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2044 | case midgard_alu_op_ilt: |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2045 | case midgard_alu_op_fle: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2046 | case midgard_alu_op_ile: |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2047 | ins->alu.op = mir_contrapositive(ins->alu.op); |
| 2048 | ins->invert = true; |
| 2049 | flip = true; |
| 2050 | break; |
| 2051 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2052 | case midgard_alu_op_fcsel: |
| 2053 | case midgard_alu_op_icsel: |
Alyssa Rosenzweig | 1f345bc | 2019-04-24 01:15:15 +0000 | [diff] [blame] | 2054 | DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name); |
Alyssa Rosenzweig | bb1aff3 | 2019-04-24 02:18:28 +0000 | [diff] [blame] | 2055 | default: |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2056 | break; |
Alyssa Rosenzweig | bb1aff3 | 2019-04-24 02:18:28 +0000 | [diff] [blame] | 2057 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2058 | |
Alyssa Rosenzweig | 62a5ee3 | 2019-07-26 14:25:25 -0700 | [diff] [blame] | 2059 | if (flip) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2060 | /* Flip the SSA numbers */ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 2061 | ins->ssa_args.src[0] = ins->ssa_args.src[1]; |
| 2062 | ins->ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2063 | |
| 2064 | /* And flip the modifiers */ |
| 2065 | |
| 2066 | unsigned src_temp; |
| 2067 | |
| 2068 | src_temp = ins->alu.src2; |
| 2069 | ins->alu.src2 = ins->alu.src1; |
| 2070 | ins->alu.src1 = src_temp; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2071 | } |
| 2072 | } |
| 2073 | |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 2074 | if (ins->ssa_args.src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2075 | /* Extract the source information */ |
| 2076 | |
| 2077 | midgard_vector_alu_src *src; |
| 2078 | int q = ins->alu.src2; |
| 2079 | midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q; |
| 2080 | src = m; |
| 2081 | |
| 2082 | /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */ |
| 2083 | int component = src->swizzle & 3; |
| 2084 | |
| 2085 | /* Scale constant appropriately, if we can legally */ |
| 2086 | uint16_t scaled_constant = 0; |
| 2087 | |
Alyssa Rosenzweig | e92caad | 2019-07-01 20:02:57 -0700 | [diff] [blame] | 2088 | if (midgard_is_integer_op(op) || is_16) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2089 | unsigned int *iconstants = (unsigned int *) ins->constants; |
| 2090 | scaled_constant = (uint16_t) iconstants[component]; |
| 2091 | |
| 2092 | /* Constant overflow after resize */ |
| 2093 | if (scaled_constant != iconstants[component]) |
| 2094 | continue; |
| 2095 | } else { |
Alyssa Rosenzweig | 3978614 | 2019-04-28 15:46:47 +0000 | [diff] [blame] | 2096 | float original = (float) ins->constants[component]; |
| 2097 | scaled_constant = _mesa_float_to_half(original); |
| 2098 | |
| 2099 | /* Check for loss of precision. If this is |
| 2100 | * mediump, we don't care, but for a highp |
| 2101 | * shader, we need to pay attention. NIR |
| 2102 | * doesn't yet tell us which mode we're in! |
| 2103 | * Practically this prevents most constants |
| 2104 | * from being inlined, sadly. */ |
| 2105 | |
| 2106 | float fp32 = _mesa_half_to_float(scaled_constant); |
| 2107 | |
| 2108 | if (fp32 != original) |
| 2109 | continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2110 | } |
| 2111 | |
| 2112 | /* We don't know how to handle these with a constant */ |
| 2113 | |
Alyssa Rosenzweig | c45487b | 2019-07-26 11:52:30 -0700 | [diff] [blame] | 2114 | if (mir_nontrivial_source2_mod_simple(ins) || src->rep_low || src->rep_high) { |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2115 | DBG("Bailing inline constant...\n"); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2116 | continue; |
| 2117 | } |
| 2118 | |
| 2119 | /* Make sure that the constant is not itself a |
| 2120 | * vector by checking if all accessed values |
| 2121 | * (by the swizzle) are the same. */ |
| 2122 | |
| 2123 | uint32_t *cons = (uint32_t *) ins->constants; |
| 2124 | uint32_t value = cons[component]; |
| 2125 | |
| 2126 | bool is_vector = false; |
Alyssa Rosenzweig | f8b18a4 | 2019-07-01 18:51:48 -0700 | [diff] [blame] | 2127 | unsigned mask = effective_writemask(&ins->alu, ins->mask); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2128 | |
| 2129 | for (int c = 1; c < 4; ++c) { |
| 2130 | /* We only care if this component is actually used */ |
| 2131 | if (!(mask & (1 << c))) |
| 2132 | continue; |
| 2133 | |
| 2134 | uint32_t test = cons[(src->swizzle >> (2 * c)) & 3]; |
| 2135 | |
| 2136 | if (test != value) { |
| 2137 | is_vector = true; |
| 2138 | break; |
| 2139 | } |
| 2140 | } |
| 2141 | |
| 2142 | if (is_vector) |
| 2143 | continue; |
| 2144 | |
| 2145 | /* Get rid of the embedded constant */ |
| 2146 | ins->has_constants = false; |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 2147 | ins->ssa_args.src[1] = -1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2148 | ins->ssa_args.inline_constant = true; |
| 2149 | ins->inline_constant = scaled_constant; |
| 2150 | } |
| 2151 | } |
| 2152 | } |
| 2153 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2154 | /* Dead code elimination for branches at the end of a block - only one branch |
| 2155 | * per block is legal semantically */ |
| 2156 | |
| 2157 | static void |
| 2158 | midgard_opt_cull_dead_branch(compiler_context *ctx, midgard_block *block) |
| 2159 | { |
| 2160 | bool branched = false; |
| 2161 | |
| 2162 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2163 | if (!midgard_is_branch_unit(ins->unit)) continue; |
| 2164 | |
| 2165 | /* We ignore prepacked branches since the fragment epilogue is |
| 2166 | * just generally special */ |
| 2167 | if (ins->prepacked_branch) continue; |
| 2168 | |
Alyssa Rosenzweig | e9703fb | 2019-06-10 08:21:24 -0700 | [diff] [blame] | 2169 | /* Discards are similarly special and may not correspond to the |
| 2170 | * end of a block */ |
| 2171 | |
| 2172 | if (ins->branch.target_type == TARGET_DISCARD) continue; |
| 2173 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2174 | if (branched) { |
| 2175 | /* We already branched, so this is dead */ |
| 2176 | mir_remove_instruction(ins); |
| 2177 | } |
| 2178 | |
| 2179 | branched = true; |
| 2180 | } |
| 2181 | } |
| 2182 | |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2183 | /* fmov.pos is an idiom for fpos. Propoagate the .pos up to the source, so then |
| 2184 | * the move can be propagated away entirely */ |
| 2185 | |
| 2186 | static bool |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 2187 | mir_compose_float_outmod(midgard_outmod_float *outmod, midgard_outmod_float comp) |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2188 | { |
| 2189 | /* Nothing to do */ |
| 2190 | if (comp == midgard_outmod_none) |
| 2191 | return true; |
| 2192 | |
| 2193 | if (*outmod == midgard_outmod_none) { |
| 2194 | *outmod = comp; |
| 2195 | return true; |
| 2196 | } |
| 2197 | |
| 2198 | /* TODO: Compose rules */ |
| 2199 | return false; |
| 2200 | } |
| 2201 | |
| 2202 | static bool |
| 2203 | midgard_opt_pos_propagate(compiler_context *ctx, midgard_block *block) |
| 2204 | { |
| 2205 | bool progress = false; |
| 2206 | |
| 2207 | mir_foreach_instr_in_block_safe(block, ins) { |
| 2208 | if (ins->type != TAG_ALU_4) continue; |
| 2209 | if (ins->alu.op != midgard_alu_op_fmov) continue; |
| 2210 | if (ins->alu.outmod != midgard_outmod_pos) continue; |
| 2211 | |
| 2212 | /* TODO: Registers? */ |
Alyssa Rosenzweig | d4bcca1 | 2019-08-02 15:25:02 -0700 | [diff] [blame] | 2213 | unsigned src = ins->ssa_args.src[1]; |
Alyssa Rosenzweig | 9beb339 | 2019-07-26 11:30:06 -0700 | [diff] [blame] | 2214 | if (src & IS_REG) continue; |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 2215 | assert(!mir_has_multiple_writes(ctx, src)); |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2216 | |
| 2217 | /* There might be a source modifier, too */ |
| 2218 | if (mir_nontrivial_source2_mod(ins)) continue; |
| 2219 | |
| 2220 | /* Backpropagate the modifier */ |
| 2221 | mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) { |
| 2222 | if (v->type != TAG_ALU_4) continue; |
| 2223 | if (v->ssa_args.dest != src) continue; |
| 2224 | |
Alyssa Rosenzweig | 6780481 | 2019-06-05 15:17:45 -0700 | [diff] [blame] | 2225 | /* Can we even take a float outmod? */ |
| 2226 | if (midgard_is_integer_out_op(v->alu.op)) continue; |
| 2227 | |
| 2228 | midgard_outmod_float temp = v->alu.outmod; |
| 2229 | progress |= mir_compose_float_outmod(&temp, ins->alu.outmod); |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2230 | |
| 2231 | /* Throw in the towel.. */ |
| 2232 | if (!progress) break; |
| 2233 | |
| 2234 | /* Otherwise, transfer the modifier */ |
| 2235 | v->alu.outmod = temp; |
| 2236 | ins->alu.outmod = midgard_outmod_none; |
| 2237 | |
| 2238 | break; |
| 2239 | } |
| 2240 | } |
| 2241 | |
| 2242 | return progress; |
| 2243 | } |
| 2244 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2245 | static void |
| 2246 | emit_fragment_epilogue(compiler_context *ctx) |
| 2247 | { |
Alyssa Rosenzweig | dff4986 | 2019-08-12 12:36:46 -0700 | [diff] [blame] | 2248 | /* Just emit the last chunk with the branch */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2249 | EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always); |
| 2250 | } |
| 2251 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2252 | static midgard_block * |
| 2253 | emit_block(compiler_context *ctx, nir_block *block) |
| 2254 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2255 | midgard_block *this_block = ctx->after_block; |
| 2256 | ctx->after_block = NULL; |
| 2257 | |
| 2258 | if (!this_block) |
| 2259 | this_block = calloc(sizeof(midgard_block), 1); |
| 2260 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2261 | list_addtail(&this_block->link, &ctx->blocks); |
| 2262 | |
| 2263 | this_block->is_scheduled = false; |
| 2264 | ++ctx->block_count; |
| 2265 | |
| 2266 | ctx->texture_index[0] = -1; |
| 2267 | ctx->texture_index[1] = -1; |
| 2268 | |
| 2269 | /* Set up current block */ |
| 2270 | list_inithead(&this_block->instructions); |
| 2271 | ctx->current_block = this_block; |
| 2272 | |
| 2273 | nir_foreach_instr(instr, block) { |
| 2274 | emit_instr(ctx, instr); |
| 2275 | ++ctx->instruction_count; |
| 2276 | } |
| 2277 | |
| 2278 | inline_alu_constants(ctx); |
| 2279 | embedded_to_inline_constant(ctx); |
| 2280 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2281 | /* Append fragment shader epilogue (value writeout) */ |
| 2282 | if (ctx->stage == MESA_SHADER_FRAGMENT) { |
| 2283 | if (block == nir_impl_last_block(ctx->func->impl)) { |
Alyssa Rosenzweig | 541b329 | 2019-07-01 15:02:40 -0700 | [diff] [blame] | 2284 | emit_fragment_epilogue(ctx); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2285 | } |
| 2286 | } |
| 2287 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2288 | /* Allow the next control flow to access us retroactively, for |
| 2289 | * branching etc */ |
| 2290 | ctx->current_block = this_block; |
| 2291 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2292 | return this_block; |
| 2293 | } |
| 2294 | |
| 2295 | static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list); |
| 2296 | |
| 2297 | static void |
| 2298 | emit_if(struct compiler_context *ctx, nir_if *nif) |
| 2299 | { |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2300 | midgard_block *before_block = ctx->current_block; |
| 2301 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2302 | /* Conditional branches expect the condition in r31.w; emit a move for |
| 2303 | * that in the _previous_ block (which is the current block). */ |
Alyssa Rosenzweig | 8b15f8a | 2019-04-21 00:09:13 +0000 | [diff] [blame] | 2304 | emit_condition(ctx, &nif->condition, true, COMPONENT_X); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2305 | |
| 2306 | /* Speculatively emit the branch, but we can't fill it in until later */ |
| 2307 | EMIT(branch, true, true); |
| 2308 | midgard_instruction *then_branch = mir_last_in_block(ctx->current_block); |
| 2309 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2310 | /* Emit the two subblocks. */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2311 | midgard_block *then_block = emit_cf_list(ctx, &nif->then_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2312 | midgard_block *end_then_block = ctx->current_block; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2313 | |
| 2314 | /* Emit a jump from the end of the then block to the end of the else */ |
| 2315 | EMIT(branch, false, false); |
| 2316 | midgard_instruction *then_exit = mir_last_in_block(ctx->current_block); |
| 2317 | |
| 2318 | /* Emit second block, and check if it's empty */ |
| 2319 | |
| 2320 | int else_idx = ctx->block_count; |
| 2321 | int count_in = ctx->instruction_count; |
| 2322 | midgard_block *else_block = emit_cf_list(ctx, &nif->else_list); |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2323 | midgard_block *end_else_block = ctx->current_block; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2324 | int after_else_idx = ctx->block_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2325 | |
| 2326 | /* Now that we have the subblocks emitted, fix up the branches */ |
| 2327 | |
| 2328 | assert(then_block); |
| 2329 | assert(else_block); |
| 2330 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2331 | if (ctx->instruction_count == count_in) { |
| 2332 | /* The else block is empty, so don't emit an exit jump */ |
| 2333 | mir_remove_instruction(then_exit); |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2334 | then_branch->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2335 | } else { |
| 2336 | then_branch->branch.target_block = else_idx; |
Alyssa Rosenzweig | 2c74709 | 2019-02-17 05:14:24 +0000 | [diff] [blame] | 2337 | then_exit->branch.target_block = after_else_idx; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2338 | } |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2339 | |
| 2340 | /* Wire up the successors */ |
| 2341 | |
| 2342 | ctx->after_block = calloc(sizeof(midgard_block), 1); |
| 2343 | |
| 2344 | midgard_block_add_successor(before_block, then_block); |
| 2345 | midgard_block_add_successor(before_block, else_block); |
| 2346 | |
| 2347 | midgard_block_add_successor(end_then_block, ctx->after_block); |
| 2348 | midgard_block_add_successor(end_else_block, ctx->after_block); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2349 | } |
| 2350 | |
| 2351 | static void |
| 2352 | emit_loop(struct compiler_context *ctx, nir_loop *nloop) |
| 2353 | { |
| 2354 | /* Remember where we are */ |
| 2355 | midgard_block *start_block = ctx->current_block; |
| 2356 | |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2357 | /* Allocate a loop number, growing the current inner loop depth */ |
| 2358 | int loop_idx = ++ctx->current_loop_depth; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2359 | |
| 2360 | /* Get index from before the body so we can loop back later */ |
| 2361 | int start_idx = ctx->block_count; |
| 2362 | |
| 2363 | /* Emit the body itself */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2364 | midgard_block *loop_block = emit_cf_list(ctx, &nloop->body); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2365 | |
| 2366 | /* Branch back to loop back */ |
| 2367 | struct midgard_instruction br_back = v_branch(false, false); |
| 2368 | br_back.branch.target_block = start_idx; |
| 2369 | emit_mir_instruction(ctx, br_back); |
| 2370 | |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2371 | /* Mark down that branch in the graph. */ |
| 2372 | midgard_block_add_successor(start_block, loop_block); |
| 2373 | midgard_block_add_successor(ctx->current_block, loop_block); |
Alyssa Rosenzweig | c0fb260 | 2019-04-21 03:29:47 +0000 | [diff] [blame] | 2374 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2375 | /* Find the index of the block about to follow us (note: we don't add |
| 2376 | * one; blocks are 0-indexed so we get a fencepost problem) */ |
| 2377 | int break_block_idx = ctx->block_count; |
| 2378 | |
| 2379 | /* Fix up the break statements we emitted to point to the right place, |
| 2380 | * now that we can allocate a block number for them */ |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2381 | ctx->after_block = calloc(sizeof(midgard_block), 1); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2382 | |
| 2383 | list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2384 | mir_foreach_instr_in_block(block, ins) { |
| 2385 | if (ins->type != TAG_ALU_4) continue; |
| 2386 | if (!ins->compact_branch) continue; |
| 2387 | if (ins->prepacked_branch) continue; |
| 2388 | |
| 2389 | /* We found a branch -- check the type to see if we need to do anything */ |
| 2390 | if (ins->branch.target_type != TARGET_BREAK) continue; |
| 2391 | |
| 2392 | /* It's a break! Check if it's our break */ |
| 2393 | if (ins->branch.target_break != loop_idx) continue; |
| 2394 | |
| 2395 | /* Okay, cool, we're breaking out of this loop. |
| 2396 | * Rewrite from a break to a goto */ |
| 2397 | |
| 2398 | ins->branch.target_type = TARGET_GOTO; |
| 2399 | ins->branch.target_block = break_block_idx; |
Alyssa Rosenzweig | 9aeb726 | 2019-08-02 13:48:27 -0700 | [diff] [blame] | 2400 | |
| 2401 | midgard_block_add_successor(block, ctx->after_block); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2402 | } |
| 2403 | } |
Alyssa Rosenzweig | 521ac6e | 2019-04-21 16:22:44 +0000 | [diff] [blame] | 2404 | |
| 2405 | /* Now that we've finished emitting the loop, free up the depth again |
| 2406 | * so we play nice with recursion amid nested loops */ |
| 2407 | --ctx->current_loop_depth; |
Alyssa Rosenzweig | 7ad6516 | 2019-07-09 11:10:49 -0700 | [diff] [blame] | 2408 | |
| 2409 | /* Dump loop stats */ |
| 2410 | ++ctx->loop_count; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2411 | } |
| 2412 | |
| 2413 | static midgard_block * |
| 2414 | emit_cf_list(struct compiler_context *ctx, struct exec_list *list) |
| 2415 | { |
| 2416 | midgard_block *start_block = NULL; |
| 2417 | |
| 2418 | foreach_list_typed(nir_cf_node, node, node, list) { |
| 2419 | switch (node->type) { |
| 2420 | case nir_cf_node_block: { |
| 2421 | midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node)); |
| 2422 | |
| 2423 | if (!start_block) |
| 2424 | start_block = block; |
| 2425 | |
| 2426 | break; |
| 2427 | } |
| 2428 | |
| 2429 | case nir_cf_node_if: |
| 2430 | emit_if(ctx, nir_cf_node_as_if(node)); |
| 2431 | break; |
| 2432 | |
| 2433 | case nir_cf_node_loop: |
| 2434 | emit_loop(ctx, nir_cf_node_as_loop(node)); |
| 2435 | break; |
| 2436 | |
| 2437 | case nir_cf_node_function: |
| 2438 | assert(0); |
| 2439 | break; |
| 2440 | } |
| 2441 | } |
| 2442 | |
| 2443 | return start_block; |
| 2444 | } |
| 2445 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2446 | /* Due to lookahead, we need to report the first tag executed in the command |
| 2447 | * stream and in branch targets. An initial block might be empty, so iterate |
| 2448 | * until we find one that 'works' */ |
| 2449 | |
| 2450 | static unsigned |
| 2451 | midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx) |
| 2452 | { |
| 2453 | midgard_block *initial_block = mir_get_block(ctx, block_idx); |
| 2454 | |
| 2455 | unsigned first_tag = 0; |
| 2456 | |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2457 | mir_foreach_block_from(ctx, initial_block, v) { |
| 2458 | midgard_bundle *initial_bundle = |
| 2459 | util_dynarray_element(&v->bundles, midgard_bundle, 0); |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2460 | |
| 2461 | if (initial_bundle) { |
| 2462 | first_tag = initial_bundle->tag; |
| 2463 | break; |
| 2464 | } |
Alyssa Rosenzweig | 73c40d6 | 2019-07-31 15:49:30 -0700 | [diff] [blame] | 2465 | } |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2466 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2467 | return first_tag; |
| 2468 | } |
| 2469 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2470 | int |
Alyssa Rosenzweig | 840b806 | 2019-07-23 07:59:00 -0700 | [diff] [blame] | 2471 | midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midgard_program *program, bool is_blend) |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2472 | { |
| 2473 | struct util_dynarray *compiled = &program->compiled; |
| 2474 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2475 | midgard_debug = debug_get_option_midgard_debug(); |
Tomeu Vizoso | f0b1bbe | 2019-03-08 15:04:50 +0100 | [diff] [blame] | 2476 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2477 | compiler_context ictx = { |
| 2478 | .nir = nir, |
Alyssa Rosenzweig | 840b806 | 2019-07-23 07:59:00 -0700 | [diff] [blame] | 2479 | .screen = screen, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2480 | .stage = nir->info.stage, |
Alyssa Rosenzweig | b6946d3 | 2019-07-25 08:44:53 -0700 | [diff] [blame] | 2481 | .temp_alloc = 0, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2482 | |
| 2483 | .is_blend = is_blend, |
Alyssa Rosenzweig | c9af770 | 2019-07-05 16:51:30 -0700 | [diff] [blame] | 2484 | .blend_constant_offset = 0, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2485 | |
| 2486 | .alpha_ref = program->alpha_ref |
| 2487 | }; |
| 2488 | |
| 2489 | compiler_context *ctx = &ictx; |
| 2490 | |
Alyssa Rosenzweig | 3174bc9 | 2019-07-16 14:10:08 -0700 | [diff] [blame] | 2491 | /* Start off with a safe cutoff, allowing usage of all 16 work |
| 2492 | * registers. Later, we'll promote uniform reads to uniform registers |
| 2493 | * if we determine it is beneficial to do so */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2494 | ctx->uniform_cutoff = 8; |
| 2495 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2496 | /* Initialize at a global (not block) level hash tables */ |
| 2497 | |
| 2498 | ctx->ssa_constants = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2499 | ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2500 | ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2501 | |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2502 | /* Record the varying mapping for the command stream's bookkeeping */ |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2503 | |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2504 | struct exec_list *varyings = |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2505 | ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2506 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2507 | unsigned max_varying = 0; |
Alyssa Rosenzweig | b98955e | 2019-03-15 23:25:55 +0000 | [diff] [blame] | 2508 | nir_foreach_variable(var, varyings) { |
| 2509 | unsigned loc = var->data.driver_location; |
Alyssa Rosenzweig | 1f7b388 | 2019-04-20 23:39:29 +0000 | [diff] [blame] | 2510 | unsigned sz = glsl_type_size(var->type, FALSE); |
| 2511 | |
Boris Brezillon | 749c544 | 2019-06-13 14:56:02 +0200 | [diff] [blame] | 2512 | for (int c = 0; c < sz; ++c) { |
| 2513 | program->varyings[loc + c] = var->data.location + c; |
| 2514 | max_varying = MAX2(max_varying, loc + c); |
Alyssa Rosenzweig | 1f7b388 | 2019-04-20 23:39:29 +0000 | [diff] [blame] | 2515 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2516 | } |
| 2517 | |
Alyssa Rosenzweig | de8d49a | 2019-06-06 09:15:26 -0700 | [diff] [blame] | 2518 | /* Lower gl_Position pre-optimisation, but after lowering vars to ssa |
| 2519 | * (so we don't accidentally duplicate the epilogue since mesa/st has |
| 2520 | * messed with our I/O quite a bit already) */ |
| 2521 | |
| 2522 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2523 | |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2524 | if (ctx->stage == MESA_SHADER_VERTEX) { |
Alyssa Rosenzweig | 1e2cb3e | 2019-04-07 16:37:28 +0000 | [diff] [blame] | 2525 | NIR_PASS_V(nir, nir_lower_viewport_transform); |
Alyssa Rosenzweig | bb483a9 | 2019-07-10 11:30:00 -0700 | [diff] [blame] | 2526 | NIR_PASS_V(nir, nir_clamp_psiz, 1.0, 1024.0); |
| 2527 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2528 | |
| 2529 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2530 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
| 2531 | NIR_PASS_V(nir, nir_split_var_copies); |
| 2532 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2533 | NIR_PASS_V(nir, nir_lower_global_vars_to_local); |
| 2534 | NIR_PASS_V(nir, nir_lower_var_copies); |
| 2535 | NIR_PASS_V(nir, nir_lower_vars_to_ssa); |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2536 | |
Eric Anholt | 771adff | 2019-04-08 16:32:01 -0700 | [diff] [blame] | 2537 | NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2538 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2539 | /* Optimisation passes */ |
| 2540 | |
| 2541 | optimise_nir(nir); |
| 2542 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2543 | if (midgard_debug & MIDGARD_DBG_SHADERS) { |
| 2544 | nir_print_shader(nir, stdout); |
| 2545 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2546 | |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2547 | /* Assign sysvals and counts, now that we're sure |
| 2548 | * (post-optimisation) */ |
| 2549 | |
| 2550 | midgard_nir_assign_sysvals(ctx, nir); |
| 2551 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2552 | program->uniform_count = nir->num_uniforms; |
Alyssa Rosenzweig | 7e8de5a | 2019-04-03 01:48:09 +0000 | [diff] [blame] | 2553 | program->sysval_count = ctx->sysval_count; |
| 2554 | memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2555 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2556 | nir_foreach_function(func, nir) { |
| 2557 | if (!func->impl) |
| 2558 | continue; |
| 2559 | |
| 2560 | list_inithead(&ctx->blocks); |
| 2561 | ctx->block_count = 0; |
| 2562 | ctx->func = func; |
| 2563 | |
| 2564 | emit_cf_list(ctx, &func->impl->body); |
| 2565 | emit_block(ctx, func->impl->end_block); |
| 2566 | |
| 2567 | break; /* TODO: Multi-function shaders */ |
| 2568 | } |
| 2569 | |
| 2570 | util_dynarray_init(compiled, NULL); |
| 2571 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2572 | /* MIR-level optimizations */ |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2573 | |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2574 | bool progress = false; |
| 2575 | |
| 2576 | do { |
| 2577 | progress = false; |
| 2578 | |
| 2579 | mir_foreach_block(ctx, block) { |
Alyssa Rosenzweig | 4a03d37 | 2019-05-23 03:01:32 +0000 | [diff] [blame] | 2580 | progress |= midgard_opt_pos_propagate(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2581 | progress |= midgard_opt_copy_prop(ctx, block); |
| 2582 | progress |= midgard_opt_dead_code_eliminate(ctx, block); |
Alyssa Rosenzweig | 9ce7582 | 2019-07-24 15:37:24 -0700 | [diff] [blame] | 2583 | progress |= midgard_opt_combine_projection(ctx, block); |
| 2584 | progress |= midgard_opt_varying_projection(ctx, block); |
Alyssa Rosenzweig | 620c271 | 2019-07-26 13:14:55 -0700 | [diff] [blame] | 2585 | progress |= midgard_opt_not_propagate(ctx, block); |
Alyssa Rosenzweig | d066ca35 | 2019-07-26 13:32:54 -0700 | [diff] [blame] | 2586 | progress |= midgard_opt_fuse_src_invert(ctx, block); |
Alyssa Rosenzweig | b821e1b | 2019-07-26 13:08:54 -0700 | [diff] [blame] | 2587 | progress |= midgard_opt_fuse_dest_invert(ctx, block); |
Alyssa Rosenzweig | 4d995e0 | 2019-04-22 04:58:53 +0000 | [diff] [blame] | 2588 | } |
| 2589 | } while (progress); |
Alyssa Rosenzweig | 84f09ff | 2019-04-21 16:11:11 +0000 | [diff] [blame] | 2590 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2591 | mir_foreach_block(ctx, block) { |
| 2592 | midgard_lower_invert(ctx, block); |
Alyssa Rosenzweig | 8f88732 | 2019-07-29 15:11:12 -0700 | [diff] [blame] | 2593 | midgard_lower_derivatives(ctx, block); |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2594 | } |
| 2595 | |
Alyssa Rosenzweig | ae20bee | 2019-06-06 11:19:13 -0700 | [diff] [blame] | 2596 | /* Nested control-flow can result in dead branches at the end of the |
| 2597 | * block. This messes with our analysis and is just dead code, so cull |
| 2598 | * them */ |
| 2599 | mir_foreach_block(ctx, block) { |
| 2600 | midgard_opt_cull_dead_branch(ctx, block); |
| 2601 | } |
| 2602 | |
Alyssa Rosenzweig | 159abd5 | 2019-07-26 11:15:31 -0700 | [diff] [blame] | 2603 | /* Ensure we were lowered */ |
| 2604 | mir_foreach_instr_global(ctx, ins) { |
| 2605 | assert(!ins->invert); |
| 2606 | } |
| 2607 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2608 | /* Schedule! */ |
| 2609 | schedule_program(ctx); |
| 2610 | |
| 2611 | /* Now that all the bundles are scheduled and we can calculate block |
| 2612 | * sizes, emit actual branch instructions rather than placeholders */ |
| 2613 | |
| 2614 | int br_block_idx = 0; |
| 2615 | |
| 2616 | mir_foreach_block(ctx, block) { |
| 2617 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 2618 | for (int c = 0; c < bundle->instruction_count; ++c) { |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2619 | midgard_instruction *ins = bundle->instructions[c]; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2620 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2621 | if (!midgard_is_branch_unit(ins->unit)) continue; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2622 | |
| 2623 | if (ins->prepacked_branch) continue; |
| 2624 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2625 | /* Parse some basic branch info */ |
| 2626 | bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT; |
| 2627 | bool is_conditional = ins->branch.conditional; |
| 2628 | bool is_inverted = ins->branch.invert_conditional; |
| 2629 | bool is_discard = ins->branch.target_type == TARGET_DISCARD; |
| 2630 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2631 | /* Determine the block we're jumping to */ |
| 2632 | int target_number = ins->branch.target_block; |
| 2633 | |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2634 | /* Report the destination tag */ |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2635 | int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2636 | |
Alyssa Rosenzweig | 3c7abbf | 2019-05-22 04:33:21 +0000 | [diff] [blame] | 2637 | /* Count up the number of quadwords we're |
| 2638 | * jumping over = number of quadwords until |
| 2639 | * (br_block_idx, target_number) */ |
| 2640 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2641 | int quadword_offset = 0; |
| 2642 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2643 | if (is_discard) { |
Alyssa Rosenzweig | 7f75b2b | 2019-07-30 17:07:25 -0700 | [diff] [blame] | 2644 | /* Ignored */ |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2645 | } else if (target_number > br_block_idx) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2646 | /* Jump forward */ |
| 2647 | |
| 2648 | for (int idx = br_block_idx + 1; idx < target_number; ++idx) { |
| 2649 | midgard_block *blk = mir_get_block(ctx, idx); |
| 2650 | assert(blk); |
| 2651 | |
| 2652 | quadword_offset += blk->quadword_count; |
| 2653 | } |
| 2654 | } else { |
| 2655 | /* Jump backwards */ |
| 2656 | |
| 2657 | for (int idx = br_block_idx; idx >= target_number; --idx) { |
| 2658 | midgard_block *blk = mir_get_block(ctx, idx); |
| 2659 | assert(blk); |
| 2660 | |
| 2661 | quadword_offset -= blk->quadword_count; |
| 2662 | } |
| 2663 | } |
| 2664 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2665 | /* Unconditional extended branches (far jumps) |
| 2666 | * have issues, so we always use a conditional |
| 2667 | * branch, setting the condition to always for |
| 2668 | * unconditional. For compact unconditional |
| 2669 | * branches, cond isn't used so it doesn't |
| 2670 | * matter what we pick. */ |
| 2671 | |
| 2672 | midgard_condition cond = |
| 2673 | !is_conditional ? midgard_condition_always : |
| 2674 | is_inverted ? midgard_condition_false : |
| 2675 | midgard_condition_true; |
| 2676 | |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2677 | midgard_jmp_writeout_op op = |
| 2678 | is_discard ? midgard_jmp_writeout_op_discard : |
| 2679 | (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond : |
| 2680 | midgard_jmp_writeout_op_branch_cond; |
| 2681 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2682 | if (!is_compact) { |
| 2683 | midgard_branch_extended branch = |
| 2684 | midgard_create_branch_extended( |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2685 | cond, op, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2686 | dest_tag, |
| 2687 | quadword_offset); |
| 2688 | |
| 2689 | memcpy(&ins->branch_extended, &branch, sizeof(branch)); |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2690 | } else if (is_conditional || is_discard) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2691 | midgard_branch_cond branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2692 | .op = op, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2693 | .dest_tag = dest_tag, |
| 2694 | .offset = quadword_offset, |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2695 | .cond = cond |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2696 | }; |
| 2697 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2698 | assert(branch.offset == quadword_offset); |
| 2699 | |
| 2700 | memcpy(&ins->br_compact, &branch, sizeof(branch)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2701 | } else { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2702 | assert(op == midgard_jmp_writeout_op_branch_uncond); |
| 2703 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2704 | midgard_branch_uncond branch = { |
Alyssa Rosenzweig | 779e140 | 2019-02-17 23:24:39 +0000 | [diff] [blame] | 2705 | .op = op, |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2706 | .dest_tag = dest_tag, |
| 2707 | .offset = quadword_offset, |
| 2708 | .unknown = 1 |
| 2709 | }; |
| 2710 | |
Alyssa Rosenzweig | 5abb7b5 | 2019-02-17 22:09:09 +0000 | [diff] [blame] | 2711 | assert(branch.offset == quadword_offset); |
| 2712 | |
| 2713 | memcpy(&ins->br_compact, &branch, sizeof(branch)); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2714 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2715 | } |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2716 | } |
| 2717 | |
| 2718 | ++br_block_idx; |
| 2719 | } |
| 2720 | |
| 2721 | /* Emit flat binary from the instruction arrays. Iterate each block in |
| 2722 | * sequence. Save instruction boundaries such that lookahead tags can |
| 2723 | * be assigned easily */ |
| 2724 | |
| 2725 | /* Cache _all_ bundles in source order for lookahead across failed branches */ |
| 2726 | |
| 2727 | int bundle_count = 0; |
| 2728 | mir_foreach_block(ctx, block) { |
| 2729 | bundle_count += block->bundles.size / sizeof(midgard_bundle); |
| 2730 | } |
| 2731 | midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count); |
| 2732 | int bundle_idx = 0; |
| 2733 | mir_foreach_block(ctx, block) { |
| 2734 | util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) { |
| 2735 | source_order_bundles[bundle_idx++] = bundle; |
| 2736 | } |
| 2737 | } |
| 2738 | |
| 2739 | int current_bundle = 0; |
| 2740 | |
Alyssa Rosenzweig | 2a79afc | 2019-05-23 01:56:03 +0000 | [diff] [blame] | 2741 | /* Midgard prefetches instruction types, so during emission we |
| 2742 | * need to lookahead. Unless this is the last instruction, in |
| 2743 | * which we return 1. Or if this is the second to last and the |
| 2744 | * last is an ALU, then it's also 1... */ |
| 2745 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2746 | mir_foreach_block(ctx, block) { |
Alyssa Rosenzweig | d3ad8d6 | 2019-06-06 11:19:44 -0700 | [diff] [blame] | 2747 | mir_foreach_bundle_in_block(block, bundle) { |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2748 | int lookahead = 1; |
| 2749 | |
| 2750 | if (current_bundle + 1 < bundle_count) { |
| 2751 | uint8_t next = source_order_bundles[current_bundle + 1]->tag; |
| 2752 | |
| 2753 | if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) { |
| 2754 | lookahead = 1; |
| 2755 | } else { |
| 2756 | lookahead = next; |
| 2757 | } |
| 2758 | } |
| 2759 | |
| 2760 | emit_binary_bundle(ctx, bundle, compiled, lookahead); |
| 2761 | ++current_bundle; |
| 2762 | } |
| 2763 | |
| 2764 | /* TODO: Free deeper */ |
| 2765 | //util_dynarray_fini(&block->instructions); |
| 2766 | } |
| 2767 | |
| 2768 | free(source_order_bundles); |
| 2769 | |
Alyssa Rosenzweig | 5e55c11 | 2019-02-17 03:35:03 +0000 | [diff] [blame] | 2770 | /* Report the very first tag executed */ |
| 2771 | program->first_tag = midgard_get_first_tag_from_block(ctx, 0); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2772 | |
| 2773 | /* Deal with off-by-one related to the fencepost problem */ |
| 2774 | program->work_register_count = ctx->work_registers + 1; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2775 | program->uniform_cutoff = ctx->uniform_cutoff; |
| 2776 | |
| 2777 | program->blend_patch_offset = ctx->blend_constant_offset; |
Alyssa Rosenzweig | f0d0061 | 2019-07-19 16:23:52 -0700 | [diff] [blame] | 2778 | program->tls_size = ctx->tls_size; |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2779 | |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2780 | if (midgard_debug & MIDGARD_DBG_SHADERS) |
Alyssa Rosenzweig | c4a4f3d | 2019-08-14 09:19:54 -0700 | [diff] [blame] | 2781 | disassemble_midgard(program->compiled.data, program->compiled.size, false, 0, ""); |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2782 | |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2783 | if (midgard_debug & MIDGARD_DBG_SHADERDB) { |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 2784 | unsigned nr_bundles = 0, nr_ins = 0, nr_quadwords = 0; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2785 | |
| 2786 | /* Count instructions and bundles */ |
| 2787 | |
| 2788 | mir_foreach_instr_global(ctx, ins) { |
| 2789 | nr_ins++; |
| 2790 | } |
| 2791 | |
| 2792 | mir_foreach_block(ctx, block) { |
| 2793 | nr_bundles += util_dynarray_num_elements( |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2794 | &block->bundles, midgard_bundle); |
Alyssa Rosenzweig | 2d739f6 | 2019-07-09 11:16:57 -0700 | [diff] [blame] | 2795 | |
| 2796 | nr_quadwords += block->quadword_count; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2797 | } |
| 2798 | |
| 2799 | /* Calculate thread count. There are certain cutoffs by |
| 2800 | * register count for thread count */ |
| 2801 | |
| 2802 | unsigned nr_registers = program->work_register_count; |
| 2803 | |
| 2804 | unsigned nr_threads = |
| 2805 | (nr_registers <= 4) ? 4 : |
| 2806 | (nr_registers <= 8) ? 2 : |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2807 | 1; |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2808 | |
| 2809 | /* Dump stats */ |
| 2810 | |
| 2811 | fprintf(stderr, "shader%d - %s shader: " |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2812 | "%u inst, %u bundles, %u quadwords, " |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2813 | "%u registers, %u threads, %u loops, " |
| 2814 | "%d:%d spills:fills\n", |
Alyssa Rosenzweig | e4bd6fb | 2019-07-10 10:00:50 -0700 | [diff] [blame] | 2815 | SHADER_DB_COUNT++, |
| 2816 | gl_shader_stage_name(ctx->stage), |
| 2817 | nr_ins, nr_bundles, nr_quadwords, |
| 2818 | nr_registers, nr_threads, |
Alyssa Rosenzweig | e8dca7e | 2019-07-22 06:32:48 -0700 | [diff] [blame] | 2819 | ctx->loop_count, |
| 2820 | ctx->spills, ctx->fills); |
Alyssa Rosenzweig | 138e40d | 2019-07-08 16:42:29 -0700 | [diff] [blame] | 2821 | } |
| 2822 | |
| 2823 | |
Alyssa Rosenzweig | e67e072 | 2019-01-30 01:11:31 +0000 | [diff] [blame] | 2824 | return 0; |
| 2825 | } |