Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2019 Alyssa Rosenzweig |
| 3 | * Copyright (C) 2014-2017 Broadcom |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 6 | * copy of this software and associated documentation files (the "Software"), |
| 7 | * to deal in the Software without restriction, including without limitation |
| 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 9 | * and/or sell copies of the Software, and to permit persons to whom the |
| 10 | * Software is furnished to do so, subject to the following conditions: |
| 11 | * |
| 12 | * The above copyright notice and this permission notice (including the next |
| 13 | * paragraph) shall be included in all copies or substantial portions of the |
| 14 | * Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | * |
| 24 | */ |
| 25 | |
Boris Brezillon | 20b00e1 | 2019-08-02 19:18:40 +0200 | [diff] [blame] | 26 | #include <assert.h> |
| 27 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 28 | #include "drm-uapi/panfrost_drm.h" |
| 29 | |
Boris Brezillon | 154cb72 | 2019-09-14 09:58:55 +0200 | [diff] [blame] | 30 | #include "pan_bo.h" |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 31 | #include "pan_context.h" |
| 32 | #include "util/hash_table.h" |
| 33 | #include "util/ralloc.h" |
Eric Anholt | 882ca6d | 2019-06-27 15:05:31 -0700 | [diff] [blame] | 34 | #include "util/format/u_format.h" |
Alyssa Rosenzweig | 7692ad1 | 2019-06-28 18:46:43 -0700 | [diff] [blame] | 35 | #include "util/u_pack_color.h" |
Alyssa Rosenzweig | f9283ef | 2020-05-14 12:27:12 -0400 | [diff] [blame] | 36 | #include "util/rounding.h" |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 37 | #include "pan_util.h" |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 38 | #include "pan_blending.h" |
Alyssa Rosenzweig | 1c62b55 | 2020-08-05 16:16:00 -0400 | [diff] [blame] | 39 | #include "decode.h" |
Tomeu Vizoso | 6887ff4 | 2019-11-28 10:21:06 +0100 | [diff] [blame] | 40 | #include "panfrost-quirks.h" |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 41 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 42 | /* panfrost_bo_access is here to help us keep track of batch accesses to BOs |
| 43 | * and build a proper dependency graph such that batches can be pipelined for |
| 44 | * better GPU utilization. |
| 45 | * |
| 46 | * Each accessed BO has a corresponding entry in the ->accessed_bos hash table. |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 47 | * A BO is either being written or read at any time (see if writer != NULL). |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 48 | * When the last access is a write, the batch writing the BO might have read |
| 49 | * dependencies (readers that have not been executed yet and want to read the |
| 50 | * previous BO content), and when the last access is a read, all readers might |
| 51 | * depend on another batch to push its results to memory. That's what the |
| 52 | * readers/writers keep track off. |
| 53 | * There can only be one writer at any given time, if a new batch wants to |
| 54 | * write to the same BO, a dependency will be added between the new writer and |
| 55 | * the old writer (at the batch level), and panfrost_bo_access->writer will be |
| 56 | * updated to point to the new writer. |
| 57 | */ |
| 58 | struct panfrost_bo_access { |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 59 | struct util_dynarray readers; |
| 60 | struct panfrost_batch_fence *writer; |
| 61 | }; |
| 62 | |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 63 | static struct panfrost_batch_fence * |
| 64 | panfrost_create_batch_fence(struct panfrost_batch *batch) |
| 65 | { |
| 66 | struct panfrost_batch_fence *fence; |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 67 | |
| 68 | fence = rzalloc(NULL, struct panfrost_batch_fence); |
| 69 | assert(fence); |
| 70 | pipe_reference_init(&fence->reference, 1); |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 71 | fence->batch = batch; |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 72 | |
| 73 | return fence; |
| 74 | } |
| 75 | |
| 76 | static void |
| 77 | panfrost_free_batch_fence(struct panfrost_batch_fence *fence) |
| 78 | { |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 79 | ralloc_free(fence); |
| 80 | } |
| 81 | |
| 82 | void |
| 83 | panfrost_batch_fence_unreference(struct panfrost_batch_fence *fence) |
| 84 | { |
| 85 | if (pipe_reference(&fence->reference, NULL)) |
| 86 | panfrost_free_batch_fence(fence); |
| 87 | } |
| 88 | |
| 89 | void |
| 90 | panfrost_batch_fence_reference(struct panfrost_batch_fence *fence) |
| 91 | { |
| 92 | pipe_reference(NULL, &fence->reference); |
| 93 | } |
| 94 | |
Alyssa Rosenzweig | a2e4159 | 2020-08-14 18:31:05 -0400 | [diff] [blame] | 95 | static void |
| 96 | panfrost_batch_add_fbo_bos(struct panfrost_batch *batch); |
| 97 | |
Boris Brezillon | 2b771b8 | 2019-09-13 18:32:42 +0200 | [diff] [blame] | 98 | static struct panfrost_batch * |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 99 | panfrost_create_batch(struct panfrost_context *ctx, |
| 100 | const struct pipe_framebuffer_state *key) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 101 | { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 102 | struct panfrost_batch *batch = rzalloc(ctx, struct panfrost_batch); |
Alyssa Rosenzweig | 17c617c | 2020-08-17 10:31:02 -0400 | [diff] [blame] | 103 | struct panfrost_device *dev = pan_device(ctx->base.screen); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 104 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 105 | batch->ctx = ctx; |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 106 | |
Boris Brezillon | a8bd265 | 2019-09-15 09:27:14 +0200 | [diff] [blame] | 107 | batch->bos = _mesa_hash_table_create(batch, _mesa_hash_pointer, |
Alyssa Rosenzweig | 8882d6a | 2020-07-07 14:46:40 -0400 | [diff] [blame] | 108 | _mesa_key_pointer_equal); |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 109 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 110 | batch->minx = batch->miny = ~0; |
| 111 | batch->maxx = batch->maxy = 0; |
Alyssa Rosenzweig | f085474 | 2019-06-19 11:27:59 -0700 | [diff] [blame] | 112 | |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 113 | batch->out_sync = panfrost_create_batch_fence(batch); |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 114 | util_copy_framebuffer_state(&batch->key, key); |
Alyssa Rosenzweig | a2d0ea9 | 2019-07-10 10:10:31 -0700 | [diff] [blame] | 115 | |
Alyssa Rosenzweig | 17c617c | 2020-08-17 10:31:02 -0400 | [diff] [blame] | 116 | /* Preallocate the main pool, since every batch has at least one job |
| 117 | * structure so it will be used */ |
Boris Brezillon | 4047c69 | 2020-08-24 11:24:57 +0200 | [diff] [blame] | 118 | panfrost_pool_init(&batch->pool, batch, dev, 0, true); |
Alyssa Rosenzweig | 17c617c | 2020-08-17 10:31:02 -0400 | [diff] [blame] | 119 | |
| 120 | /* Don't preallocate the invisible pool, since not every batch will use |
| 121 | * the pre-allocation, particularly if the varyings are larger than the |
| 122 | * preallocation and a reallocation is needed after anyway. */ |
Boris Brezillon | 4047c69 | 2020-08-24 11:24:57 +0200 | [diff] [blame] | 123 | panfrost_pool_init(&batch->invisible_pool, batch, dev, PAN_BO_INVISIBLE, false); |
Alyssa Rosenzweig | 8882d6a | 2020-07-07 14:46:40 -0400 | [diff] [blame] | 124 | |
Alyssa Rosenzweig | a2e4159 | 2020-08-14 18:31:05 -0400 | [diff] [blame] | 125 | panfrost_batch_add_fbo_bos(batch); |
| 126 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 127 | return batch; |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 128 | } |
| 129 | |
Boris Brezillon | 2b771b8 | 2019-09-13 18:32:42 +0200 | [diff] [blame] | 130 | static void |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 131 | panfrost_freeze_batch(struct panfrost_batch *batch) |
| 132 | { |
| 133 | struct panfrost_context *ctx = batch->ctx; |
| 134 | struct hash_entry *entry; |
| 135 | |
| 136 | /* Remove the entry in the FBO -> batch hash table if the batch |
Alyssa Rosenzweig | 4b21c8b | 2020-08-14 18:22:36 -0400 | [diff] [blame] | 137 | * matches and drop the context reference. This way, next draws/clears |
| 138 | * targeting this FBO will trigger the creation of a new batch. |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 139 | */ |
| 140 | entry = _mesa_hash_table_search(ctx->batches, &batch->key); |
| 141 | if (entry && entry->data == batch) |
| 142 | _mesa_hash_table_remove(ctx->batches, entry); |
| 143 | |
Alyssa Rosenzweig | 4b21c8b | 2020-08-14 18:22:36 -0400 | [diff] [blame] | 144 | if (ctx->batch == batch) |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 145 | ctx->batch = NULL; |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 146 | } |
| 147 | |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 148 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 149 | static bool panfrost_batch_is_frozen(struct panfrost_batch *batch) |
| 150 | { |
| 151 | struct panfrost_context *ctx = batch->ctx; |
| 152 | struct hash_entry *entry; |
| 153 | |
| 154 | entry = _mesa_hash_table_search(ctx->batches, &batch->key); |
| 155 | if (entry && entry->data == batch) |
| 156 | return false; |
| 157 | |
| 158 | if (ctx->batch == batch) |
| 159 | return false; |
| 160 | |
| 161 | return true; |
| 162 | } |
| 163 | #endif |
| 164 | |
| 165 | static void |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 166 | panfrost_free_batch(struct panfrost_batch *batch) |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 167 | { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 168 | if (!batch) |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 169 | return; |
| 170 | |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 171 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 172 | assert(panfrost_batch_is_frozen(batch)); |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 173 | #endif |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 174 | |
Boris Brezillon | a8bd265 | 2019-09-15 09:27:14 +0200 | [diff] [blame] | 175 | hash_table_foreach(batch->bos, entry) |
| 176 | panfrost_bo_unreference((struct panfrost_bo *)entry->key); |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 177 | |
Boris Brezillon | d87ab72 | 2020-08-24 11:48:10 +0200 | [diff] [blame] | 178 | panfrost_pool_cleanup(&batch->pool); |
| 179 | panfrost_pool_cleanup(&batch->invisible_pool); |
Alyssa Rosenzweig | 17c617c | 2020-08-17 10:31:02 -0400 | [diff] [blame] | 180 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 181 | util_dynarray_foreach(&batch->dependencies, |
| 182 | struct panfrost_batch_fence *, dep) { |
| 183 | panfrost_batch_fence_unreference(*dep); |
| 184 | } |
| 185 | |
Alyssa Rosenzweig | 1cb47f8 | 2020-08-18 08:41:37 -0400 | [diff] [blame] | 186 | util_dynarray_fini(&batch->dependencies); |
| 187 | |
Boris Brezillon | 6936b7f | 2019-09-15 10:27:07 +0200 | [diff] [blame] | 188 | /* The out_sync fence lifetime is different from the the batch one |
| 189 | * since other batches might want to wait on a fence of already |
| 190 | * submitted/signaled batch. All we need to do here is make sure the |
| 191 | * fence does not point to an invalid batch, which the core will |
| 192 | * interpret as 'batch is already submitted'. |
| 193 | */ |
| 194 | batch->out_sync->batch = NULL; |
| 195 | panfrost_batch_fence_unreference(batch->out_sync); |
| 196 | |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 197 | util_unreference_framebuffer_state(&batch->key); |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 198 | ralloc_free(batch); |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 199 | } |
| 200 | |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 201 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 202 | static bool |
| 203 | panfrost_dep_graph_contains_batch(struct panfrost_batch *root, |
| 204 | struct panfrost_batch *batch) |
| 205 | { |
| 206 | if (!root) |
| 207 | return false; |
| 208 | |
| 209 | util_dynarray_foreach(&root->dependencies, |
| 210 | struct panfrost_batch_fence *, dep) { |
| 211 | if ((*dep)->batch == batch || |
| 212 | panfrost_dep_graph_contains_batch((*dep)->batch, batch)) |
| 213 | return true; |
| 214 | } |
| 215 | |
| 216 | return false; |
| 217 | } |
| 218 | #endif |
| 219 | |
| 220 | static void |
| 221 | panfrost_batch_add_dep(struct panfrost_batch *batch, |
| 222 | struct panfrost_batch_fence *newdep) |
| 223 | { |
| 224 | if (batch == newdep->batch) |
| 225 | return; |
| 226 | |
| 227 | /* We might want to turn ->dependencies into a set if the number of |
| 228 | * deps turns out to be big enough to make this 'is dep already there' |
| 229 | * search inefficient. |
| 230 | */ |
| 231 | util_dynarray_foreach(&batch->dependencies, |
| 232 | struct panfrost_batch_fence *, dep) { |
| 233 | if (*dep == newdep) |
| 234 | return; |
| 235 | } |
| 236 | |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 237 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 238 | /* Make sure the dependency graph is acyclic. */ |
| 239 | assert(!panfrost_dep_graph_contains_batch(newdep->batch, batch)); |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 240 | #endif |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 241 | |
| 242 | panfrost_batch_fence_reference(newdep); |
| 243 | util_dynarray_append(&batch->dependencies, |
| 244 | struct panfrost_batch_fence *, newdep); |
| 245 | |
| 246 | /* We now have a batch depending on us, let's make sure new draw/clear |
| 247 | * calls targeting the same FBO use a new batch object. |
| 248 | */ |
| 249 | if (newdep->batch) |
| 250 | panfrost_freeze_batch(newdep->batch); |
| 251 | } |
| 252 | |
Boris Brezillon | 2b771b8 | 2019-09-13 18:32:42 +0200 | [diff] [blame] | 253 | static struct panfrost_batch * |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 254 | panfrost_get_batch(struct panfrost_context *ctx, |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 255 | const struct pipe_framebuffer_state *key) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 256 | { |
| 257 | /* Lookup the job first */ |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 258 | struct hash_entry *entry = _mesa_hash_table_search(ctx->batches, key); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 259 | |
| 260 | if (entry) |
| 261 | return entry->data; |
| 262 | |
| 263 | /* Otherwise, let's create a job */ |
| 264 | |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 265 | struct panfrost_batch *batch = panfrost_create_batch(ctx, key); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 266 | |
| 267 | /* Save the created job */ |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 268 | _mesa_hash_table_insert(ctx->batches, &batch->key, batch); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 269 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 270 | return batch; |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 271 | } |
| 272 | |
| 273 | /* Get the job corresponding to the FBO we're currently rendering into */ |
| 274 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 275 | struct panfrost_batch * |
| 276 | panfrost_get_batch_for_fbo(struct panfrost_context *ctx) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 277 | { |
Alyssa Rosenzweig | f085474 | 2019-06-19 11:27:59 -0700 | [diff] [blame] | 278 | /* If we're wallpapering, we special case to workaround |
| 279 | * u_blitter abuse */ |
| 280 | |
| 281 | if (ctx->wallpaper_batch) |
| 282 | return ctx->wallpaper_batch; |
| 283 | |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 284 | /* If we already began rendering, use that */ |
| 285 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 286 | if (ctx->batch) { |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 287 | assert(util_framebuffer_state_equal(&ctx->batch->key, |
| 288 | &ctx->pipe_framebuffer)); |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 289 | return ctx->batch; |
Boris Brezillon | 20b00e1 | 2019-08-02 19:18:40 +0200 | [diff] [blame] | 290 | } |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 291 | |
| 292 | /* If not, look up the job */ |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 293 | struct panfrost_batch *batch = panfrost_get_batch(ctx, |
| 294 | &ctx->pipe_framebuffer); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 295 | |
Boris Brezillon | 20b00e1 | 2019-08-02 19:18:40 +0200 | [diff] [blame] | 296 | /* Set this job as the current FBO job. Will be reset when updating the |
| 297 | * FB state and when submitting or releasing a job. |
| 298 | */ |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 299 | ctx->batch = batch; |
| 300 | return batch; |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 301 | } |
| 302 | |
Boris Brezillon | c138ca8 | 2019-09-19 15:52:02 +0200 | [diff] [blame] | 303 | struct panfrost_batch * |
| 304 | panfrost_get_fresh_batch_for_fbo(struct panfrost_context *ctx) |
| 305 | { |
| 306 | struct panfrost_batch *batch; |
| 307 | |
| 308 | batch = panfrost_get_batch(ctx, &ctx->pipe_framebuffer); |
| 309 | |
| 310 | /* The batch has no draw/clear queued, let's return it directly. |
| 311 | * Note that it's perfectly fine to re-use a batch with an |
| 312 | * existing clear, we'll just update it with the new clear request. |
| 313 | */ |
Alyssa Rosenzweig | 31197c2 | 2020-07-07 17:07:34 -0400 | [diff] [blame] | 314 | if (!batch->scoreboard.first_job) |
Boris Brezillon | c138ca8 | 2019-09-19 15:52:02 +0200 | [diff] [blame] | 315 | return batch; |
| 316 | |
| 317 | /* Otherwise, we need to freeze the existing one and instantiate a new |
| 318 | * one. |
| 319 | */ |
| 320 | panfrost_freeze_batch(batch); |
| 321 | return panfrost_get_batch(ctx, &ctx->pipe_framebuffer); |
| 322 | } |
| 323 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 324 | static void |
| 325 | panfrost_bo_access_gc_fences(struct panfrost_context *ctx, |
| 326 | struct panfrost_bo_access *access, |
| 327 | const struct panfrost_bo *bo) |
| 328 | { |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 329 | if (access->writer) { |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 330 | panfrost_batch_fence_unreference(access->writer); |
| 331 | access->writer = NULL; |
| 332 | } |
| 333 | |
Icecream95 | 5e8386c | 2020-01-16 09:51:17 +1300 | [diff] [blame] | 334 | struct panfrost_batch_fence **readers_array = util_dynarray_begin(&access->readers); |
| 335 | struct panfrost_batch_fence **new_readers = readers_array; |
| 336 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 337 | util_dynarray_foreach(&access->readers, struct panfrost_batch_fence *, |
| 338 | reader) { |
| 339 | if (!(*reader)) |
| 340 | continue; |
| 341 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 342 | panfrost_batch_fence_unreference(*reader); |
| 343 | *reader = NULL; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 344 | } |
| 345 | |
Icecream95 | 5e8386c | 2020-01-16 09:51:17 +1300 | [diff] [blame] | 346 | if (!util_dynarray_resize(&access->readers, struct panfrost_batch_fence *, |
| 347 | new_readers - readers_array) && |
| 348 | new_readers != readers_array) |
| 349 | unreachable("Invalid dynarray access->readers"); |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 350 | } |
| 351 | |
| 352 | /* Collect signaled fences to keep the kernel-side syncobj-map small. The |
| 353 | * idea is to collect those signaled fences at the end of each flush_all |
| 354 | * call. This function is likely to collect only fences from previous |
| 355 | * batch flushes not the one that have just have just been submitted and |
| 356 | * are probably still in flight when we trigger the garbage collection. |
| 357 | * Anyway, we need to do this garbage collection at some point if we don't |
| 358 | * want the BO access map to keep invalid entries around and retain |
| 359 | * syncobjs forever. |
| 360 | */ |
| 361 | static void |
| 362 | panfrost_gc_fences(struct panfrost_context *ctx) |
| 363 | { |
| 364 | hash_table_foreach(ctx->accessed_bos, entry) { |
| 365 | struct panfrost_bo_access *access = entry->data; |
| 366 | |
| 367 | assert(access); |
| 368 | panfrost_bo_access_gc_fences(ctx, access, entry->key); |
| 369 | if (!util_dynarray_num_elements(&access->readers, |
| 370 | struct panfrost_batch_fence *) && |
Daniel Ogorchock | 2848edc | 2020-01-06 17:33:49 -0600 | [diff] [blame] | 371 | !access->writer) { |
| 372 | ralloc_free(access); |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 373 | _mesa_hash_table_remove(ctx->accessed_bos, entry); |
Daniel Ogorchock | 2848edc | 2020-01-06 17:33:49 -0600 | [diff] [blame] | 374 | } |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 375 | } |
| 376 | } |
| 377 | |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 378 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 379 | static bool |
| 380 | panfrost_batch_in_readers(struct panfrost_batch *batch, |
| 381 | struct panfrost_bo_access *access) |
| 382 | { |
| 383 | util_dynarray_foreach(&access->readers, struct panfrost_batch_fence *, |
| 384 | reader) { |
| 385 | if (*reader && (*reader)->batch == batch) |
| 386 | return true; |
| 387 | } |
| 388 | |
| 389 | return false; |
| 390 | } |
| 391 | #endif |
| 392 | |
| 393 | static void |
| 394 | panfrost_batch_update_bo_access(struct panfrost_batch *batch, |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 395 | struct panfrost_bo *bo, bool writes, |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 396 | bool already_accessed) |
| 397 | { |
| 398 | struct panfrost_context *ctx = batch->ctx; |
| 399 | struct panfrost_bo_access *access; |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 400 | bool old_writes = false; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 401 | struct hash_entry *entry; |
| 402 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 403 | entry = _mesa_hash_table_search(ctx->accessed_bos, bo); |
| 404 | access = entry ? entry->data : NULL; |
| 405 | if (access) { |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 406 | old_writes = access->writer != NULL; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 407 | } else { |
| 408 | access = rzalloc(ctx, struct panfrost_bo_access); |
| 409 | util_dynarray_init(&access->readers, access); |
| 410 | _mesa_hash_table_insert(ctx->accessed_bos, bo, access); |
| 411 | /* We are the first to access this BO, let's initialize |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 412 | * old_writes to our own access type in that case. |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 413 | */ |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 414 | old_writes = writes; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 415 | } |
| 416 | |
| 417 | assert(access); |
| 418 | |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 419 | if (writes && !old_writes) { |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 420 | /* Previous access was a read and we want to write this BO. |
| 421 | * We first need to add explicit deps between our batch and |
| 422 | * the previous readers. |
| 423 | */ |
| 424 | util_dynarray_foreach(&access->readers, |
| 425 | struct panfrost_batch_fence *, reader) { |
| 426 | /* We were already reading the BO, no need to add a dep |
| 427 | * on ourself (the acyclic check would complain about |
| 428 | * that). |
| 429 | */ |
| 430 | if (!(*reader) || (*reader)->batch == batch) |
| 431 | continue; |
| 432 | |
| 433 | panfrost_batch_add_dep(batch, *reader); |
| 434 | } |
| 435 | panfrost_batch_fence_reference(batch->out_sync); |
| 436 | |
Alyssa Rosenzweig | 20dd370 | 2020-05-26 16:57:44 -0400 | [diff] [blame] | 437 | if (access->writer) |
| 438 | panfrost_batch_fence_unreference(access->writer); |
| 439 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 440 | /* We now are the new writer. */ |
| 441 | access->writer = batch->out_sync; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 442 | |
| 443 | /* Release the previous readers and reset the readers array. */ |
| 444 | util_dynarray_foreach(&access->readers, |
| 445 | struct panfrost_batch_fence *, |
| 446 | reader) { |
| 447 | if (!*reader) |
| 448 | continue; |
| 449 | panfrost_batch_fence_unreference(*reader); |
| 450 | } |
| 451 | |
| 452 | util_dynarray_clear(&access->readers); |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 453 | } else if (writes && old_writes) { |
| 454 | /* First check if we were the previous writer, in that case |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 455 | * there's nothing to do. Otherwise we need to add a |
| 456 | * dependency between the new writer and the old one. |
| 457 | */ |
| 458 | if (access->writer != batch->out_sync) { |
| 459 | if (access->writer) { |
| 460 | panfrost_batch_add_dep(batch, access->writer); |
| 461 | panfrost_batch_fence_unreference(access->writer); |
| 462 | } |
| 463 | panfrost_batch_fence_reference(batch->out_sync); |
| 464 | access->writer = batch->out_sync; |
| 465 | } |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 466 | } else if (!writes && old_writes) { |
| 467 | /* First check if we were the previous writer, in that case |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 468 | * we want to keep the access type unchanged, as a write is |
| 469 | * more constraining than a read. |
| 470 | */ |
| 471 | if (access->writer != batch->out_sync) { |
| 472 | /* Add a dependency on the previous writer. */ |
| 473 | panfrost_batch_add_dep(batch, access->writer); |
| 474 | |
| 475 | /* The previous access was a write, there's no reason |
| 476 | * to have entries in the readers array. |
| 477 | */ |
| 478 | assert(!util_dynarray_num_elements(&access->readers, |
| 479 | struct panfrost_batch_fence *)); |
| 480 | |
| 481 | /* Add ourselves to the readers array. */ |
| 482 | panfrost_batch_fence_reference(batch->out_sync); |
| 483 | util_dynarray_append(&access->readers, |
| 484 | struct panfrost_batch_fence *, |
| 485 | batch->out_sync); |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 486 | access->writer = NULL; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 487 | } |
| 488 | } else { |
| 489 | /* We already accessed this BO before, so we should already be |
| 490 | * in the reader array. |
| 491 | */ |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 492 | #ifdef PAN_BATCH_DEBUG |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 493 | if (already_accessed) { |
| 494 | assert(panfrost_batch_in_readers(batch, access)); |
| 495 | return; |
| 496 | } |
Alyssa Rosenzweig | 1d647b1 | 2020-05-22 21:49:06 -0400 | [diff] [blame] | 497 | #endif |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 498 | |
| 499 | /* Previous access was a read and we want to read this BO. |
| 500 | * Add ourselves to the readers array and add a dependency on |
| 501 | * the previous writer if any. |
| 502 | */ |
| 503 | panfrost_batch_fence_reference(batch->out_sync); |
| 504 | util_dynarray_append(&access->readers, |
| 505 | struct panfrost_batch_fence *, |
| 506 | batch->out_sync); |
| 507 | |
| 508 | if (access->writer) |
| 509 | panfrost_batch_add_dep(batch, access->writer); |
| 510 | } |
| 511 | } |
| 512 | |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 513 | void |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 514 | panfrost_batch_add_bo(struct panfrost_batch *batch, struct panfrost_bo *bo, |
| 515 | uint32_t flags) |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 516 | { |
| 517 | if (!bo) |
| 518 | return; |
| 519 | |
Boris Brezillon | a8bd265 | 2019-09-15 09:27:14 +0200 | [diff] [blame] | 520 | struct hash_entry *entry; |
| 521 | uint32_t old_flags = 0; |
| 522 | |
| 523 | entry = _mesa_hash_table_search(batch->bos, bo); |
| 524 | if (!entry) { |
| 525 | entry = _mesa_hash_table_insert(batch->bos, bo, |
| 526 | (void *)(uintptr_t)flags); |
| 527 | panfrost_bo_reference(bo); |
| 528 | } else { |
| 529 | old_flags = (uintptr_t)entry->data; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 530 | |
| 531 | /* All batches have to agree on the shared flag. */ |
| 532 | assert((old_flags & PAN_BO_ACCESS_SHARED) == |
| 533 | (flags & PAN_BO_ACCESS_SHARED)); |
Boris Brezillon | a8bd265 | 2019-09-15 09:27:14 +0200 | [diff] [blame] | 534 | } |
| 535 | |
| 536 | assert(entry); |
| 537 | |
| 538 | if (old_flags == flags) |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 539 | return; |
| 540 | |
Boris Brezillon | a8bd265 | 2019-09-15 09:27:14 +0200 | [diff] [blame] | 541 | flags |= old_flags; |
| 542 | entry->data = (void *)(uintptr_t)flags; |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 543 | |
| 544 | /* If this is not a shared BO, we don't really care about dependency |
| 545 | * tracking. |
| 546 | */ |
| 547 | if (!(flags & PAN_BO_ACCESS_SHARED)) |
| 548 | return; |
| 549 | |
| 550 | /* All dependencies should have been flushed before we execute the |
| 551 | * wallpaper draw, so it should be harmless to skip the |
| 552 | * update_bo_access() call. |
| 553 | */ |
| 554 | if (batch == batch->ctx->wallpaper_batch) |
| 555 | return; |
| 556 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 557 | assert(flags & PAN_BO_ACCESS_RW); |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 558 | panfrost_batch_update_bo_access(batch, bo, flags & PAN_BO_ACCESS_WRITE, |
| 559 | old_flags != 0); |
Alyssa Rosenzweig | e008d4f | 2019-04-14 22:42:44 +0000 | [diff] [blame] | 560 | } |
| 561 | |
Tomeu Vizoso | a4d41a1 | 2020-05-04 09:01:19 +0200 | [diff] [blame] | 562 | static void |
| 563 | panfrost_batch_add_resource_bos(struct panfrost_batch *batch, |
| 564 | struct panfrost_resource *rsrc, |
| 565 | uint32_t flags) |
| 566 | { |
| 567 | panfrost_batch_add_bo(batch, rsrc->bo, flags); |
| 568 | |
| 569 | for (unsigned i = 0; i < MAX_MIP_LEVELS; i++) |
| 570 | if (rsrc->slices[i].checksum_bo) |
| 571 | panfrost_batch_add_bo(batch, rsrc->slices[i].checksum_bo, flags); |
Alyssa Rosenzweig | e7765a8 | 2020-05-21 16:35:48 -0400 | [diff] [blame] | 572 | |
| 573 | if (rsrc->separate_stencil) |
| 574 | panfrost_batch_add_bo(batch, rsrc->separate_stencil->bo, flags); |
Tomeu Vizoso | a4d41a1 | 2020-05-04 09:01:19 +0200 | [diff] [blame] | 575 | } |
| 576 | |
Alyssa Rosenzweig | a2e4159 | 2020-08-14 18:31:05 -0400 | [diff] [blame] | 577 | static void |
| 578 | panfrost_batch_add_fbo_bos(struct panfrost_batch *batch) |
Boris Brezillon | 0eec73a | 2019-09-14 18:40:23 +0200 | [diff] [blame] | 579 | { |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 580 | uint32_t flags = PAN_BO_ACCESS_SHARED | PAN_BO_ACCESS_WRITE | |
| 581 | PAN_BO_ACCESS_VERTEX_TILER | |
| 582 | PAN_BO_ACCESS_FRAGMENT; |
| 583 | |
Boris Brezillon | 0eec73a | 2019-09-14 18:40:23 +0200 | [diff] [blame] | 584 | for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) { |
| 585 | struct panfrost_resource *rsrc = pan_resource(batch->key.cbufs[i]->texture); |
Tomeu Vizoso | a4d41a1 | 2020-05-04 09:01:19 +0200 | [diff] [blame] | 586 | panfrost_batch_add_resource_bos(batch, rsrc, flags); |
Boris Brezillon | 0eec73a | 2019-09-14 18:40:23 +0200 | [diff] [blame] | 587 | } |
| 588 | |
| 589 | if (batch->key.zsbuf) { |
| 590 | struct panfrost_resource *rsrc = pan_resource(batch->key.zsbuf->texture); |
Tomeu Vizoso | a4d41a1 | 2020-05-04 09:01:19 +0200 | [diff] [blame] | 591 | panfrost_batch_add_resource_bos(batch, rsrc, flags); |
Boris Brezillon | 0eec73a | 2019-09-14 18:40:23 +0200 | [diff] [blame] | 592 | } |
| 593 | } |
| 594 | |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 595 | struct panfrost_bo * |
| 596 | panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size, |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 597 | uint32_t create_flags, uint32_t access_flags) |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 598 | { |
| 599 | struct panfrost_bo *bo; |
| 600 | |
Alyssa Rosenzweig | 1d88f07 | 2020-07-07 16:19:39 -0400 | [diff] [blame] | 601 | bo = panfrost_bo_create(pan_device(batch->ctx->base.screen), size, |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 602 | create_flags); |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 603 | panfrost_batch_add_bo(batch, bo, access_flags); |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 604 | |
| 605 | /* panfrost_batch_add_bo() has retained a reference and |
Alyssa Rosenzweig | 1d88f07 | 2020-07-07 16:19:39 -0400 | [diff] [blame] | 606 | * panfrost_bo_create() initialize the refcnt to 1, so let's |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 607 | * unreference the BO here so it gets released when the batch is |
| 608 | * destroyed (unless it's retained by someone else in the meantime). |
| 609 | */ |
| 610 | panfrost_bo_unreference(bo); |
| 611 | return bo; |
| 612 | } |
| 613 | |
Alyssa Rosenzweig | cd98d94 | 2019-08-02 19:18:48 +0200 | [diff] [blame] | 614 | /* Returns the polygon list's GPU address if available, or otherwise allocates |
| 615 | * the polygon list. It's perfectly fast to use allocate/free BO directly, |
| 616 | * since we'll hit the BO cache and this is one-per-batch anyway. */ |
| 617 | |
| 618 | mali_ptr |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 619 | panfrost_batch_get_polygon_list(struct panfrost_batch *batch, unsigned size) |
Alyssa Rosenzweig | cd98d94 | 2019-08-02 19:18:48 +0200 | [diff] [blame] | 620 | { |
| 621 | if (batch->polygon_list) { |
| 622 | assert(batch->polygon_list->size >= size); |
| 623 | } else { |
Alyssa Rosenzweig | cd98d94 | 2019-08-02 19:18:48 +0200 | [diff] [blame] | 624 | /* Create the BO as invisible, as there's no reason to map */ |
Alyssa Rosenzweig | caf55e7 | 2019-12-16 11:46:32 -0500 | [diff] [blame] | 625 | size = util_next_power_of_two(size); |
Alyssa Rosenzweig | cd98d94 | 2019-08-02 19:18:48 +0200 | [diff] [blame] | 626 | |
Boris Brezillon | 5a4d095 | 2019-09-14 17:57:06 +0200 | [diff] [blame] | 627 | batch->polygon_list = panfrost_batch_create_bo(batch, size, |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 628 | PAN_BO_INVISIBLE, |
| 629 | PAN_BO_ACCESS_PRIVATE | |
| 630 | PAN_BO_ACCESS_RW | |
| 631 | PAN_BO_ACCESS_VERTEX_TILER | |
| 632 | PAN_BO_ACCESS_FRAGMENT); |
Alyssa Rosenzweig | cd98d94 | 2019-08-02 19:18:48 +0200 | [diff] [blame] | 633 | } |
| 634 | |
| 635 | return batch->polygon_list->gpu; |
| 636 | } |
| 637 | |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 638 | struct panfrost_bo * |
Alyssa Rosenzweig | 4f7fddb | 2019-12-09 11:02:15 -0500 | [diff] [blame] | 639 | panfrost_batch_get_scratchpad(struct panfrost_batch *batch, |
Alyssa Rosenzweig | b41692c | 2020-08-17 12:30:49 -0400 | [diff] [blame] | 640 | unsigned size_per_thread, |
Alyssa Rosenzweig | 4f7fddb | 2019-12-09 11:02:15 -0500 | [diff] [blame] | 641 | unsigned thread_tls_alloc, |
| 642 | unsigned core_count) |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 643 | { |
Alyssa Rosenzweig | b41692c | 2020-08-17 12:30:49 -0400 | [diff] [blame] | 644 | unsigned size = panfrost_get_total_stack_size(size_per_thread, |
Alyssa Rosenzweig | 4f7fddb | 2019-12-09 11:02:15 -0500 | [diff] [blame] | 645 | thread_tls_alloc, |
| 646 | core_count); |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 647 | |
Alyssa Rosenzweig | 0eb84eb | 2019-12-13 14:12:48 -0500 | [diff] [blame] | 648 | if (batch->scratchpad) { |
| 649 | assert(batch->scratchpad->size >= size); |
| 650 | } else { |
| 651 | batch->scratchpad = panfrost_batch_create_bo(batch, size, |
Alyssa Rosenzweig | 4f7fddb | 2019-12-09 11:02:15 -0500 | [diff] [blame] | 652 | PAN_BO_INVISIBLE, |
| 653 | PAN_BO_ACCESS_PRIVATE | |
| 654 | PAN_BO_ACCESS_RW | |
| 655 | PAN_BO_ACCESS_VERTEX_TILER | |
| 656 | PAN_BO_ACCESS_FRAGMENT); |
Alyssa Rosenzweig | 0eb84eb | 2019-12-13 14:12:48 -0500 | [diff] [blame] | 657 | } |
| 658 | |
| 659 | return batch->scratchpad; |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 660 | } |
| 661 | |
| 662 | struct panfrost_bo * |
Alyssa Rosenzweig | 9603126 | 2020-02-06 14:29:42 -0500 | [diff] [blame] | 663 | panfrost_batch_get_shared_memory(struct panfrost_batch *batch, |
| 664 | unsigned size, |
| 665 | unsigned workgroup_count) |
| 666 | { |
| 667 | if (batch->shared_memory) { |
| 668 | assert(batch->shared_memory->size >= size); |
| 669 | } else { |
| 670 | batch->shared_memory = panfrost_batch_create_bo(batch, size, |
| 671 | PAN_BO_INVISIBLE, |
| 672 | PAN_BO_ACCESS_PRIVATE | |
| 673 | PAN_BO_ACCESS_RW | |
| 674 | PAN_BO_ACCESS_VERTEX_TILER); |
| 675 | } |
| 676 | |
| 677 | return batch->shared_memory; |
| 678 | } |
| 679 | |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 680 | mali_ptr |
Boris Brezillon | efce73d | 2020-09-08 10:11:26 +0200 | [diff] [blame^] | 681 | panfrost_batch_get_bifrost_tiler(struct panfrost_batch *batch, unsigned vertex_count) |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 682 | { |
| 683 | if (!vertex_count) |
| 684 | return 0; |
| 685 | |
| 686 | if (batch->tiler_meta) |
| 687 | return batch->tiler_meta; |
| 688 | |
Alyssa Rosenzweig | d8deb1e | 2020-08-17 13:14:54 -0400 | [diff] [blame] | 689 | struct panfrost_device *dev = pan_device(batch->ctx->base.screen); |
Boris Brezillon | efce73d | 2020-09-08 10:11:26 +0200 | [diff] [blame^] | 690 | struct panfrost_transfer t = |
| 691 | panfrost_pool_alloc_aligned(&batch->pool, MALI_BIFROST_TILER_HEAP_LENGTH, 64); |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 692 | |
Boris Brezillon | efce73d | 2020-09-08 10:11:26 +0200 | [diff] [blame^] | 693 | pan_pack(t.cpu, BIFROST_TILER_HEAP, heap) { |
| 694 | heap.size = dev->tiler_heap->size; |
| 695 | heap.base = dev->tiler_heap->gpu; |
| 696 | heap.bottom = dev->tiler_heap->gpu; |
| 697 | heap.top = dev->tiler_heap->gpu + dev->tiler_heap->size; |
| 698 | } |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 699 | |
Boris Brezillon | efce73d | 2020-09-08 10:11:26 +0200 | [diff] [blame^] | 700 | mali_ptr heap = t.gpu; |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 701 | |
Boris Brezillon | efce73d | 2020-09-08 10:11:26 +0200 | [diff] [blame^] | 702 | t = panfrost_pool_alloc_aligned(&batch->pool, MALI_BIFROST_TILER_LENGTH, 64); |
| 703 | pan_pack(t.cpu, BIFROST_TILER, tiler) { |
| 704 | tiler.hierarchy_mask = 0x28; |
| 705 | tiler.fb_width = batch->key.width; |
| 706 | tiler.fb_height = batch->key.height; |
| 707 | tiler.heap = heap; |
| 708 | } |
| 709 | |
| 710 | batch->tiler_meta = t.gpu; |
Tomeu Vizoso | 46e4246 | 2020-04-08 15:58:42 +0200 | [diff] [blame] | 711 | return batch->tiler_meta; |
| 712 | } |
| 713 | |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 714 | struct panfrost_bo * |
| 715 | panfrost_batch_get_tiler_dummy(struct panfrost_batch *batch) |
| 716 | { |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 717 | struct panfrost_device *dev = pan_device(batch->ctx->base.screen); |
Tomeu Vizoso | 6469c1a | 2019-10-29 15:42:03 +0100 | [diff] [blame] | 718 | |
| 719 | uint32_t create_flags = 0; |
| 720 | |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 721 | if (batch->tiler_dummy) |
| 722 | return batch->tiler_dummy; |
| 723 | |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 724 | if (!(dev->quirks & MIDGARD_NO_HIER_TILING)) |
Tomeu Vizoso | 6469c1a | 2019-10-29 15:42:03 +0100 | [diff] [blame] | 725 | create_flags = PAN_BO_INVISIBLE; |
| 726 | |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 727 | batch->tiler_dummy = panfrost_batch_create_bo(batch, 4096, |
Tomeu Vizoso | 6469c1a | 2019-10-29 15:42:03 +0100 | [diff] [blame] | 728 | create_flags, |
Boris Brezillon | ada752a | 2019-09-15 09:21:13 +0200 | [diff] [blame] | 729 | PAN_BO_ACCESS_PRIVATE | |
| 730 | PAN_BO_ACCESS_RW | |
| 731 | PAN_BO_ACCESS_VERTEX_TILER | |
| 732 | PAN_BO_ACCESS_FRAGMENT); |
Boris Brezillon | 1e483a8 | 2019-09-14 19:18:51 +0200 | [diff] [blame] | 733 | assert(batch->tiler_dummy); |
| 734 | return batch->tiler_dummy; |
| 735 | } |
| 736 | |
Alyssa Rosenzweig | 34a0310 | 2020-07-15 13:10:02 -0400 | [diff] [blame] | 737 | mali_ptr |
| 738 | panfrost_batch_reserve_framebuffer(struct panfrost_batch *batch) |
| 739 | { |
| 740 | struct panfrost_device *dev = pan_device(batch->ctx->base.screen); |
| 741 | |
| 742 | /* If we haven't, reserve space for the framebuffer */ |
| 743 | |
| 744 | if (!batch->framebuffer.gpu) { |
| 745 | unsigned size = (dev->quirks & MIDGARD_SFBD) ? |
Boris Brezillon | 95eb7d9 | 2020-09-06 11:01:09 +0200 | [diff] [blame] | 746 | MALI_SINGLE_TARGET_FRAMEBUFFER_LENGTH : |
Alyssa Rosenzweig | 34a0310 | 2020-07-15 13:10:02 -0400 | [diff] [blame] | 747 | sizeof(struct mali_framebuffer); |
| 748 | |
Alyssa Rosenzweig | 373a204 | 2020-08-17 14:27:57 -0400 | [diff] [blame] | 749 | batch->framebuffer = panfrost_pool_alloc_aligned(&batch->pool, size, 64); |
Alyssa Rosenzweig | 34a0310 | 2020-07-15 13:10:02 -0400 | [diff] [blame] | 750 | |
| 751 | /* Tag the pointer */ |
| 752 | if (!(dev->quirks & MIDGARD_SFBD)) |
| 753 | batch->framebuffer.gpu |= MALI_MFBD; |
| 754 | } |
| 755 | |
| 756 | return batch->framebuffer.gpu; |
| 757 | } |
| 758 | |
| 759 | |
| 760 | |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 761 | static void |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 762 | panfrost_load_surface(struct panfrost_batch *batch, struct pipe_surface *surf, unsigned loc) |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 763 | { |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 764 | if (!surf) |
Boris Brezillon | 71eda74 | 2019-09-20 08:55:54 +0200 | [diff] [blame] | 765 | return; |
| 766 | |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 767 | struct panfrost_resource *rsrc = pan_resource(surf->texture); |
| 768 | unsigned level = surf->u.tex.level; |
| 769 | |
| 770 | if (!rsrc->slices[level].initialized) |
| 771 | return; |
| 772 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 773 | if (!rsrc->damage.inverted_len) |
| 774 | return; |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 775 | |
| 776 | /* Clamp the rendering area to the damage extent. The |
| 777 | * KHR_partial_update() spec states that trying to render outside of |
| 778 | * the damage region is "undefined behavior", so we should be safe. |
| 779 | */ |
| 780 | unsigned damage_width = (rsrc->damage.extent.maxx - rsrc->damage.extent.minx); |
| 781 | unsigned damage_height = (rsrc->damage.extent.maxy - rsrc->damage.extent.miny); |
| 782 | |
| 783 | if (damage_width && damage_height) { |
| 784 | panfrost_batch_intersection_scissor(batch, |
| 785 | rsrc->damage.extent.minx, |
| 786 | rsrc->damage.extent.miny, |
| 787 | rsrc->damage.extent.maxx, |
| 788 | rsrc->damage.extent.maxy); |
| 789 | } |
| 790 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 791 | /* XXX: Native blits on Bifrost */ |
| 792 | if (batch->pool.dev->quirks & IS_BIFROST) { |
| 793 | if (loc != FRAG_RESULT_DATA0) |
| 794 | return; |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 795 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 796 | /* XXX: why align on *twice* the tile length? */ |
| 797 | batch->minx = batch->minx & ~((MALI_TILE_LENGTH * 2) - 1); |
| 798 | batch->miny = batch->miny & ~((MALI_TILE_LENGTH * 2) - 1); |
| 799 | batch->maxx = MIN2(ALIGN_POT(batch->maxx, MALI_TILE_LENGTH * 2), |
| 800 | rsrc->base.width0); |
| 801 | batch->maxy = MIN2(ALIGN_POT(batch->maxy, MALI_TILE_LENGTH * 2), |
| 802 | rsrc->base.height0); |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 803 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 804 | struct pipe_box rect; |
| 805 | batch->ctx->wallpaper_batch = batch; |
| 806 | u_box_2d(batch->minx, batch->miny, batch->maxx - batch->minx, |
| 807 | batch->maxy - batch->miny, &rect); |
| 808 | panfrost_blit_wallpaper(batch->ctx, &rect); |
| 809 | batch->ctx->wallpaper_batch = NULL; |
| 810 | return; |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 811 | } |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 812 | |
| 813 | enum pipe_format format = rsrc->base.format; |
| 814 | |
| 815 | if (loc == FRAG_RESULT_DEPTH) { |
| 816 | if (!util_format_has_depth(util_format_description(format))) |
| 817 | return; |
| 818 | |
| 819 | format = util_format_get_depth_only(format); |
| 820 | } else if (loc == FRAG_RESULT_STENCIL) { |
| 821 | if (!util_format_has_stencil(util_format_description(format))) |
| 822 | return; |
| 823 | |
| 824 | if (rsrc->separate_stencil) { |
| 825 | rsrc = rsrc->separate_stencil; |
| 826 | format = rsrc->base.format; |
| 827 | } |
| 828 | |
| 829 | format = util_format_stencil_only(format); |
| 830 | } |
| 831 | |
Alyssa Rosenzweig | f008a63 | 2020-08-11 17:27:36 -0400 | [diff] [blame] | 832 | enum mali_texture_dimension dim = |
| 833 | panfrost_translate_texture_dimension(rsrc->base.target); |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 834 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 835 | struct pan_image img = { |
| 836 | .width0 = rsrc->base.width0, |
| 837 | .height0 = rsrc->base.height0, |
| 838 | .depth0 = rsrc->base.depth0, |
| 839 | .format = format, |
Alyssa Rosenzweig | f008a63 | 2020-08-11 17:27:36 -0400 | [diff] [blame] | 840 | .dim = dim, |
Alyssa Rosenzweig | 965537df | 2020-07-22 10:23:50 -0400 | [diff] [blame] | 841 | .modifier = rsrc->modifier, |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 842 | .array_size = rsrc->base.array_size, |
| 843 | .first_level = level, |
| 844 | .last_level = level, |
| 845 | .first_layer = surf->u.tex.first_layer, |
| 846 | .last_layer = surf->u.tex.last_layer, |
Alyssa Rosenzweig | b75427c | 2020-07-21 18:54:18 -0400 | [diff] [blame] | 847 | .nr_samples = rsrc->base.nr_samples, |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 848 | .cubemap_stride = rsrc->cubemap_stride, |
| 849 | .bo = rsrc->bo, |
| 850 | .slices = rsrc->slices |
| 851 | }; |
| 852 | |
| 853 | mali_ptr blend_shader = 0; |
| 854 | |
| 855 | if (loc >= FRAG_RESULT_DATA0 && !panfrost_can_fixed_blend(rsrc->base.format)) { |
| 856 | struct panfrost_blend_shader *b = |
| 857 | panfrost_get_blend_shader(batch->ctx, &batch->ctx->blit_blend, rsrc->base.format, loc - FRAG_RESULT_DATA0); |
| 858 | |
| 859 | struct panfrost_bo *bo = panfrost_batch_create_bo(batch, b->size, |
| 860 | PAN_BO_EXECUTE, |
| 861 | PAN_BO_ACCESS_PRIVATE | |
| 862 | PAN_BO_ACCESS_READ | |
| 863 | PAN_BO_ACCESS_FRAGMENT); |
| 864 | |
| 865 | memcpy(bo->cpu, b->buffer, b->size); |
| 866 | assert(b->work_count <= 4); |
| 867 | |
| 868 | blend_shader = bo->gpu | b->first_tag; |
| 869 | } |
| 870 | |
Alyssa Rosenzweig | 373a204 | 2020-08-17 14:27:57 -0400 | [diff] [blame] | 871 | struct panfrost_transfer transfer = panfrost_pool_alloc_aligned(&batch->pool, |
| 872 | 4 * 4 * 6 * rsrc->damage.inverted_len, 64); |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 873 | |
| 874 | for (unsigned i = 0; i < rsrc->damage.inverted_len; ++i) { |
| 875 | float *o = (float *) (transfer.cpu + (4 * 4 * 6 * i)); |
| 876 | struct pan_rect r = rsrc->damage.inverted_rects[i]; |
| 877 | |
| 878 | float rect[] = { |
| 879 | r.minx, rsrc->base.height0 - r.miny, 0.0, 1.0, |
| 880 | r.maxx, rsrc->base.height0 - r.miny, 0.0, 1.0, |
| 881 | r.minx, rsrc->base.height0 - r.maxy, 0.0, 1.0, |
| 882 | |
| 883 | r.maxx, rsrc->base.height0 - r.miny, 0.0, 1.0, |
| 884 | r.minx, rsrc->base.height0 - r.maxy, 0.0, 1.0, |
| 885 | r.maxx, rsrc->base.height0 - r.maxy, 0.0, 1.0, |
| 886 | }; |
| 887 | |
| 888 | assert(sizeof(rect) == 4 * 4 * 6); |
| 889 | memcpy(o, rect, sizeof(rect)); |
| 890 | } |
| 891 | |
| 892 | panfrost_load_midg(&batch->pool, &batch->scoreboard, |
| 893 | blend_shader, |
| 894 | batch->framebuffer.gpu, transfer.gpu, |
| 895 | rsrc->damage.inverted_len * 6, |
| 896 | &img, loc); |
| 897 | |
| 898 | panfrost_batch_add_bo(batch, batch->pool.dev->blit_shaders.bo, |
| 899 | PAN_BO_ACCESS_SHARED | PAN_BO_ACCESS_READ | PAN_BO_ACCESS_FRAGMENT); |
| 900 | } |
| 901 | |
| 902 | static void |
| 903 | panfrost_batch_draw_wallpaper(struct panfrost_batch *batch) |
| 904 | { |
| 905 | panfrost_batch_reserve_framebuffer(batch); |
| 906 | |
| 907 | /* Assume combined. If either depth or stencil is written, they will |
| 908 | * both be written so we need to be careful for reloading */ |
| 909 | |
| 910 | unsigned draws = batch->draws; |
| 911 | |
| 912 | if (draws & PIPE_CLEAR_DEPTHSTENCIL) |
| 913 | draws |= PIPE_CLEAR_DEPTHSTENCIL; |
| 914 | |
| 915 | /* Mask of buffers which need reload since they are not cleared and |
| 916 | * they are drawn. (If they are cleared, reload is useless; if they are |
| 917 | * not drawn and also not cleared, we can generally omit the attachment |
| 918 | * at the framebuffer descriptor level */ |
| 919 | |
| 920 | unsigned reload = ~batch->clear & draws; |
| 921 | |
| 922 | for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) { |
| 923 | if (reload & (PIPE_CLEAR_COLOR0 << i)) |
| 924 | panfrost_load_surface(batch, batch->key.cbufs[i], FRAG_RESULT_DATA0 + i); |
| 925 | } |
| 926 | |
| 927 | if (reload & PIPE_CLEAR_DEPTH) |
| 928 | panfrost_load_surface(batch, batch->key.zsbuf, FRAG_RESULT_DEPTH); |
| 929 | |
| 930 | if (reload & PIPE_CLEAR_STENCIL) |
| 931 | panfrost_load_surface(batch, batch->key.zsbuf, FRAG_RESULT_STENCIL); |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 932 | } |
| 933 | |
Alyssa Rosenzweig | 8882d6a | 2020-07-07 14:46:40 -0400 | [diff] [blame] | 934 | static void |
| 935 | panfrost_batch_record_bo(struct hash_entry *entry, unsigned *bo_handles, unsigned idx) |
| 936 | { |
| 937 | struct panfrost_bo *bo = (struct panfrost_bo *)entry->key; |
| 938 | uint32_t flags = (uintptr_t)entry->data; |
| 939 | |
| 940 | assert(bo->gem_handle > 0); |
| 941 | bo_handles[idx] = bo->gem_handle; |
| 942 | |
| 943 | /* Update the BO access flags so that panfrost_bo_wait() knows |
| 944 | * about all pending accesses. |
| 945 | * We only keep the READ/WRITE info since this is all the BO |
| 946 | * wait logic cares about. |
| 947 | * We also preserve existing flags as this batch might not |
| 948 | * be the first one to access the BO. |
| 949 | */ |
| 950 | bo->gpu_access |= flags & (PAN_BO_ACCESS_RW); |
| 951 | } |
| 952 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 953 | static int |
| 954 | panfrost_batch_submit_ioctl(struct panfrost_batch *batch, |
| 955 | mali_ptr first_job_desc, |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 956 | uint32_t reqs, |
| 957 | uint32_t out_sync) |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 958 | { |
| 959 | struct panfrost_context *ctx = batch->ctx; |
| 960 | struct pipe_context *gallium = (struct pipe_context *) ctx; |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 961 | struct panfrost_device *dev = pan_device(gallium->screen); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 962 | struct drm_panfrost_submit submit = {0,}; |
Alyssa Rosenzweig | 85a2216 | 2020-07-20 11:55:25 -0400 | [diff] [blame] | 963 | uint32_t *bo_handles; |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 964 | int ret; |
| 965 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 966 | /* If we trace, we always need a syncobj, so make one of our own if we |
| 967 | * weren't given one to use. Remember that we did so, so we can free it |
| 968 | * after we're done but preventing double-frees if we were given a |
| 969 | * syncobj */ |
| 970 | |
| 971 | bool our_sync = false; |
| 972 | |
| 973 | if (!out_sync && dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC)) { |
| 974 | drmSyncobjCreate(dev->fd, 0, &out_sync); |
Vinson Lee | ffbdbd6 | 2020-09-14 17:53:31 -0700 | [diff] [blame] | 975 | our_sync = true; |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 976 | } |
| 977 | |
| 978 | submit.out_sync = out_sync; |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 979 | submit.jc = first_job_desc; |
| 980 | submit.requirements = reqs; |
| 981 | |
Boris Brezillon | d87ab72 | 2020-08-24 11:48:10 +0200 | [diff] [blame] | 982 | bo_handles = calloc(panfrost_pool_num_bos(&batch->pool) + |
| 983 | panfrost_pool_num_bos(&batch->invisible_pool) + |
| 984 | batch->bos->entries + 1, |
| 985 | sizeof(*bo_handles)); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 986 | assert(bo_handles); |
| 987 | |
Alyssa Rosenzweig | 8882d6a | 2020-07-07 14:46:40 -0400 | [diff] [blame] | 988 | hash_table_foreach(batch->bos, entry) |
| 989 | panfrost_batch_record_bo(entry, bo_handles, submit.bo_handle_count++); |
Boris Brezillon | 2225383 | 2019-08-31 18:51:20 +0200 | [diff] [blame] | 990 | |
Boris Brezillon | d87ab72 | 2020-08-24 11:48:10 +0200 | [diff] [blame] | 991 | panfrost_pool_get_bo_handles(&batch->pool, bo_handles + submit.bo_handle_count); |
| 992 | submit.bo_handle_count += panfrost_pool_num_bos(&batch->pool); |
| 993 | panfrost_pool_get_bo_handles(&batch->invisible_pool, bo_handles + submit.bo_handle_count); |
| 994 | submit.bo_handle_count += panfrost_pool_num_bos(&batch->invisible_pool); |
Alyssa Rosenzweig | 17c617c | 2020-08-17 10:31:02 -0400 | [diff] [blame] | 995 | |
Alyssa Rosenzweig | d8deb1e | 2020-08-17 13:14:54 -0400 | [diff] [blame] | 996 | /* Used by all tiler jobs (XXX: skip for compute-only) */ |
| 997 | if (!(reqs & PANFROST_JD_REQ_FS)) |
| 998 | bo_handles[submit.bo_handle_count++] = dev->tiler_heap->gem_handle; |
| 999 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1000 | submit.bo_handles = (u64) (uintptr_t) bo_handles; |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 1001 | ret = drmIoctl(dev->fd, DRM_IOCTL_PANFROST_SUBMIT, &submit); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1002 | free(bo_handles); |
Boris Brezillon | 819738e | 2019-09-15 10:57:26 +0200 | [diff] [blame] | 1003 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1004 | if (ret) { |
Alyssa Rosenzweig | ed1910d | 2020-07-07 16:15:45 -0400 | [diff] [blame] | 1005 | if (dev->debug & PAN_DBG_MSGS) |
| 1006 | fprintf(stderr, "Error submitting: %m\n"); |
| 1007 | |
Vinson Lee | ffbdbd6 | 2020-09-14 17:53:31 -0700 | [diff] [blame] | 1008 | if (our_sync) |
| 1009 | drmSyncobjDestroy(dev->fd, out_sync); |
| 1010 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1011 | return errno; |
| 1012 | } |
| 1013 | |
| 1014 | /* Trace the job if we're doing that */ |
Alyssa Rosenzweig | ed1910d | 2020-07-07 16:15:45 -0400 | [diff] [blame] | 1015 | if (dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC)) { |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1016 | /* Wait so we can get errors reported back */ |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1017 | drmSyncobjWait(dev->fd, &out_sync, 1, |
Boris Brezillon | 819738e | 2019-09-15 10:57:26 +0200 | [diff] [blame] | 1018 | INT64_MAX, 0, NULL); |
Alyssa Rosenzweig | c46a090 | 2020-02-18 07:50:19 -0500 | [diff] [blame] | 1019 | |
| 1020 | /* Trace gets priority over sync */ |
Alyssa Rosenzweig | ed1910d | 2020-07-07 16:15:45 -0400 | [diff] [blame] | 1021 | bool minimal = !(dev->debug & PAN_DBG_TRACE); |
Tomeu Vizoso | 30e7027 | 2020-04-07 18:22:37 +0200 | [diff] [blame] | 1022 | pandecode_jc(submit.jc, dev->quirks & IS_BIFROST, dev->gpu_id, minimal); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1023 | } |
| 1024 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1025 | /* Cleanup if we created the syncobj */ |
| 1026 | if (our_sync) |
| 1027 | drmSyncobjDestroy(dev->fd, out_sync); |
| 1028 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1029 | return 0; |
| 1030 | } |
| 1031 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1032 | /* Submit both vertex/tiler and fragment jobs for a batch, possibly with an |
| 1033 | * outsync corresponding to the later of the two (since there will be an |
| 1034 | * implicit dep between them) */ |
| 1035 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1036 | static int |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1037 | panfrost_batch_submit_jobs(struct panfrost_batch *batch, uint32_t out_sync) |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1038 | { |
Alyssa Rosenzweig | 31197c2 | 2020-07-07 17:07:34 -0400 | [diff] [blame] | 1039 | bool has_draws = batch->scoreboard.first_job; |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1040 | bool has_frag = batch->scoreboard.tiler_dep || batch->clear; |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1041 | int ret = 0; |
| 1042 | |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1043 | if (has_draws) { |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1044 | ret = panfrost_batch_submit_ioctl(batch, batch->scoreboard.first_job, |
| 1045 | 0, has_frag ? 0 : out_sync); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1046 | assert(!ret); |
| 1047 | } |
| 1048 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1049 | if (has_frag) { |
Alyssa Rosenzweig | afa4b32 | 2020-07-16 13:33:32 -0400 | [diff] [blame] | 1050 | /* Whether we program the fragment job for draws or not depends |
| 1051 | * on whether there is any *tiler* activity (so fragment |
| 1052 | * shaders). If there are draws but entirely RASTERIZER_DISCARD |
| 1053 | * (say, for transform feedback), we want a fragment job that |
| 1054 | * *only* clears, since otherwise the tiler structures will be |
| 1055 | * uninitialized leading to faults (or state leaks) */ |
| 1056 | |
| 1057 | mali_ptr fragjob = panfrost_fragment_job(batch, |
| 1058 | batch->scoreboard.tiler_dep != 0); |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1059 | ret = panfrost_batch_submit_ioctl(batch, fragjob, |
| 1060 | PANFROST_JD_REQ_FS, out_sync); |
Boris Brezillon | 0500c9e | 2019-09-14 08:00:27 +0200 | [diff] [blame] | 1061 | assert(!ret); |
| 1062 | } |
| 1063 | |
| 1064 | return ret; |
| 1065 | } |
| 1066 | |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1067 | static void |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1068 | panfrost_batch_submit(struct panfrost_batch *batch, uint32_t out_sync) |
Rohan Garg | 0f43a2a | 2019-06-05 16:20:59 +0200 | [diff] [blame] | 1069 | { |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1070 | assert(batch); |
Alyssa Rosenzweig | ed1910d | 2020-07-07 16:15:45 -0400 | [diff] [blame] | 1071 | struct panfrost_device *dev = pan_device(batch->ctx->base.screen); |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1072 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1073 | /* Submit the dependencies first. Don't pass along the out_sync since |
| 1074 | * they are guaranteed to terminate sooner */ |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 1075 | util_dynarray_foreach(&batch->dependencies, |
| 1076 | struct panfrost_batch_fence *, dep) { |
| 1077 | if ((*dep)->batch) |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1078 | panfrost_batch_submit((*dep)->batch, 0); |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 1079 | } |
| 1080 | |
Rohan Garg | 0f43a2a | 2019-06-05 16:20:59 +0200 | [diff] [blame] | 1081 | int ret; |
| 1082 | |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1083 | /* Nothing to do! */ |
Icenowy Zheng | 9e39795 | 2020-08-06 04:48:05 +0800 | [diff] [blame] | 1084 | if (!batch->scoreboard.first_job && !batch->clear) { |
| 1085 | if (out_sync) |
| 1086 | drmSyncobjSignal(dev->fd, &out_sync, 1); |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1087 | goto out; |
Icenowy Zheng | 9e39795 | 2020-08-06 04:48:05 +0800 | [diff] [blame] | 1088 | } |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1089 | |
Boris Brezillon | 71eda74 | 2019-09-20 08:55:54 +0200 | [diff] [blame] | 1090 | panfrost_batch_draw_wallpaper(batch); |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1091 | |
Alyssa Rosenzweig | b0e915b | 2019-12-09 11:00:42 -0500 | [diff] [blame] | 1092 | /* Now that all draws are in, we can finally prepare the |
| 1093 | * FBD for the batch */ |
| 1094 | |
Alyssa Rosenzweig | 31197c2 | 2020-07-07 17:07:34 -0400 | [diff] [blame] | 1095 | if (batch->framebuffer.gpu && batch->scoreboard.first_job) { |
Alyssa Rosenzweig | b0e915b | 2019-12-09 11:00:42 -0500 | [diff] [blame] | 1096 | struct panfrost_context *ctx = batch->ctx; |
| 1097 | struct pipe_context *gallium = (struct pipe_context *) ctx; |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 1098 | struct panfrost_device *dev = pan_device(gallium->screen); |
Alyssa Rosenzweig | b0e915b | 2019-12-09 11:00:42 -0500 | [diff] [blame] | 1099 | |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 1100 | if (dev->quirks & MIDGARD_SFBD) |
Alyssa Rosenzweig | b0e915b | 2019-12-09 11:00:42 -0500 | [diff] [blame] | 1101 | panfrost_attach_sfbd(batch, ~0); |
| 1102 | else |
| 1103 | panfrost_attach_mfbd(batch, ~0); |
| 1104 | } |
| 1105 | |
Alyssa Rosenzweig | fa72288 | 2020-07-07 17:14:43 -0400 | [diff] [blame] | 1106 | mali_ptr polygon_list = panfrost_batch_get_polygon_list(batch, |
Boris Brezillon | e855698 | 2020-09-05 18:16:37 +0200 | [diff] [blame] | 1107 | MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE); |
Alyssa Rosenzweig | fa72288 | 2020-07-07 17:14:43 -0400 | [diff] [blame] | 1108 | |
Alyssa Rosenzweig | 7ec6ee4 | 2020-07-07 17:19:31 -0400 | [diff] [blame] | 1109 | panfrost_scoreboard_initialize_tiler(&batch->pool, &batch->scoreboard, polygon_list); |
Alyssa Rosenzweig | f085474 | 2019-06-19 11:27:59 -0700 | [diff] [blame] | 1110 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1111 | ret = panfrost_batch_submit_jobs(batch, out_sync); |
Rohan Garg | 0f43a2a | 2019-06-05 16:20:59 +0200 | [diff] [blame] | 1112 | |
Alyssa Rosenzweig | ed1910d | 2020-07-07 16:15:45 -0400 | [diff] [blame] | 1113 | if (ret && dev->debug & PAN_DBG_MSGS) |
| 1114 | fprintf(stderr, "panfrost_batch_submit failed: %d\n", ret); |
Boris Brezillon | 20b00e1 | 2019-08-02 19:18:40 +0200 | [diff] [blame] | 1115 | |
Boris Brezillon | c6e2096 | 2019-11-14 09:35:27 +0100 | [diff] [blame] | 1116 | /* We must reset the damage info of our render targets here even |
| 1117 | * though a damage reset normally happens when the DRI layer swaps |
| 1118 | * buffers. That's because there can be implicit flushes the GL |
| 1119 | * app is not aware of, and those might impact the damage region: if |
| 1120 | * part of the damaged portion is drawn during those implicit flushes, |
| 1121 | * you have to reload those areas before next draws are pushed, and |
| 1122 | * since the driver can't easily know what's been modified by the draws |
| 1123 | * it flushed, the easiest solution is to reload everything. |
| 1124 | */ |
| 1125 | for (unsigned i = 0; i < batch->key.nr_cbufs; i++) { |
Boris Brezillon | c6e2096 | 2019-11-14 09:35:27 +0100 | [diff] [blame] | 1126 | if (!batch->key.cbufs[i]) |
| 1127 | continue; |
| 1128 | |
Alyssa Rosenzweig | 293f251 | 2020-07-09 13:42:25 -0400 | [diff] [blame] | 1129 | panfrost_resource_set_damage_region(NULL, |
| 1130 | batch->key.cbufs[i]->texture, 0, NULL); |
Boris Brezillon | c6e2096 | 2019-11-14 09:35:27 +0100 | [diff] [blame] | 1131 | } |
| 1132 | |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1133 | out: |
Boris Brezillon | 40a07bf | 2019-09-15 12:14:22 +0200 | [diff] [blame] | 1134 | panfrost_freeze_batch(batch); |
Boris Brezillon | 6ddfd37 | 2019-09-05 20:47:45 +0200 | [diff] [blame] | 1135 | panfrost_free_batch(batch); |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1136 | } |
| 1137 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1138 | /* Submit all batches, applying the out_sync to the currently bound batch */ |
| 1139 | |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1140 | void |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1141 | panfrost_flush_all_batches(struct panfrost_context *ctx, uint32_t out_sync) |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1142 | { |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1143 | struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx); |
| 1144 | panfrost_batch_submit(batch, out_sync); |
| 1145 | |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1146 | hash_table_foreach(ctx->batches, hentry) { |
| 1147 | struct panfrost_batch *batch = hentry->data; |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1148 | assert(batch); |
| 1149 | |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1150 | panfrost_batch_submit(batch, 0); |
Boris Brezillon | a45984b | 2019-09-15 19:15:16 +0200 | [diff] [blame] | 1151 | } |
| 1152 | |
| 1153 | assert(!ctx->batches->entries); |
| 1154 | |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 1155 | /* Collect batch fences before returning */ |
| 1156 | panfrost_gc_fences(ctx); |
Rohan Garg | 0f43a2a | 2019-06-05 16:20:59 +0200 | [diff] [blame] | 1157 | } |
| 1158 | |
Boris Brezillon | 7fa5cd3 | 2019-10-10 15:12:30 +0200 | [diff] [blame] | 1159 | bool |
| 1160 | panfrost_pending_batches_access_bo(struct panfrost_context *ctx, |
| 1161 | const struct panfrost_bo *bo) |
| 1162 | { |
| 1163 | struct panfrost_bo_access *access; |
| 1164 | struct hash_entry *hentry; |
| 1165 | |
| 1166 | hentry = _mesa_hash_table_search(ctx->accessed_bos, bo); |
| 1167 | access = hentry ? hentry->data : NULL; |
| 1168 | if (!access) |
| 1169 | return false; |
| 1170 | |
| 1171 | if (access->writer && access->writer->batch) |
| 1172 | return true; |
| 1173 | |
| 1174 | util_dynarray_foreach(&access->readers, struct panfrost_batch_fence *, |
| 1175 | reader) { |
| 1176 | if (*reader && (*reader)->batch) |
| 1177 | return true; |
| 1178 | } |
| 1179 | |
| 1180 | return false; |
| 1181 | } |
| 1182 | |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 1183 | /* We always flush writers. We might also need to flush readers */ |
| 1184 | |
Rohan Garg | 0f43a2a | 2019-06-05 16:20:59 +0200 | [diff] [blame] | 1185 | void |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1186 | panfrost_flush_batches_accessing_bo(struct panfrost_context *ctx, |
| 1187 | struct panfrost_bo *bo, |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 1188 | bool flush_readers) |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1189 | { |
| 1190 | struct panfrost_bo_access *access; |
| 1191 | struct hash_entry *hentry; |
| 1192 | |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1193 | hentry = _mesa_hash_table_search(ctx->accessed_bos, bo); |
| 1194 | access = hentry ? hentry->data : NULL; |
| 1195 | if (!access) |
| 1196 | return; |
| 1197 | |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 1198 | if (access->writer && access->writer->batch) |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1199 | panfrost_batch_submit(access->writer->batch, 0); |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1200 | |
Alyssa Rosenzweig | c6ebff3 | 2020-05-26 14:47:21 -0400 | [diff] [blame] | 1201 | if (!flush_readers) |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1202 | return; |
| 1203 | |
| 1204 | util_dynarray_foreach(&access->readers, struct panfrost_batch_fence *, |
| 1205 | reader) { |
| 1206 | if (*reader && (*reader)->batch) |
Alyssa Rosenzweig | 64d6f56 | 2020-07-20 13:34:42 -0400 | [diff] [blame] | 1207 | panfrost_batch_submit((*reader)->batch, 0); |
Boris Brezillon | 82399b5 | 2019-09-15 20:17:14 +0200 | [diff] [blame] | 1208 | } |
| 1209 | } |
| 1210 | |
| 1211 | void |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1212 | panfrost_batch_set_requirements(struct panfrost_batch *batch) |
Rohan Garg | bfca21b | 2019-06-05 17:49:14 +0200 | [diff] [blame] | 1213 | { |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1214 | struct panfrost_context *ctx = batch->ctx; |
| 1215 | |
Alyssa Rosenzweig | ec35159 | 2020-08-14 17:50:44 -0400 | [diff] [blame] | 1216 | if (ctx->rasterizer->base.multisample) |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1217 | batch->requirements |= PAN_REQ_MSAA; |
Rohan Garg | bfca21b | 2019-06-05 17:49:14 +0200 | [diff] [blame] | 1218 | |
Alyssa Rosenzweig | 6afd4ad | 2020-08-12 11:42:11 -0400 | [diff] [blame] | 1219 | if (ctx->depth_stencil && ctx->depth_stencil->base.depth.writemask) { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1220 | batch->requirements |= PAN_REQ_DEPTH_WRITE; |
Alyssa Rosenzweig | 5d0d8fa | 2020-07-15 17:35:58 -0400 | [diff] [blame] | 1221 | batch->draws |= PIPE_CLEAR_DEPTH; |
| 1222 | } |
| 1223 | |
Alyssa Rosenzweig | 6afd4ad | 2020-08-12 11:42:11 -0400 | [diff] [blame] | 1224 | if (ctx->depth_stencil && ctx->depth_stencil->base.stencil[0].enabled) |
Alyssa Rosenzweig | 5d0d8fa | 2020-07-15 17:35:58 -0400 | [diff] [blame] | 1225 | batch->draws |= PIPE_CLEAR_STENCIL; |
Rohan Garg | bfca21b | 2019-06-05 17:49:14 +0200 | [diff] [blame] | 1226 | } |
| 1227 | |
Boris Brezillon | 79f8850 | 2020-03-05 08:58:10 +0100 | [diff] [blame] | 1228 | void |
| 1229 | panfrost_batch_adjust_stack_size(struct panfrost_batch *batch) |
| 1230 | { |
| 1231 | struct panfrost_context *ctx = batch->ctx; |
| 1232 | |
| 1233 | for (unsigned i = 0; i < PIPE_SHADER_TYPES; ++i) { |
| 1234 | struct panfrost_shader_state *ss; |
| 1235 | |
| 1236 | ss = panfrost_get_shader_state(ctx, i); |
| 1237 | if (!ss) |
| 1238 | continue; |
| 1239 | |
| 1240 | batch->stack_size = MAX2(batch->stack_size, ss->stack_size); |
| 1241 | } |
| 1242 | } |
| 1243 | |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1244 | /* Helper to smear a 32-bit color across 128-bit components */ |
| 1245 | |
| 1246 | static void |
| 1247 | pan_pack_color_32(uint32_t *packed, uint32_t v) |
| 1248 | { |
| 1249 | for (unsigned i = 0; i < 4; ++i) |
| 1250 | packed[i] = v; |
| 1251 | } |
| 1252 | |
| 1253 | static void |
| 1254 | pan_pack_color_64(uint32_t *packed, uint32_t lo, uint32_t hi) |
| 1255 | { |
| 1256 | for (unsigned i = 0; i < 4; i += 2) { |
| 1257 | packed[i + 0] = lo; |
| 1258 | packed[i + 1] = hi; |
| 1259 | } |
| 1260 | } |
| 1261 | |
| 1262 | static void |
| 1263 | pan_pack_color(uint32_t *packed, const union pipe_color_union *color, enum pipe_format format) |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1264 | { |
| 1265 | /* Alpha magicked to 1.0 if there is no alpha */ |
| 1266 | |
| 1267 | bool has_alpha = util_format_has_alpha(format); |
| 1268 | float clear_alpha = has_alpha ? color->f[3] : 1.0f; |
| 1269 | |
| 1270 | /* Packed color depends on the framebuffer format */ |
| 1271 | |
| 1272 | const struct util_format_description *desc = |
| 1273 | util_format_description(format); |
| 1274 | |
Alyssa Rosenzweig | 6ffebfb | 2020-07-09 15:21:32 -0400 | [diff] [blame] | 1275 | if (util_format_is_rgba8_variant(desc) && desc->colorspace != UTIL_FORMAT_COLORSPACE_SRGB) { |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1276 | pan_pack_color_32(packed, |
Tomeu Vizoso | 99d4c71 | 2019-12-12 14:49:57 +0100 | [diff] [blame] | 1277 | ((uint32_t) float_to_ubyte(clear_alpha) << 24) | |
| 1278 | ((uint32_t) float_to_ubyte(color->f[2]) << 16) | |
| 1279 | ((uint32_t) float_to_ubyte(color->f[1]) << 8) | |
| 1280 | ((uint32_t) float_to_ubyte(color->f[0]) << 0)); |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1281 | } else if (format == PIPE_FORMAT_B5G6R5_UNORM) { |
| 1282 | /* First, we convert the components to R5, G6, B5 separately */ |
Alyssa Rosenzweig | 9983c4c | 2020-05-19 11:02:09 -0400 | [diff] [blame] | 1283 | unsigned r5 = _mesa_roundevenf(SATURATE(color->f[0]) * 31.0); |
| 1284 | unsigned g6 = _mesa_roundevenf(SATURATE(color->f[1]) * 63.0); |
| 1285 | unsigned b5 = _mesa_roundevenf(SATURATE(color->f[2]) * 31.0); |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1286 | |
| 1287 | /* Then we pack into a sparse u32. TODO: Why these shifts? */ |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1288 | pan_pack_color_32(packed, (b5 << 25) | (g6 << 14) | (r5 << 5)); |
| 1289 | } else if (format == PIPE_FORMAT_B4G4R4A4_UNORM) { |
Alyssa Rosenzweig | f9283ef | 2020-05-14 12:27:12 -0400 | [diff] [blame] | 1290 | /* Convert to 4-bits */ |
Alyssa Rosenzweig | 9983c4c | 2020-05-19 11:02:09 -0400 | [diff] [blame] | 1291 | unsigned r4 = _mesa_roundevenf(SATURATE(color->f[0]) * 15.0); |
| 1292 | unsigned g4 = _mesa_roundevenf(SATURATE(color->f[1]) * 15.0); |
| 1293 | unsigned b4 = _mesa_roundevenf(SATURATE(color->f[2]) * 15.0); |
| 1294 | unsigned a4 = _mesa_roundevenf(SATURATE(clear_alpha) * 15.0); |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1295 | |
| 1296 | /* Pack on *byte* intervals */ |
Alyssa Rosenzweig | f9283ef | 2020-05-14 12:27:12 -0400 | [diff] [blame] | 1297 | pan_pack_color_32(packed, (a4 << 28) | (b4 << 20) | (g4 << 12) | (r4 << 4)); |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1298 | } else if (format == PIPE_FORMAT_B5G5R5A1_UNORM) { |
| 1299 | /* Scale as expected but shift oddly */ |
Alyssa Rosenzweig | 9983c4c | 2020-05-19 11:02:09 -0400 | [diff] [blame] | 1300 | unsigned r5 = _mesa_roundevenf(SATURATE(color->f[0]) * 31.0); |
| 1301 | unsigned g5 = _mesa_roundevenf(SATURATE(color->f[1]) * 31.0); |
| 1302 | unsigned b5 = _mesa_roundevenf(SATURATE(color->f[2]) * 31.0); |
| 1303 | unsigned a1 = _mesa_roundevenf(SATURATE(clear_alpha) * 1.0); |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1304 | |
| 1305 | pan_pack_color_32(packed, (a1 << 31) | (b5 << 25) | (g5 << 15) | (r5 << 5)); |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1306 | } else { |
Alyssa Rosenzweig | c46b114 | 2020-05-14 19:33:18 -0400 | [diff] [blame] | 1307 | /* Otherwise, it's generic subject to replication */ |
Alyssa Rosenzweig | 7692ad1 | 2019-06-28 18:46:43 -0700 | [diff] [blame] | 1308 | |
Alyssa Rosenzweig | c46b114 | 2020-05-14 19:33:18 -0400 | [diff] [blame] | 1309 | union util_color out = { 0 }; |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1310 | unsigned size = util_format_get_blocksize(format); |
| 1311 | |
Alyssa Rosenzweig | c46b114 | 2020-05-14 19:33:18 -0400 | [diff] [blame] | 1312 | util_pack_color(color->f, format, &out); |
| 1313 | |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1314 | if (size == 1) { |
| 1315 | unsigned b = out.ui[0]; |
| 1316 | unsigned s = b | (b << 8); |
| 1317 | pan_pack_color_32(packed, s | (s << 16)); |
| 1318 | } else if (size == 2) |
| 1319 | pan_pack_color_32(packed, out.ui[0] | (out.ui[0] << 16)); |
Boris Brezillon | 35e92a1 | 2019-10-09 14:05:18 +0200 | [diff] [blame] | 1320 | else if (size == 3 || size == 4) |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1321 | pan_pack_color_32(packed, out.ui[0]); |
Alyssa Rosenzweig | 1b86e09 | 2019-12-31 17:52:03 -0500 | [diff] [blame] | 1322 | else if (size == 6) |
| 1323 | pan_pack_color_64(packed, out.ui[0], out.ui[1] | (out.ui[1] << 16)); /* RGB16F -- RGBB */ |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1324 | else if (size == 8) |
| 1325 | pan_pack_color_64(packed, out.ui[0], out.ui[1]); |
| 1326 | else if (size == 16) |
| 1327 | memcpy(packed, out.ui, 16); |
| 1328 | else |
| 1329 | unreachable("Unknown generic format size packing clear colour"); |
| 1330 | } |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1331 | } |
| 1332 | |
| 1333 | void |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1334 | panfrost_batch_clear(struct panfrost_batch *batch, |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1335 | unsigned buffers, |
| 1336 | const union pipe_color_union *color, |
| 1337 | double depth, unsigned stencil) |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1338 | { |
Boris Brezillon | 12d8a17 | 2019-09-05 21:41:28 +0200 | [diff] [blame] | 1339 | struct panfrost_context *ctx = batch->ctx; |
| 1340 | |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1341 | if (buffers & PIPE_CLEAR_COLOR) { |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1342 | for (unsigned i = 0; i < PIPE_MAX_COLOR_BUFS; ++i) { |
| 1343 | if (!(buffers & (PIPE_CLEAR_COLOR0 << i))) |
| 1344 | continue; |
| 1345 | |
| 1346 | enum pipe_format format = ctx->pipe_framebuffer.cbufs[i]->format; |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1347 | pan_pack_color(batch->clear_color[i], color, format); |
Alyssa Rosenzweig | 8e4e467 | 2019-07-01 11:49:06 -0700 | [diff] [blame] | 1348 | } |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1349 | } |
| 1350 | |
| 1351 | if (buffers & PIPE_CLEAR_DEPTH) { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1352 | batch->clear_depth = depth; |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1353 | } |
| 1354 | |
| 1355 | if (buffers & PIPE_CLEAR_STENCIL) { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1356 | batch->clear_stencil = stencil; |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1357 | } |
| 1358 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1359 | batch->clear |= buffers; |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 1360 | |
| 1361 | /* Clearing affects the entire framebuffer (by definition -- this is |
| 1362 | * the Gallium clear callback, which clears the whole framebuffer. If |
Marek Olšák | 8c9b9aa | 2019-12-03 20:38:14 -0500 | [diff] [blame] | 1363 | * the scissor test were enabled from the GL side, the gallium frontend |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 1364 | * would emit a quad instead and we wouldn't go down this code path) */ |
| 1365 | |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1366 | panfrost_batch_union_scissor(batch, 0, 0, |
| 1367 | ctx->pipe_framebuffer.width, |
| 1368 | ctx->pipe_framebuffer.height); |
Rohan Garg | ad284f7 | 2019-06-05 19:04:04 +0200 | [diff] [blame] | 1369 | } |
| 1370 | |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1371 | static bool |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1372 | panfrost_batch_compare(const void *a, const void *b) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1373 | { |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 1374 | return util_framebuffer_state_equal(a, b); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1375 | } |
| 1376 | |
| 1377 | static uint32_t |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1378 | panfrost_batch_hash(const void *key) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1379 | { |
Boris Brezillon | 1b5873b | 2019-09-01 10:24:30 +0200 | [diff] [blame] | 1380 | return _mesa_hash_data(key, sizeof(struct pipe_framebuffer_state)); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1381 | } |
| 1382 | |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 1383 | /* Given a new bounding rectangle (scissor), let the job cover the union of the |
| 1384 | * new and old bounding rectangles */ |
| 1385 | |
| 1386 | void |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1387 | panfrost_batch_union_scissor(struct panfrost_batch *batch, |
| 1388 | unsigned minx, unsigned miny, |
| 1389 | unsigned maxx, unsigned maxy) |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 1390 | { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1391 | batch->minx = MIN2(batch->minx, minx); |
| 1392 | batch->miny = MIN2(batch->miny, miny); |
| 1393 | batch->maxx = MAX2(batch->maxx, maxx); |
| 1394 | batch->maxy = MAX2(batch->maxy, maxy); |
Alyssa Rosenzweig | c378829 | 2019-06-18 12:30:55 -0700 | [diff] [blame] | 1395 | } |
| 1396 | |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1397 | void |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1398 | panfrost_batch_intersection_scissor(struct panfrost_batch *batch, |
Boris Brezillon | 65ae86b | 2019-08-12 12:07:08 +0200 | [diff] [blame] | 1399 | unsigned minx, unsigned miny, |
| 1400 | unsigned maxx, unsigned maxy) |
| 1401 | { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1402 | batch->minx = MAX2(batch->minx, minx); |
| 1403 | batch->miny = MAX2(batch->miny, miny); |
| 1404 | batch->maxx = MIN2(batch->maxx, maxx); |
| 1405 | batch->maxy = MIN2(batch->maxy, maxy); |
Boris Brezillon | 65ae86b | 2019-08-12 12:07:08 +0200 | [diff] [blame] | 1406 | } |
| 1407 | |
Alyssa Rosenzweig | ca8c625 | 2020-03-23 18:44:21 -0400 | [diff] [blame] | 1408 | /* Are we currently rendering to the dev (rather than an FBO)? */ |
Boris Brezillon | e46d95d | 2019-09-01 10:54:38 +0200 | [diff] [blame] | 1409 | |
| 1410 | bool |
| 1411 | panfrost_batch_is_scanout(struct panfrost_batch *batch) |
| 1412 | { |
| 1413 | /* If there is no color buffer, it's an FBO */ |
| 1414 | if (batch->key.nr_cbufs != 1) |
| 1415 | return false; |
| 1416 | |
| 1417 | /* If we're too early that no framebuffer was sent, it's scanout */ |
| 1418 | if (!batch->key.cbufs[0]) |
| 1419 | return true; |
| 1420 | |
| 1421 | return batch->key.cbufs[0]->texture->bind & PIPE_BIND_DISPLAY_TARGET || |
| 1422 | batch->key.cbufs[0]->texture->bind & PIPE_BIND_SCANOUT || |
| 1423 | batch->key.cbufs[0]->texture->bind & PIPE_BIND_SHARED; |
| 1424 | } |
| 1425 | |
Boris Brezillon | 65ae86b | 2019-08-12 12:07:08 +0200 | [diff] [blame] | 1426 | void |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1427 | panfrost_batch_init(struct panfrost_context *ctx) |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1428 | { |
Boris Brezillon | 2c52699 | 2019-09-05 21:41:26 +0200 | [diff] [blame] | 1429 | ctx->batches = _mesa_hash_table_create(ctx, |
| 1430 | panfrost_batch_hash, |
| 1431 | panfrost_batch_compare); |
Boris Brezillon | 2dad9fd | 2019-09-15 13:39:52 +0200 | [diff] [blame] | 1432 | ctx->accessed_bos = _mesa_hash_table_create(ctx, _mesa_hash_pointer, |
| 1433 | _mesa_key_pointer_equal); |
Alyssa Rosenzweig | 59c9623 | 2019-02-25 05:32:16 +0000 | [diff] [blame] | 1434 | } |