Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright © 2017 Intel Corporation |
| 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| 21 | * IN THE SOFTWARE. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #include <unistd.h> |
| 26 | #include <stdlib.h> |
| 27 | #include <stdint.h> |
| 28 | #include <stdio.h> |
| 29 | #include <string.h> |
| 30 | #include <fcntl.h> |
| 31 | #include <inttypes.h> |
| 32 | #include <errno.h> |
| 33 | #include <poll.h> |
| 34 | #include <sys/stat.h> |
| 35 | #include <sys/types.h> |
| 36 | #include <sys/ioctl.h> |
| 37 | #include <sys/time.h> |
| 38 | #include <sys/wait.h> |
| 39 | #include <time.h> |
| 40 | #include <assert.h> |
| 41 | #include <limits.h> |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 42 | #include <pthread.h> |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 43 | |
| 44 | |
| 45 | #include "intel_chipset.h" |
| 46 | #include "drm.h" |
| 47 | #include "ioctl_wrappers.h" |
| 48 | #include "drmtest.h" |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 49 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 50 | #include "intel_io.h" |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 51 | #include "igt_aux.h" |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 52 | #include "igt_rand.h" |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 53 | #include "sw_sync.h" |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 54 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 55 | #include "ewma.h" |
| 56 | |
Arkadiusz Hiler | 200d0f5 | 2017-06-07 12:11:37 +0200 | [diff] [blame] | 57 | #define LOCAL_I915_EXEC_FENCE_IN (1<<16) |
| 58 | #define LOCAL_I915_EXEC_FENCE_OUT (1<<17) |
| 59 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 60 | enum intel_engine_id { |
| 61 | RCS, |
| 62 | BCS, |
| 63 | VCS, |
| 64 | VCS1, |
| 65 | VCS2, |
| 66 | VECS, |
| 67 | NUM_ENGINES |
| 68 | }; |
| 69 | |
| 70 | struct duration { |
| 71 | unsigned int min, max; |
| 72 | }; |
| 73 | |
| 74 | enum w_type |
| 75 | { |
| 76 | BATCH, |
| 77 | SYNC, |
| 78 | DELAY, |
| 79 | PERIOD, |
| 80 | THROTTLE, |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 81 | QD_THROTTLE, |
| 82 | SW_FENCE, |
| 83 | SW_FENCE_SIGNAL |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 84 | }; |
| 85 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 86 | struct deps |
| 87 | { |
| 88 | int nr; |
| 89 | int *list; |
| 90 | }; |
| 91 | |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 92 | struct w_arg { |
| 93 | char *filename; |
| 94 | char *desc; |
| 95 | int prio; |
| 96 | }; |
| 97 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 98 | struct w_step |
| 99 | { |
| 100 | /* Workload step metadata */ |
| 101 | enum w_type type; |
| 102 | unsigned int context; |
| 103 | unsigned int engine; |
| 104 | struct duration duration; |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 105 | struct deps data_deps; |
| 106 | struct deps fence_deps; |
| 107 | int emit_fence; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 108 | union { |
| 109 | int sync; |
| 110 | int delay; |
| 111 | int period; |
| 112 | int target; |
| 113 | int throttle; |
| 114 | int fence_signal; |
| 115 | }; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 116 | |
| 117 | /* Implementation details */ |
| 118 | unsigned int idx; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 119 | struct igt_list rq_link; |
Chris Wilson | 12e2def | 2017-05-09 22:50:19 +0100 | [diff] [blame] | 120 | unsigned int request; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 121 | |
| 122 | struct drm_i915_gem_execbuffer2 eb; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 123 | struct drm_i915_gem_exec_object2 *obj; |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 124 | struct drm_i915_gem_relocation_entry reloc[4]; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 125 | unsigned long bb_sz; |
| 126 | uint32_t bb_handle; |
| 127 | uint32_t *mapped_batch; |
| 128 | uint32_t *seqno_value; |
| 129 | uint32_t *seqno_address; |
| 130 | uint32_t *rt0_value; |
| 131 | uint32_t *rt0_address; |
| 132 | uint32_t *rt1_address; |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 133 | uint32_t *latch_value; |
| 134 | uint32_t *latch_address; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 135 | unsigned int mapped_len; |
| 136 | }; |
| 137 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 138 | DECLARE_EWMA(uint64_t, rt, 4, 2) |
| 139 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 140 | struct workload |
| 141 | { |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 142 | unsigned int id; |
| 143 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 144 | unsigned int nr_steps; |
| 145 | struct w_step *steps; |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 146 | int prio; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 147 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 148 | pthread_t thread; |
| 149 | bool run; |
| 150 | bool background; |
| 151 | const struct workload_balancer *balancer; |
| 152 | unsigned int repeat; |
| 153 | unsigned int flags; |
| 154 | bool print_stats; |
| 155 | |
Chris Wilson | 62a1f54 | 2017-05-09 12:42:41 +0100 | [diff] [blame] | 156 | uint32_t prng; |
| 157 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 158 | struct timespec repeat_start; |
| 159 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 160 | unsigned int nr_ctxs; |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 161 | struct { |
| 162 | uint32_t id; |
| 163 | unsigned int static_vcs; |
| 164 | } *ctx_list; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 165 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 166 | int sync_timeline; |
| 167 | uint32_t sync_seqno; |
| 168 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 169 | uint32_t seqno[NUM_ENGINES]; |
Chris Wilson | 36dec3d | 2017-05-10 10:57:16 +0100 | [diff] [blame] | 170 | struct drm_i915_gem_exec_object2 status_object[2]; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 171 | uint32_t *status_page; |
Chris Wilson | 36dec3d | 2017-05-10 10:57:16 +0100 | [diff] [blame] | 172 | uint32_t *status_cs; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 173 | unsigned int vcs_rr; |
| 174 | |
| 175 | unsigned long qd_sum[NUM_ENGINES]; |
| 176 | unsigned long nr_bb[NUM_ENGINES]; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 177 | |
| 178 | struct igt_list requests[NUM_ENGINES]; |
| 179 | unsigned int nrequest[NUM_ENGINES]; |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 180 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 181 | struct workload *global_wrk; |
| 182 | const struct workload_balancer *global_balancer; |
| 183 | pthread_mutex_t mutex; |
| 184 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 185 | union { |
| 186 | struct rtavg { |
| 187 | struct ewma_rt avg[NUM_ENGINES]; |
| 188 | uint32_t last[NUM_ENGINES]; |
| 189 | } rt; |
| 190 | }; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 191 | }; |
| 192 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 193 | static const unsigned int nop_calibration_us = 1000; |
| 194 | static unsigned long nop_calibration; |
| 195 | |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 196 | static unsigned int context_vcs_rr; |
| 197 | |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 198 | static int verbose = 1; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 199 | static int fd; |
| 200 | |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 201 | #define SWAPVCS (1<<0) |
| 202 | #define SEQNO (1<<1) |
| 203 | #define BALANCE (1<<2) |
| 204 | #define RT (1<<3) |
| 205 | #define VCS2REMAP (1<<4) |
Tvrtko Ursulin | 7736d7e | 2017-05-09 09:21:03 +0100 | [diff] [blame] | 206 | #define INITVCSRR (1<<5) |
Tvrtko Ursulin | 8540b91 | 2017-05-09 09:39:17 +0100 | [diff] [blame] | 207 | #define SYNCEDCLIENTS (1<<6) |
Chris Wilson | 70d3814 | 2017-05-10 11:38:39 +0100 | [diff] [blame] | 208 | #define HEARTBEAT (1<<7) |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 209 | #define GLOBAL_BALANCE (1<<8) |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 210 | #define DEPSYNC (1<<9) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 211 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 212 | #define SEQNO_IDX(engine) ((engine) * 16) |
| 213 | #define SEQNO_OFFSET(engine) (SEQNO_IDX(engine) * sizeof(uint32_t)) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 214 | |
| 215 | #define RCS_TIMESTAMP (0x2000 + 0x358) |
| 216 | #define REG(x) (volatile uint32_t *)((volatile char *)igt_global_mmio + x) |
| 217 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 218 | static const char *ring_str_map[NUM_ENGINES] = { |
| 219 | [RCS] = "RCS", |
| 220 | [BCS] = "BCS", |
| 221 | [VCS] = "VCS", |
| 222 | [VCS1] = "VCS1", |
| 223 | [VCS2] = "VCS2", |
| 224 | [VECS] = "VECS", |
| 225 | }; |
| 226 | |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 227 | static int |
| 228 | parse_dependencies(unsigned int nr_steps, struct w_step *w, char *_desc) |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 229 | { |
| 230 | char *desc = strdup(_desc); |
| 231 | char *token, *tctx = NULL, *tstart = desc; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 232 | |
| 233 | igt_assert(desc); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 234 | igt_assert(!w->data_deps.nr && w->data_deps.nr == w->fence_deps.nr); |
| 235 | igt_assert(!w->data_deps.list && |
| 236 | w->data_deps.list == w->fence_deps.list); |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 237 | |
| 238 | while ((token = strtok_r(tstart, "/", &tctx)) != NULL) { |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 239 | char *str = token; |
| 240 | struct deps *deps; |
| 241 | int dep; |
| 242 | |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 243 | tstart = NULL; |
| 244 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 245 | if (strlen(token) > 1 && token[0] == 'f') { |
| 246 | deps = &w->fence_deps; |
| 247 | str++; |
| 248 | } else { |
| 249 | deps = &w->data_deps; |
| 250 | } |
| 251 | |
| 252 | dep = atoi(str); |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 253 | if (dep > 0 || ((int)nr_steps + dep) < 0) { |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 254 | if (deps->list) |
| 255 | free(deps->list); |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 256 | return -1; |
| 257 | } |
| 258 | |
| 259 | if (dep < 0) { |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 260 | deps->nr++; |
| 261 | /* Multiple fences not yet supported. */ |
| 262 | igt_assert(deps->nr == 1 || deps != &w->fence_deps); |
| 263 | deps->list = realloc(deps->list, |
| 264 | sizeof(*deps->list) * deps->nr); |
| 265 | igt_assert(deps->list); |
| 266 | deps->list[deps->nr - 1] = dep; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 267 | } |
| 268 | } |
| 269 | |
| 270 | free(desc); |
| 271 | |
| 272 | return 0; |
| 273 | } |
| 274 | |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 275 | static struct workload * |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 276 | parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 277 | { |
| 278 | struct workload *wrk; |
| 279 | unsigned int nr_steps = 0; |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 280 | char *desc = strdup(arg->desc); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 281 | char *_token, *token, *tctx = NULL, *tstart = desc; |
| 282 | char *field, *fctx = NULL, *fstart; |
| 283 | struct w_step step, *steps = NULL; |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 284 | bool bcs_used = false; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 285 | unsigned int valid; |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 286 | int i, j, tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 287 | |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 288 | igt_assert(desc); |
| 289 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 290 | while ((_token = strtok_r(tstart, ",", &tctx)) != NULL) { |
| 291 | tstart = NULL; |
| 292 | token = strdup(_token); |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 293 | igt_assert(token); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 294 | fstart = token; |
| 295 | valid = 0; |
| 296 | memset(&step, 0, sizeof(step)); |
| 297 | |
| 298 | if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { |
| 299 | fstart = NULL; |
| 300 | |
| 301 | if (!strcasecmp(field, "d")) { |
| 302 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 303 | NULL) { |
| 304 | tmp = atoi(field); |
| 305 | if (tmp <= 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 306 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 307 | fprintf(stderr, |
| 308 | "Invalid delay at step %u!\n", |
| 309 | nr_steps); |
| 310 | return NULL; |
| 311 | } |
| 312 | |
| 313 | step.type = DELAY; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 314 | step.delay = tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 315 | goto add_step; |
| 316 | } |
| 317 | } else if (!strcasecmp(field, "p")) { |
| 318 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 319 | NULL) { |
| 320 | tmp = atoi(field); |
| 321 | if (tmp <= 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 322 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 323 | fprintf(stderr, |
| 324 | "Invalid period at step %u!\n", |
| 325 | nr_steps); |
| 326 | return NULL; |
| 327 | } |
| 328 | |
| 329 | step.type = PERIOD; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 330 | step.period = tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 331 | goto add_step; |
| 332 | } |
| 333 | } else if (!strcasecmp(field, "s")) { |
| 334 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 335 | NULL) { |
| 336 | tmp = atoi(field); |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 337 | if (tmp >= 0 || |
| 338 | ((int)nr_steps + tmp) < 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 339 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 340 | fprintf(stderr, |
| 341 | "Invalid sync target at step %u!\n", |
| 342 | nr_steps); |
| 343 | return NULL; |
| 344 | } |
| 345 | |
| 346 | step.type = SYNC; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 347 | step.target = tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 348 | goto add_step; |
| 349 | } |
| 350 | } else if (!strcasecmp(field, "t")) { |
| 351 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 352 | NULL) { |
| 353 | tmp = atoi(field); |
| 354 | if (tmp < 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 355 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 356 | fprintf(stderr, |
| 357 | "Invalid throttle at step %u!\n", |
| 358 | nr_steps); |
| 359 | return NULL; |
| 360 | } |
| 361 | |
| 362 | step.type = THROTTLE; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 363 | step.throttle = tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 364 | goto add_step; |
| 365 | } |
| 366 | } else if (!strcasecmp(field, "q")) { |
| 367 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 368 | NULL) { |
| 369 | tmp = atoi(field); |
| 370 | if (tmp < 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 371 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 372 | fprintf(stderr, |
| 373 | "Invalid qd throttle at step %u!\n", |
| 374 | nr_steps); |
| 375 | return NULL; |
| 376 | } |
| 377 | |
| 378 | step.type = QD_THROTTLE; |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 379 | step.throttle = tmp; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 380 | goto add_step; |
| 381 | } |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 382 | } else if (!strcasecmp(field, "a")) { |
| 383 | if ((field = strtok_r(fstart, ".", &fctx)) != |
| 384 | NULL) { |
| 385 | tmp = atoi(field); |
| 386 | if (tmp >= 0) { |
| 387 | if (verbose) |
| 388 | fprintf(stderr, |
| 389 | "Invalid sw fence signal at step %u!\n", |
| 390 | nr_steps); |
| 391 | return NULL; |
| 392 | } |
| 393 | |
| 394 | step.type = SW_FENCE_SIGNAL; |
| 395 | step.target = tmp; |
| 396 | goto add_step; |
| 397 | } |
| 398 | } else if (!strcasecmp(field, "f")) { |
| 399 | step.type = SW_FENCE; |
| 400 | goto add_step; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 401 | } |
| 402 | |
| 403 | tmp = atoi(field); |
| 404 | if (tmp < 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 405 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 406 | fprintf(stderr, |
| 407 | "Invalid ctx id at step %u!\n", |
| 408 | nr_steps); |
| 409 | return NULL; |
| 410 | } |
| 411 | step.context = tmp; |
| 412 | |
| 413 | valid++; |
| 414 | } |
| 415 | |
| 416 | if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 417 | unsigned int old_valid = valid; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 418 | |
| 419 | fstart = NULL; |
| 420 | |
| 421 | for (i = 0; i < ARRAY_SIZE(ring_str_map); i++) { |
| 422 | if (!strcasecmp(field, ring_str_map[i])) { |
| 423 | step.engine = i; |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 424 | if (step.engine == BCS) |
| 425 | bcs_used = true; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 426 | valid++; |
| 427 | break; |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | if (old_valid == valid) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 432 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 433 | fprintf(stderr, |
| 434 | "Invalid engine id at step %u!\n", |
| 435 | nr_steps); |
| 436 | return NULL; |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { |
| 441 | char *sep = NULL; |
| 442 | long int tmpl; |
| 443 | |
| 444 | fstart = NULL; |
| 445 | |
| 446 | tmpl = strtol(field, &sep, 10); |
Tvrtko Ursulin | 424faaa | 2017-06-05 12:30:24 +0100 | [diff] [blame] | 447 | if (tmpl <= 0 || tmpl == LONG_MIN || tmpl == LONG_MAX) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 448 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 449 | fprintf(stderr, |
| 450 | "Invalid duration at step %u!\n", |
| 451 | nr_steps); |
| 452 | return NULL; |
| 453 | } |
| 454 | step.duration.min = tmpl; |
| 455 | |
| 456 | if (sep && *sep == '-') { |
| 457 | tmpl = strtol(sep + 1, NULL, 10); |
Tvrtko Ursulin | 424faaa | 2017-06-05 12:30:24 +0100 | [diff] [blame] | 458 | if (tmpl <= 0 || tmpl <= step.duration.min || |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 459 | tmpl == LONG_MIN || tmpl == LONG_MAX) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 460 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 461 | fprintf(stderr, |
| 462 | "Invalid duration range at step %u!\n", |
| 463 | nr_steps); |
| 464 | return NULL; |
| 465 | } |
| 466 | step.duration.max = tmpl; |
| 467 | } else { |
| 468 | step.duration.max = step.duration.min; |
| 469 | } |
| 470 | |
| 471 | valid++; |
| 472 | } |
| 473 | |
| 474 | if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { |
| 475 | fstart = NULL; |
| 476 | |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 477 | tmp = parse_dependencies(nr_steps, &step, field); |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 478 | if (tmp < 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 479 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 480 | fprintf(stderr, |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 481 | "Invalid dependency at step %u!\n", |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 482 | nr_steps); |
| 483 | return NULL; |
| 484 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 485 | |
| 486 | valid++; |
| 487 | } |
| 488 | |
| 489 | if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { |
| 490 | fstart = NULL; |
| 491 | |
Tvrtko Ursulin | b957642 | 2017-05-09 10:33:00 +0100 | [diff] [blame] | 492 | if (strlen(field) != 1 || |
| 493 | (field[0] != '0' && field[0] != '1')) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 494 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 495 | fprintf(stderr, |
| 496 | "Invalid wait boolean at step %u!\n", |
| 497 | nr_steps); |
| 498 | return NULL; |
| 499 | } |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 500 | step.sync = field[0] - '0'; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 501 | |
| 502 | valid++; |
| 503 | } |
| 504 | |
| 505 | if (valid != 5) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 506 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 507 | fprintf(stderr, "Invalid record at step %u!\n", |
| 508 | nr_steps); |
| 509 | return NULL; |
| 510 | } |
| 511 | |
| 512 | step.type = BATCH; |
| 513 | |
| 514 | add_step: |
| 515 | step.idx = nr_steps++; |
Chris Wilson | 12e2def | 2017-05-09 22:50:19 +0100 | [diff] [blame] | 516 | step.request = -1; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 517 | steps = realloc(steps, sizeof(step) * nr_steps); |
| 518 | igt_assert(steps); |
| 519 | |
| 520 | memcpy(&steps[nr_steps - 1], &step, sizeof(step)); |
| 521 | |
| 522 | free(token); |
| 523 | } |
| 524 | |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 525 | if (app_w) { |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 526 | steps = realloc(steps, sizeof(step) * |
| 527 | (nr_steps + app_w->nr_steps)); |
| 528 | igt_assert(steps); |
| 529 | |
| 530 | memcpy(&steps[nr_steps], app_w->steps, |
| 531 | sizeof(step) * app_w->nr_steps); |
| 532 | |
| 533 | for (i = 0; i < app_w->nr_steps; i++) |
| 534 | steps[nr_steps + i].idx += nr_steps; |
| 535 | |
| 536 | nr_steps += app_w->nr_steps; |
| 537 | } |
| 538 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 539 | wrk = malloc(sizeof(*wrk)); |
| 540 | igt_assert(wrk); |
| 541 | |
| 542 | wrk->nr_steps = nr_steps; |
| 543 | wrk->steps = steps; |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 544 | wrk->prio = arg->prio; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 545 | |
| 546 | free(desc); |
| 547 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 548 | /* |
| 549 | * Tag all steps which need to emit a sync fence if another step is |
| 550 | * referencing them as a sync fence dependency. |
| 551 | */ |
| 552 | for (i = 0; i < nr_steps; i++) { |
| 553 | for (j = 0; j < steps[i].fence_deps.nr; j++) { |
| 554 | tmp = steps[i].idx + steps[i].fence_deps.list[j]; |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 555 | if (tmp < 0 || tmp >= i || |
| 556 | (steps[tmp].type != BATCH && |
| 557 | steps[tmp].type != SW_FENCE)) { |
| 558 | if (verbose) |
| 559 | fprintf(stderr, |
| 560 | "Invalid dependency target %u!\n", |
| 561 | i); |
| 562 | return NULL; |
| 563 | } |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 564 | steps[tmp].emit_fence = -1; |
| 565 | } |
| 566 | } |
| 567 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 568 | /* Validate SW_FENCE_SIGNAL targets. */ |
| 569 | for (i = 0; i < nr_steps; i++) { |
| 570 | if (steps[i].type == SW_FENCE_SIGNAL) { |
| 571 | tmp = steps[i].idx + steps[i].target; |
| 572 | if (tmp < 0 || tmp >= i || |
| 573 | steps[tmp].type != SW_FENCE) { |
| 574 | if (verbose) |
| 575 | fprintf(stderr, |
| 576 | "Invalid sw fence target %u!\n", |
| 577 | i); |
| 578 | return NULL; |
| 579 | } |
| 580 | } |
| 581 | } |
| 582 | |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 583 | if (bcs_used && verbose) |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 584 | printf("BCS usage in workload with VCS2 remapping enabled!\n"); |
| 585 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 586 | return wrk; |
| 587 | } |
| 588 | |
| 589 | static struct workload * |
| 590 | clone_workload(struct workload *_wrk) |
| 591 | { |
| 592 | struct workload *wrk; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 593 | int i; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 594 | |
| 595 | wrk = malloc(sizeof(*wrk)); |
| 596 | igt_assert(wrk); |
| 597 | memset(wrk, 0, sizeof(*wrk)); |
| 598 | |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 599 | wrk->prio = _wrk->prio; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 600 | wrk->nr_steps = _wrk->nr_steps; |
| 601 | wrk->steps = calloc(wrk->nr_steps, sizeof(struct w_step)); |
| 602 | igt_assert(wrk->steps); |
| 603 | |
| 604 | memcpy(wrk->steps, _wrk->steps, sizeof(struct w_step) * wrk->nr_steps); |
| 605 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 606 | /* Check if we need a sw sync timeline. */ |
| 607 | for (i = 0; i < wrk->nr_steps; i++) { |
| 608 | if (wrk->steps[i].type == SW_FENCE) { |
| 609 | wrk->sync_timeline = sw_sync_timeline_create(); |
| 610 | igt_assert(wrk->sync_timeline >= 0); |
| 611 | break; |
| 612 | } |
| 613 | } |
| 614 | |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 615 | for (i = 0; i < NUM_ENGINES; i++) |
| 616 | igt_list_init(&wrk->requests[i]); |
| 617 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 618 | return wrk; |
| 619 | } |
| 620 | |
| 621 | #define rounddown(x, y) (x - (x%y)) |
| 622 | #ifndef PAGE_SIZE |
| 623 | #define PAGE_SIZE (4096) |
| 624 | #endif |
| 625 | |
Tvrtko Ursulin | 26a5e65 | 2017-05-16 08:11:29 +0100 | [diff] [blame] | 626 | static unsigned int get_duration(struct w_step *w) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 627 | { |
Tvrtko Ursulin | 26a5e65 | 2017-05-16 08:11:29 +0100 | [diff] [blame] | 628 | struct duration *dur = &w->duration; |
| 629 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 630 | if (dur->min == dur->max) |
| 631 | return dur->min; |
| 632 | else |
| 633 | return dur->min + hars_petruska_f54_1_random_unsafe() % |
| 634 | (dur->max + 1 - dur->min); |
| 635 | } |
| 636 | |
| 637 | static unsigned long get_bb_sz(unsigned int duration) |
| 638 | { |
| 639 | return ALIGN(duration * nop_calibration * sizeof(uint32_t) / |
| 640 | nop_calibration_us, sizeof(uint32_t)); |
| 641 | } |
| 642 | |
| 643 | static void |
| 644 | terminate_bb(struct w_step *w, unsigned int flags) |
| 645 | { |
| 646 | const uint32_t bbe = 0xa << 23; |
| 647 | unsigned long mmap_start, mmap_len; |
| 648 | unsigned long batch_start = w->bb_sz; |
| 649 | uint32_t *ptr, *cs; |
| 650 | |
| 651 | igt_assert(((flags & RT) && (flags & SEQNO)) || !(flags & RT)); |
| 652 | |
| 653 | batch_start -= sizeof(uint32_t); /* bbend */ |
| 654 | if (flags & SEQNO) |
| 655 | batch_start -= 4 * sizeof(uint32_t); |
| 656 | if (flags & RT) |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 657 | batch_start -= 12 * sizeof(uint32_t); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 658 | |
| 659 | mmap_start = rounddown(batch_start, PAGE_SIZE); |
| 660 | mmap_len = w->bb_sz - mmap_start; |
| 661 | |
| 662 | gem_set_domain(fd, w->bb_handle, |
| 663 | I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); |
| 664 | |
| 665 | ptr = gem_mmap__wc(fd, w->bb_handle, mmap_start, mmap_len, PROT_WRITE); |
| 666 | cs = (uint32_t *)((char *)ptr + batch_start - mmap_start); |
| 667 | |
| 668 | if (flags & SEQNO) { |
| 669 | w->reloc[0].offset = batch_start + sizeof(uint32_t); |
| 670 | batch_start += 4 * sizeof(uint32_t); |
| 671 | |
| 672 | *cs++ = MI_STORE_DWORD_IMM; |
| 673 | w->seqno_address = cs; |
| 674 | *cs++ = 0; |
| 675 | *cs++ = 0; |
| 676 | w->seqno_value = cs; |
| 677 | *cs++ = 0; |
| 678 | } |
| 679 | |
| 680 | if (flags & RT) { |
| 681 | w->reloc[1].offset = batch_start + sizeof(uint32_t); |
| 682 | batch_start += 4 * sizeof(uint32_t); |
| 683 | |
| 684 | *cs++ = MI_STORE_DWORD_IMM; |
| 685 | w->rt0_address = cs; |
| 686 | *cs++ = 0; |
| 687 | *cs++ = 0; |
| 688 | w->rt0_value = cs; |
| 689 | *cs++ = 0; |
| 690 | |
| 691 | w->reloc[2].offset = batch_start + 2 * sizeof(uint32_t); |
| 692 | batch_start += 4 * sizeof(uint32_t); |
| 693 | |
| 694 | *cs++ = 0x24 << 23 | 2; /* MI_STORE_REG_MEM */ |
| 695 | *cs++ = RCS_TIMESTAMP; |
| 696 | w->rt1_address = cs; |
| 697 | *cs++ = 0; |
| 698 | *cs++ = 0; |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 699 | |
| 700 | w->reloc[3].offset = batch_start + sizeof(uint32_t); |
| 701 | batch_start += 4 * sizeof(uint32_t); |
| 702 | |
| 703 | *cs++ = MI_STORE_DWORD_IMM; |
| 704 | w->latch_address = cs; |
| 705 | *cs++ = 0; |
| 706 | *cs++ = 0; |
| 707 | w->latch_value = cs; |
| 708 | *cs++ = 0; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 709 | } |
| 710 | |
| 711 | *cs = bbe; |
| 712 | |
| 713 | w->mapped_batch = ptr; |
| 714 | w->mapped_len = mmap_len; |
| 715 | } |
| 716 | |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 717 | static const unsigned int eb_engine_map[NUM_ENGINES] = { |
| 718 | [RCS] = I915_EXEC_RENDER, |
| 719 | [BCS] = I915_EXEC_BLT, |
| 720 | [VCS] = I915_EXEC_BSD, |
| 721 | [VCS1] = I915_EXEC_BSD | I915_EXEC_BSD_RING1, |
| 722 | [VCS2] = I915_EXEC_BSD | I915_EXEC_BSD_RING2, |
| 723 | [VECS] = I915_EXEC_VEBOX |
| 724 | }; |
| 725 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 726 | static void |
Tvrtko Ursulin | c14a260 | 2017-06-07 11:40:43 +0100 | [diff] [blame] | 727 | eb_set_engine(struct drm_i915_gem_execbuffer2 *eb, |
| 728 | enum intel_engine_id engine, |
| 729 | unsigned int flags) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 730 | { |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 731 | if (engine == VCS2 && (flags & VCS2REMAP)) |
| 732 | engine = BCS; |
| 733 | |
Tvrtko Ursulin | c14a260 | 2017-06-07 11:40:43 +0100 | [diff] [blame] | 734 | eb->flags = eb_engine_map[engine]; |
| 735 | } |
| 736 | |
| 737 | static void |
| 738 | eb_update_flags(struct w_step *w, enum intel_engine_id engine, |
| 739 | unsigned int flags) |
| 740 | { |
| 741 | eb_set_engine(&w->eb, engine, flags); |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 742 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 743 | w->eb.flags |= I915_EXEC_HANDLE_LUT; |
| 744 | w->eb.flags |= I915_EXEC_NO_RELOC; |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 745 | |
| 746 | igt_assert(w->emit_fence <= 0); |
| 747 | if (w->emit_fence) |
Arkadiusz Hiler | 200d0f5 | 2017-06-07 12:11:37 +0200 | [diff] [blame] | 748 | w->eb.flags |= LOCAL_I915_EXEC_FENCE_OUT; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 749 | } |
| 750 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 751 | static struct drm_i915_gem_exec_object2 * |
| 752 | get_status_objects(struct workload *wrk) |
| 753 | { |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 754 | if (wrk->flags & GLOBAL_BALANCE) |
| 755 | return wrk->global_wrk->status_object; |
| 756 | else |
| 757 | return wrk->status_object; |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 758 | } |
| 759 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 760 | static void |
| 761 | alloc_step_batch(struct workload *wrk, struct w_step *w, unsigned int flags) |
| 762 | { |
| 763 | enum intel_engine_id engine = w->engine; |
Tvrtko Ursulin | b087257 | 2017-05-05 18:55:23 +0100 | [diff] [blame] | 764 | unsigned int j = 0; |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 765 | unsigned int nr_obj = 3 + w->data_deps.nr; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 766 | unsigned int i; |
| 767 | |
| 768 | w->obj = calloc(nr_obj, sizeof(*w->obj)); |
| 769 | igt_assert(w->obj); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 770 | |
| 771 | w->obj[j].handle = gem_create(fd, 4096); |
| 772 | w->obj[j].flags = EXEC_OBJECT_WRITE; |
| 773 | j++; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 774 | igt_assert(j < nr_obj); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 775 | |
| 776 | if (flags & SEQNO) { |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 777 | w->obj[j++] = get_status_objects(wrk)[0]; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 778 | igt_assert(j < nr_obj); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 779 | } |
| 780 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 781 | for (i = 0; i < w->data_deps.nr; i++) { |
| 782 | igt_assert(w->data_deps.list[i] <= 0); |
| 783 | if (w->data_deps.list[i]) { |
| 784 | int dep_idx = w->idx + w->data_deps.list[i]; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 785 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 786 | igt_assert(dep_idx >= 0 && dep_idx < w->idx); |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 787 | igt_assert(wrk->steps[dep_idx].type == BATCH); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 788 | |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 789 | w->obj[j].handle = wrk->steps[dep_idx].obj[0].handle; |
| 790 | j++; |
| 791 | igt_assert(j < nr_obj); |
| 792 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 793 | } |
| 794 | |
Tvrtko Ursulin | b087257 | 2017-05-05 18:55:23 +0100 | [diff] [blame] | 795 | w->bb_sz = get_bb_sz(w->duration.max); |
| 796 | w->bb_handle = w->obj[j].handle = gem_create(fd, w->bb_sz); |
| 797 | terminate_bb(w, flags); |
| 798 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 799 | if (flags & SEQNO) { |
Tvrtko Ursulin | b087257 | 2017-05-05 18:55:23 +0100 | [diff] [blame] | 800 | w->obj[j].relocs_ptr = to_user_pointer(&w->reloc); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 801 | if (flags & RT) |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 802 | w->obj[j].relocation_count = 4; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 803 | else |
Tvrtko Ursulin | b087257 | 2017-05-05 18:55:23 +0100 | [diff] [blame] | 804 | w->obj[j].relocation_count = 1; |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 805 | for (i = 0; i < w->obj[j].relocation_count; i++) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 806 | w->reloc[i].target_handle = 1; |
| 807 | } |
| 808 | |
| 809 | w->eb.buffers_ptr = to_user_pointer(w->obj); |
Tvrtko Ursulin | b087257 | 2017-05-05 18:55:23 +0100 | [diff] [blame] | 810 | w->eb.buffer_count = j + 1; |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 811 | w->eb.rsvd1 = wrk->ctx_list[w->context].id; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 812 | |
| 813 | if (flags & SWAPVCS && engine == VCS1) |
| 814 | engine = VCS2; |
| 815 | else if (flags & SWAPVCS && engine == VCS2) |
| 816 | engine = VCS1; |
| 817 | eb_update_flags(w, engine, flags); |
| 818 | #ifdef DEBUG |
Tvrtko Ursulin | 07a8aa0 | 2017-05-08 13:31:50 +0100 | [diff] [blame] | 819 | printf("%u: %u:|", w->idx, w->eb.buffer_count); |
| 820 | for (i = 0; i <= j; i++) |
| 821 | printf("%x|", w->obj[i].handle); |
| 822 | printf(" %10lu flags=%llx bb=%x[%u] ctx[%u]=%u\n", |
| 823 | w->bb_sz, w->eb.flags, w->bb_handle, j, w->context, |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 824 | wrk->ctx_list[w->context].id); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 825 | #endif |
| 826 | } |
| 827 | |
| 828 | static void |
Tvrtko Ursulin | 7736d7e | 2017-05-09 09:21:03 +0100 | [diff] [blame] | 829 | prepare_workload(unsigned int id, struct workload *wrk, unsigned int flags) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 830 | { |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 831 | unsigned int ctx_vcs = 0; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 832 | int max_ctx = -1; |
| 833 | struct w_step *w; |
| 834 | int i; |
| 835 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 836 | wrk->id = id; |
Chris Wilson | 62a1f54 | 2017-05-09 12:42:41 +0100 | [diff] [blame] | 837 | wrk->prng = rand(); |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 838 | wrk->run = true; |
Chris Wilson | 62a1f54 | 2017-05-09 12:42:41 +0100 | [diff] [blame] | 839 | |
Tvrtko Ursulin | 7736d7e | 2017-05-09 09:21:03 +0100 | [diff] [blame] | 840 | if (flags & INITVCSRR) |
| 841 | wrk->vcs_rr = id & 1; |
| 842 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 843 | if (flags & GLOBAL_BALANCE) { |
| 844 | int ret = pthread_mutex_init(&wrk->mutex, NULL); |
| 845 | igt_assert(ret == 0); |
| 846 | } |
| 847 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 848 | if (flags & SEQNO) { |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 849 | if (!(flags & GLOBAL_BALANCE) || id == 0) { |
| 850 | uint32_t handle; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 851 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 852 | handle = gem_create(fd, 4096); |
| 853 | gem_set_caching(fd, handle, I915_CACHING_CACHED); |
| 854 | wrk->status_object[0].handle = handle; |
| 855 | wrk->status_page = gem_mmap__cpu(fd, handle, 0, 4096, |
| 856 | PROT_READ); |
Chris Wilson | 36dec3d | 2017-05-10 10:57:16 +0100 | [diff] [blame] | 857 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 858 | handle = gem_create(fd, 4096); |
| 859 | wrk->status_object[1].handle = handle; |
| 860 | wrk->status_cs = gem_mmap__wc(fd, handle, |
| 861 | 0, 4096, PROT_WRITE); |
| 862 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 863 | } |
| 864 | |
| 865 | for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) { |
| 866 | if ((int)w->context > max_ctx) { |
| 867 | int delta = w->context + 1 - wrk->nr_ctxs; |
| 868 | |
| 869 | wrk->nr_ctxs += delta; |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 870 | wrk->ctx_list = realloc(wrk->ctx_list, |
| 871 | wrk->nr_ctxs * |
| 872 | sizeof(*wrk->ctx_list)); |
| 873 | memset(&wrk->ctx_list[wrk->nr_ctxs - delta], 0, |
| 874 | delta * sizeof(*wrk->ctx_list)); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 875 | |
| 876 | max_ctx = w->context; |
| 877 | } |
| 878 | |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 879 | if (!wrk->ctx_list[w->context].id) { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 880 | struct drm_i915_gem_context_create arg = {}; |
| 881 | |
| 882 | drmIoctl(fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &arg); |
| 883 | igt_assert(arg.ctx_id); |
| 884 | |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 885 | wrk->ctx_list[w->context].id = arg.ctx_id; |
| 886 | |
| 887 | if (flags & GLOBAL_BALANCE) { |
| 888 | wrk->ctx_list[w->context].static_vcs = context_vcs_rr; |
| 889 | context_vcs_rr ^= 1; |
| 890 | } else { |
| 891 | wrk->ctx_list[w->context].static_vcs = ctx_vcs; |
| 892 | ctx_vcs ^= 1; |
| 893 | } |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 894 | |
| 895 | if (wrk->prio) { |
| 896 | struct local_i915_gem_context_param param = { |
| 897 | .context = arg.ctx_id, |
| 898 | .param = 0x6, |
| 899 | .value = wrk->prio, |
| 900 | }; |
| 901 | gem_context_set_param(fd, ¶m); |
| 902 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 903 | } |
| 904 | } |
| 905 | |
| 906 | for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) { |
| 907 | unsigned int _flags = flags; |
| 908 | enum intel_engine_id engine = w->engine; |
| 909 | |
| 910 | if (w->type != BATCH) |
| 911 | continue; |
| 912 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 913 | if (engine == VCS) |
| 914 | _flags &= ~SWAPVCS; |
| 915 | |
| 916 | alloc_step_batch(wrk, w, _flags); |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | static double elapsed(const struct timespec *start, const struct timespec *end) |
| 921 | { |
| 922 | return (end->tv_sec - start->tv_sec) + |
| 923 | (end->tv_nsec - start->tv_nsec) / 1e9; |
| 924 | } |
| 925 | |
| 926 | static int elapsed_us(const struct timespec *start, const struct timespec *end) |
| 927 | { |
| 928 | return elapsed(start, end) * 1e6; |
| 929 | } |
| 930 | |
| 931 | static enum intel_engine_id get_vcs_engine(unsigned int n) |
| 932 | { |
| 933 | const enum intel_engine_id vcs_engines[2] = { VCS1, VCS2 }; |
| 934 | |
| 935 | igt_assert(n < ARRAY_SIZE(vcs_engines)); |
| 936 | |
| 937 | return vcs_engines[n]; |
| 938 | } |
| 939 | |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 940 | static uint32_t new_seqno(struct workload *wrk, enum intel_engine_id engine) |
| 941 | { |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 942 | uint32_t seqno; |
| 943 | int ret; |
| 944 | |
| 945 | if (wrk->flags & GLOBAL_BALANCE) { |
| 946 | igt_assert(wrk->global_wrk); |
| 947 | wrk = wrk->global_wrk; |
| 948 | |
| 949 | ret = pthread_mutex_lock(&wrk->mutex); |
| 950 | igt_assert(ret == 0); |
| 951 | } |
| 952 | |
| 953 | seqno = ++wrk->seqno[engine]; |
| 954 | |
| 955 | if (wrk->flags & GLOBAL_BALANCE) { |
| 956 | ret = pthread_mutex_unlock(&wrk->mutex); |
| 957 | igt_assert(ret == 0); |
| 958 | } |
| 959 | |
| 960 | return seqno; |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 961 | } |
| 962 | |
| 963 | static uint32_t |
| 964 | current_seqno(struct workload *wrk, enum intel_engine_id engine) |
| 965 | { |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 966 | if (wrk->flags & GLOBAL_BALANCE) |
| 967 | return wrk->global_wrk->seqno[engine]; |
| 968 | else |
| 969 | return wrk->seqno[engine]; |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 970 | } |
| 971 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 972 | #define READ_ONCE(x) (*(volatile typeof(x) *)(&(x))) |
| 973 | |
| 974 | static uint32_t |
| 975 | read_status_page(struct workload *wrk, unsigned int idx) |
| 976 | { |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 977 | if (wrk->flags & GLOBAL_BALANCE) |
| 978 | return READ_ONCE(wrk->global_wrk->status_page[idx]); |
| 979 | else |
| 980 | return READ_ONCE(wrk->status_page[idx]); |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 981 | } |
| 982 | |
| 983 | static uint32_t |
| 984 | current_gpu_seqno(struct workload *wrk, enum intel_engine_id engine) |
| 985 | { |
| 986 | return read_status_page(wrk, SEQNO_IDX(engine)); |
| 987 | } |
| 988 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 989 | struct workload_balancer { |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 990 | unsigned int id; |
| 991 | const char *name; |
| 992 | const char *desc; |
| 993 | unsigned int flags; |
| 994 | unsigned int min_gen; |
| 995 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 996 | unsigned int (*get_qd)(const struct workload_balancer *balancer, |
| 997 | struct workload *wrk, |
| 998 | enum intel_engine_id engine); |
| 999 | enum intel_engine_id (*balance)(const struct workload_balancer *balancer, |
| 1000 | struct workload *wrk, struct w_step *w); |
| 1001 | }; |
| 1002 | |
| 1003 | static enum intel_engine_id |
| 1004 | rr_balance(const struct workload_balancer *balancer, |
| 1005 | struct workload *wrk, struct w_step *w) |
| 1006 | { |
| 1007 | unsigned int engine; |
| 1008 | |
| 1009 | engine = get_vcs_engine(wrk->vcs_rr); |
| 1010 | wrk->vcs_rr ^= 1; |
| 1011 | |
| 1012 | return engine; |
| 1013 | } |
| 1014 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1015 | static enum intel_engine_id |
| 1016 | rand_balance(const struct workload_balancer *balancer, |
| 1017 | struct workload *wrk, struct w_step *w) |
| 1018 | { |
| 1019 | return get_vcs_engine(hars_petruska_f54_1_random(&wrk->prng) & 1); |
| 1020 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1021 | |
| 1022 | static unsigned int |
| 1023 | get_qd_depth(const struct workload_balancer *balancer, |
| 1024 | struct workload *wrk, enum intel_engine_id engine) |
| 1025 | { |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1026 | return current_seqno(wrk, engine) - current_gpu_seqno(wrk, engine); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1027 | } |
| 1028 | |
| 1029 | static enum intel_engine_id |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1030 | __qd_select_engine(struct workload *wrk, const unsigned long *qd, bool random) |
| 1031 | { |
| 1032 | unsigned int n; |
| 1033 | |
| 1034 | if (qd[VCS1] < qd[VCS2]) |
| 1035 | n = 0; |
| 1036 | else if (qd[VCS1] > qd[VCS2]) |
| 1037 | n = 1; |
| 1038 | else if (random) |
| 1039 | n = hars_petruska_f54_1_random(&wrk->prng) & 1; |
| 1040 | else |
| 1041 | n = wrk->vcs_rr; |
| 1042 | wrk->vcs_rr = n ^ 1; |
| 1043 | |
| 1044 | return get_vcs_engine(n); |
| 1045 | } |
| 1046 | |
| 1047 | static enum intel_engine_id |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1048 | __qd_balance(const struct workload_balancer *balancer, |
| 1049 | struct workload *wrk, struct w_step *w, bool random) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1050 | { |
| 1051 | enum intel_engine_id engine; |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1052 | unsigned long qd[NUM_ENGINES]; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1053 | |
| 1054 | igt_assert(w->engine == VCS); |
| 1055 | |
| 1056 | qd[VCS1] = balancer->get_qd(balancer, wrk, VCS1); |
| 1057 | wrk->qd_sum[VCS1] += qd[VCS1]; |
| 1058 | |
| 1059 | qd[VCS2] = balancer->get_qd(balancer, wrk, VCS2); |
| 1060 | wrk->qd_sum[VCS2] += qd[VCS2]; |
| 1061 | |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1062 | engine = __qd_select_engine(wrk, qd, random); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1063 | |
| 1064 | #ifdef DEBUG |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 1065 | printf("qd_balance[%u]: 1:%ld 2:%ld rr:%u = %u\t(%u - %u) (%u - %u)\n", |
| 1066 | wrk->id, qd[VCS1], qd[VCS2], wrk->vcs_rr, engine, |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1067 | current_seqno(wrk, VCS1), current_gpu_seqno(wrk, VCS1), |
| 1068 | current_seqno(wrk, VCS2), current_gpu_seqno(wrk, VCS2)); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1069 | #endif |
| 1070 | return engine; |
| 1071 | } |
| 1072 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1073 | static enum intel_engine_id |
| 1074 | qd_balance(const struct workload_balancer *balancer, |
| 1075 | struct workload *wrk, struct w_step *w) |
| 1076 | { |
| 1077 | return __qd_balance(balancer, wrk, w, false); |
| 1078 | } |
| 1079 | |
| 1080 | static enum intel_engine_id |
| 1081 | qdr_balance(const struct workload_balancer *balancer, |
| 1082 | struct workload *wrk, struct w_step *w) |
| 1083 | { |
| 1084 | return __qd_balance(balancer, wrk, w, true); |
| 1085 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1086 | |
| 1087 | static enum intel_engine_id |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1088 | qdavg_balance(const struct workload_balancer *balancer, |
| 1089 | struct workload *wrk, struct w_step *w) |
| 1090 | { |
| 1091 | unsigned long qd[NUM_ENGINES]; |
| 1092 | unsigned int engine; |
| 1093 | |
| 1094 | igt_assert(w->engine == VCS); |
| 1095 | |
| 1096 | for (engine = VCS1; engine <= VCS2; engine++) { |
| 1097 | qd[engine] = balancer->get_qd(balancer, wrk, engine); |
| 1098 | wrk->qd_sum[engine] += qd[engine]; |
| 1099 | |
| 1100 | ewma_rt_add(&wrk->rt.avg[engine], qd[engine]); |
| 1101 | qd[engine] = ewma_rt_read(&wrk->rt.avg[engine]); |
| 1102 | } |
| 1103 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 1104 | engine = __qd_select_engine(wrk, qd, false); |
| 1105 | #ifdef DEBUG |
| 1106 | printf("qdavg_balance[%u]: 1:%ld 2:%ld rr:%u = %u\t(%u - %u) (%u - %u)\n", |
| 1107 | wrk->id, qd[VCS1], qd[VCS2], wrk->vcs_rr, engine, |
| 1108 | current_seqno(wrk, VCS1), current_gpu_seqno(wrk, VCS1), |
| 1109 | current_seqno(wrk, VCS2), current_gpu_seqno(wrk, VCS2)); |
| 1110 | #endif |
| 1111 | return engine; |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1112 | } |
| 1113 | |
| 1114 | static enum intel_engine_id |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1115 | __rt_select_engine(struct workload *wrk, unsigned long *qd, bool random) |
| 1116 | { |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1117 | qd[VCS1] >>= 10; |
| 1118 | qd[VCS2] >>= 10; |
| 1119 | |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1120 | return __qd_select_engine(wrk, qd, random); |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1121 | } |
| 1122 | |
Chris Wilson | cb16119 | 2017-05-09 17:31:27 +0100 | [diff] [blame] | 1123 | struct rt_depth { |
| 1124 | uint32_t seqno; |
| 1125 | uint32_t submitted; |
| 1126 | uint32_t completed; |
| 1127 | }; |
| 1128 | |
Chris Wilson | cb16119 | 2017-05-09 17:31:27 +0100 | [diff] [blame] | 1129 | static void get_rt_depth(struct workload *wrk, |
| 1130 | unsigned int engine, |
| 1131 | struct rt_depth *rt) |
| 1132 | { |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1133 | const unsigned int idx = SEQNO_IDX(engine); |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1134 | uint32_t latch; |
Chris Wilson | cb16119 | 2017-05-09 17:31:27 +0100 | [diff] [blame] | 1135 | |
Chris Wilson | cb16119 | 2017-05-09 17:31:27 +0100 | [diff] [blame] | 1136 | do { |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1137 | latch = read_status_page(wrk, idx + 3); |
| 1138 | rt->submitted = read_status_page(wrk, idx + 1); |
| 1139 | rt->completed = read_status_page(wrk, idx + 2); |
| 1140 | rt->seqno = read_status_page(wrk, idx); |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1141 | } while (latch != rt->seqno); |
Chris Wilson | cb16119 | 2017-05-09 17:31:27 +0100 | [diff] [blame] | 1142 | } |
| 1143 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1144 | static enum intel_engine_id |
Tvrtko Ursulin | 8111653 | 2017-05-08 18:37:03 +0100 | [diff] [blame] | 1145 | __rt_balance(const struct workload_balancer *balancer, |
| 1146 | struct workload *wrk, struct w_step *w, bool random) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1147 | { |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1148 | unsigned long qd[NUM_ENGINES]; |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1149 | unsigned int engine; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1150 | |
| 1151 | igt_assert(w->engine == VCS); |
| 1152 | |
| 1153 | /* Estimate the "speed" of the most recent batch |
| 1154 | * (finish time - submit time) |
| 1155 | * and use that as an approximate for the total remaining time for |
Chris Wilson | 6e6ad40 | 2017-05-09 12:43:13 +0100 | [diff] [blame] | 1156 | * all batches on that engine, plus the time we expect this batch to |
| 1157 | * take. We try to keep the total balanced between the engines. |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1158 | */ |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1159 | for (engine = VCS1; engine <= VCS2; engine++) { |
| 1160 | struct rt_depth rt; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1161 | |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1162 | get_rt_depth(wrk, engine, &rt); |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1163 | qd[engine] = current_seqno(wrk, engine) - rt.seqno; |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1164 | wrk->qd_sum[engine] += qd[engine]; |
| 1165 | qd[engine] = (qd[engine] + 1) * (rt.completed - rt.submitted); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1166 | #ifdef DEBUG |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1167 | printf("rt[0] = %d (%d - %d) x %d (%d - %d) = %ld\n", |
| 1168 | current_seqno(wrk, engine) - rt.seqno, |
| 1169 | current_seqno(wrk, engine), rt.seqno, |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1170 | rt.completed - rt.submitted, |
| 1171 | rt.completed, rt.submitted, |
| 1172 | qd[engine]); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1173 | #endif |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1174 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1175 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1176 | return __rt_select_engine(wrk, qd, random); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1177 | } |
| 1178 | |
Tvrtko Ursulin | 8111653 | 2017-05-08 18:37:03 +0100 | [diff] [blame] | 1179 | static enum intel_engine_id |
| 1180 | rt_balance(const struct workload_balancer *balancer, |
| 1181 | struct workload *wrk, struct w_step *w) |
| 1182 | { |
| 1183 | |
| 1184 | return __rt_balance(balancer, wrk, w, false); |
| 1185 | } |
| 1186 | |
Tvrtko Ursulin | 8111653 | 2017-05-08 18:37:03 +0100 | [diff] [blame] | 1187 | static enum intel_engine_id |
| 1188 | rtr_balance(const struct workload_balancer *balancer, |
| 1189 | struct workload *wrk, struct w_step *w) |
| 1190 | { |
Tvrtko Ursulin | 8111653 | 2017-05-08 18:37:03 +0100 | [diff] [blame] | 1191 | return __rt_balance(balancer, wrk, w, true); |
| 1192 | } |
| 1193 | |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1194 | static enum intel_engine_id |
| 1195 | rtavg_balance(const struct workload_balancer *balancer, |
| 1196 | struct workload *wrk, struct w_step *w) |
| 1197 | { |
| 1198 | unsigned long qd[NUM_ENGINES]; |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1199 | unsigned int engine; |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1200 | |
| 1201 | igt_assert(w->engine == VCS); |
| 1202 | |
| 1203 | /* Estimate the average "speed" of the most recent batches |
| 1204 | * (finish time - submit time) |
| 1205 | * and use that as an approximate for the total remaining time for |
| 1206 | * all batches on that engine plus the time we expect to execute in. |
| 1207 | * We try to keep the total remaining balanced between the engines. |
| 1208 | */ |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1209 | for (engine = VCS1; engine <= VCS2; engine++) { |
| 1210 | struct rt_depth rt; |
| 1211 | |
| 1212 | get_rt_depth(wrk, engine, &rt); |
| 1213 | if (rt.seqno != wrk->rt.last[engine]) { |
| 1214 | igt_assert((long)(rt.completed - rt.submitted) > 0); |
| 1215 | ewma_rt_add(&wrk->rt.avg[engine], |
| 1216 | rt.completed - rt.submitted); |
| 1217 | wrk->rt.last[engine] = rt.seqno; |
| 1218 | } |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1219 | qd[engine] = current_seqno(wrk, engine) - rt.seqno; |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1220 | wrk->qd_sum[engine] += qd[engine]; |
| 1221 | qd[engine] = |
| 1222 | (qd[engine] + 1) * ewma_rt_read(&wrk->rt.avg[engine]); |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1223 | |
| 1224 | #ifdef DEBUG |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1225 | printf("rtavg[%d] = %d (%d - %d) x %ld (%d) = %ld\n", |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1226 | engine, |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1227 | current_seqno(wrk, engine) - rt.seqno, |
| 1228 | current_seqno(wrk, engine), rt.seqno, |
Chris Wilson | 43f6fce | 2017-05-10 13:07:43 +0100 | [diff] [blame] | 1229 | ewma_rt_read(&wrk->rt.avg[engine]), |
| 1230 | rt.completed - rt.submitted, |
| 1231 | qd[engine]); |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1232 | #endif |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1233 | } |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1234 | |
| 1235 | return __rt_select_engine(wrk, qd, false); |
| 1236 | } |
| 1237 | |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 1238 | static enum intel_engine_id |
| 1239 | context_balance(const struct workload_balancer *balancer, |
| 1240 | struct workload *wrk, struct w_step *w) |
| 1241 | { |
| 1242 | return get_vcs_engine(wrk->ctx_list[w->context].static_vcs); |
| 1243 | } |
| 1244 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1245 | static const struct workload_balancer all_balancers[] = { |
| 1246 | { |
| 1247 | .id = 0, |
| 1248 | .name = "rr", |
| 1249 | .desc = "Simple round-robin.", |
| 1250 | .balance = rr_balance, |
| 1251 | }, |
| 1252 | { |
| 1253 | .id = 6, |
| 1254 | .name = "rand", |
| 1255 | .desc = "Random selection.", |
| 1256 | .balance = rand_balance, |
| 1257 | }, |
| 1258 | { |
| 1259 | .id = 1, |
| 1260 | .name = "qd", |
| 1261 | .desc = "Queue depth estimation with round-robin on equal depth.", |
| 1262 | .flags = SEQNO, |
| 1263 | .min_gen = 8, |
| 1264 | .get_qd = get_qd_depth, |
| 1265 | .balance = qd_balance, |
| 1266 | }, |
| 1267 | { |
| 1268 | .id = 5, |
| 1269 | .name = "qdr", |
| 1270 | .desc = "Queue depth estimation with random selection on equal depth.", |
| 1271 | .flags = SEQNO, |
| 1272 | .min_gen = 8, |
| 1273 | .get_qd = get_qd_depth, |
| 1274 | .balance = qdr_balance, |
| 1275 | }, |
| 1276 | { |
Chris Wilson | 04e38c3 | 2017-05-10 12:52:58 +0100 | [diff] [blame] | 1277 | .id = 7, |
| 1278 | .name = "qdavg", |
| 1279 | .desc = "Like qd, but using an average queue depth estimator.", |
| 1280 | .flags = SEQNO, |
| 1281 | .min_gen = 8, |
| 1282 | .get_qd = get_qd_depth, |
| 1283 | .balance = qdavg_balance, |
| 1284 | }, |
| 1285 | { |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1286 | .id = 2, |
| 1287 | .name = "rt", |
| 1288 | .desc = "Queue depth plus last runtime estimation.", |
| 1289 | .flags = SEQNO | RT, |
| 1290 | .min_gen = 8, |
| 1291 | .get_qd = get_qd_depth, |
| 1292 | .balance = rt_balance, |
| 1293 | }, |
| 1294 | { |
| 1295 | .id = 3, |
| 1296 | .name = "rtr", |
| 1297 | .desc = "Like rt but with random engine selection on equal depth.", |
| 1298 | .flags = SEQNO | RT, |
| 1299 | .min_gen = 8, |
| 1300 | .get_qd = get_qd_depth, |
| 1301 | .balance = rtr_balance, |
| 1302 | }, |
| 1303 | { |
| 1304 | .id = 4, |
| 1305 | .name = "rtavg", |
| 1306 | .desc = "Improved version rt tracking average execution speed per engine.", |
| 1307 | .flags = SEQNO | RT, |
| 1308 | .min_gen = 8, |
| 1309 | .get_qd = get_qd_depth, |
| 1310 | .balance = rtavg_balance, |
| 1311 | }, |
Tvrtko Ursulin | ef74c05 | 2017-06-05 08:58:19 +0100 | [diff] [blame] | 1312 | { |
| 1313 | .id = 8, |
| 1314 | .name = "context", |
| 1315 | .desc = "Static round-robin VCS assignment at context creation.", |
| 1316 | .balance = context_balance, |
| 1317 | }, |
Chris Wilson | 9e55cca | 2017-04-25 15:12:50 +0100 | [diff] [blame] | 1318 | }; |
| 1319 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 1320 | static unsigned int |
| 1321 | global_get_qd(const struct workload_balancer *balancer, |
| 1322 | struct workload *wrk, enum intel_engine_id engine) |
| 1323 | { |
| 1324 | igt_assert(wrk->global_wrk); |
| 1325 | igt_assert(wrk->global_balancer); |
| 1326 | |
| 1327 | return wrk->global_balancer->get_qd(wrk->global_balancer, |
| 1328 | wrk->global_wrk, engine); |
| 1329 | } |
| 1330 | |
| 1331 | static enum intel_engine_id |
| 1332 | global_balance(const struct workload_balancer *balancer, |
| 1333 | struct workload *wrk, struct w_step *w) |
| 1334 | { |
| 1335 | enum intel_engine_id engine; |
| 1336 | int ret; |
| 1337 | |
| 1338 | igt_assert(wrk->global_wrk); |
| 1339 | igt_assert(wrk->global_balancer); |
| 1340 | |
| 1341 | wrk = wrk->global_wrk; |
| 1342 | |
| 1343 | ret = pthread_mutex_lock(&wrk->mutex); |
| 1344 | igt_assert(ret == 0); |
| 1345 | |
| 1346 | engine = wrk->global_balancer->balance(wrk->global_balancer, wrk, w); |
| 1347 | |
| 1348 | ret = pthread_mutex_unlock(&wrk->mutex); |
| 1349 | igt_assert(ret == 0); |
| 1350 | |
| 1351 | return engine; |
| 1352 | } |
| 1353 | |
| 1354 | static const struct workload_balancer global_balancer = { |
| 1355 | .id = ~0, |
| 1356 | .name = "global", |
| 1357 | .desc = "Global balancer", |
| 1358 | .get_qd = global_get_qd, |
| 1359 | .balance = global_balance, |
| 1360 | }; |
| 1361 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1362 | static void |
| 1363 | update_bb_seqno(struct w_step *w, enum intel_engine_id engine, uint32_t seqno) |
| 1364 | { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1365 | gem_set_domain(fd, w->bb_handle, |
| 1366 | I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); |
| 1367 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1368 | w->reloc[0].delta = SEQNO_OFFSET(engine); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1369 | |
| 1370 | *w->seqno_value = seqno; |
| 1371 | *w->seqno_address = w->reloc[0].presumed_offset + w->reloc[0].delta; |
| 1372 | |
| 1373 | /* If not using NO_RELOC, force the relocations */ |
| 1374 | if (!(w->eb.flags & I915_EXEC_NO_RELOC)) |
| 1375 | w->reloc[0].presumed_offset = -1; |
| 1376 | } |
| 1377 | |
| 1378 | static void |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1379 | update_bb_rt(struct w_step *w, enum intel_engine_id engine, uint32_t seqno) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1380 | { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1381 | gem_set_domain(fd, w->bb_handle, |
| 1382 | I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); |
| 1383 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1384 | w->reloc[1].delta = SEQNO_OFFSET(engine) + sizeof(uint32_t); |
| 1385 | w->reloc[2].delta = SEQNO_OFFSET(engine) + 2 * sizeof(uint32_t); |
| 1386 | w->reloc[3].delta = SEQNO_OFFSET(engine) + 3 * sizeof(uint32_t); |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1387 | |
| 1388 | *w->latch_value = seqno; |
| 1389 | *w->latch_address = w->reloc[3].presumed_offset + w->reloc[3].delta; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1390 | |
| 1391 | *w->rt0_value = *REG(RCS_TIMESTAMP); |
| 1392 | *w->rt0_address = w->reloc[1].presumed_offset + w->reloc[1].delta; |
Tvrtko Ursulin | 1890998 | 2017-04-25 15:46:46 +0100 | [diff] [blame] | 1393 | *w->rt1_address = w->reloc[2].presumed_offset + w->reloc[2].delta; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1394 | |
| 1395 | /* If not using NO_RELOC, force the relocations */ |
| 1396 | if (!(w->eb.flags & I915_EXEC_NO_RELOC)) { |
| 1397 | w->reloc[1].presumed_offset = -1; |
| 1398 | w->reloc[2].presumed_offset = -1; |
Tvrtko Ursulin | feaf779 | 2017-05-11 13:00:51 +0100 | [diff] [blame] | 1399 | w->reloc[3].presumed_offset = -1; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1400 | } |
| 1401 | } |
| 1402 | |
| 1403 | static void w_sync_to(struct workload *wrk, struct w_step *w, int target) |
| 1404 | { |
| 1405 | if (target < 0) |
| 1406 | target = wrk->nr_steps + target; |
| 1407 | |
| 1408 | igt_assert(target < wrk->nr_steps); |
| 1409 | |
| 1410 | while (wrk->steps[target].type != BATCH) { |
| 1411 | if (--target < 0) |
| 1412 | target = wrk->nr_steps + target; |
| 1413 | } |
| 1414 | |
| 1415 | igt_assert(target < wrk->nr_steps); |
| 1416 | igt_assert(wrk->steps[target].type == BATCH); |
| 1417 | |
| 1418 | gem_sync(fd, wrk->steps[target].obj[0].handle); |
| 1419 | } |
| 1420 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1421 | static uint32_t *get_status_cs(struct workload *wrk) |
| 1422 | { |
| 1423 | return wrk->status_cs; |
| 1424 | } |
| 1425 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1426 | #define INIT_CLOCKS 0x1 |
| 1427 | #define INIT_ALL (INIT_CLOCKS) |
| 1428 | static void init_status_page(struct workload *wrk, unsigned int flags) |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1429 | { |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1430 | struct drm_i915_gem_relocation_entry reloc[4] = {}; |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1431 | struct drm_i915_gem_exec_object2 *status_object = |
| 1432 | get_status_objects(wrk); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1433 | struct drm_i915_gem_execbuffer2 eb = { |
Chris Wilson | 36dec3d | 2017-05-10 10:57:16 +0100 | [diff] [blame] | 1434 | .buffer_count = ARRAY_SIZE(wrk->status_object), |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1435 | .buffers_ptr = to_user_pointer(status_object) |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1436 | }; |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1437 | uint32_t *base = get_status_cs(wrk); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1438 | |
| 1439 | /* Want to make sure that the balancer has a reasonable view of |
| 1440 | * the background busyness of each engine. To do that we occasionally |
| 1441 | * send a dummy batch down the pipeline. |
| 1442 | */ |
| 1443 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1444 | if (!base) |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1445 | return; |
| 1446 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1447 | gem_set_domain(fd, status_object[1].handle, |
Chris Wilson | 5a6b975 | 2017-05-09 21:59:58 +0100 | [diff] [blame] | 1448 | I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); |
Chris Wilson | 5a6b975 | 2017-05-09 21:59:58 +0100 | [diff] [blame] | 1449 | |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1450 | status_object[1].relocs_ptr = to_user_pointer(reloc); |
| 1451 | status_object[1].relocation_count = 2; |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1452 | if (flags & INIT_CLOCKS) |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1453 | status_object[1].relocation_count += 2; |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1454 | |
Chris Wilson | 22f22b4 | 2017-05-11 16:29:25 +0100 | [diff] [blame] | 1455 | for (int engine = 0; engine < NUM_ENGINES; engine++) { |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1456 | struct drm_i915_gem_relocation_entry *r = reloc; |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1457 | uint64_t presumed_offset = status_object[0].offset; |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1458 | uint32_t offset = engine * 128; |
| 1459 | uint32_t *cs = base + offset / sizeof(*cs); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1460 | uint64_t addr; |
| 1461 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1462 | r->offset = offset + sizeof(uint32_t); |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1463 | r->delta = SEQNO_OFFSET(engine); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1464 | r->presumed_offset = presumed_offset; |
| 1465 | addr = presumed_offset + r->delta; |
| 1466 | r++; |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1467 | *cs++ = MI_STORE_DWORD_IMM; |
| 1468 | *cs++ = addr; |
| 1469 | *cs++ = addr >> 32; |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1470 | *cs++ = new_seqno(wrk, engine); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1471 | offset += 4 * sizeof(uint32_t); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1472 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1473 | /* When we are busy, we can just reuse the last set of timings. |
| 1474 | * If we have been idle for a while, we want to resample the |
| 1475 | * latency on each engine (to measure external load). |
| 1476 | */ |
| 1477 | if (flags & INIT_CLOCKS) { |
| 1478 | r->offset = offset + sizeof(uint32_t); |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1479 | r->delta = SEQNO_OFFSET(engine) + sizeof(uint32_t); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1480 | r->presumed_offset = presumed_offset; |
| 1481 | addr = presumed_offset + r->delta; |
| 1482 | r++; |
| 1483 | *cs++ = MI_STORE_DWORD_IMM; |
| 1484 | *cs++ = addr; |
| 1485 | *cs++ = addr >> 32; |
| 1486 | *cs++ = *REG(RCS_TIMESTAMP); |
| 1487 | offset += 4 * sizeof(uint32_t); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1488 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1489 | r->offset = offset + 2 * sizeof(uint32_t); |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1490 | r->delta = SEQNO_OFFSET(engine) + 2*sizeof(uint32_t); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1491 | r->presumed_offset = presumed_offset; |
| 1492 | addr = presumed_offset + r->delta; |
| 1493 | r++; |
| 1494 | *cs++ = 0x24 << 23 | 2; /* MI_STORE_REG_MEM */ |
| 1495 | *cs++ = RCS_TIMESTAMP; |
| 1496 | *cs++ = addr; |
| 1497 | *cs++ = addr >> 32; |
| 1498 | offset += 4 * sizeof(uint32_t); |
| 1499 | } |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1500 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1501 | r->offset = offset + sizeof(uint32_t); |
Tvrtko Ursulin | 01959de | 2017-05-19 15:42:35 +0100 | [diff] [blame] | 1502 | r->delta = SEQNO_OFFSET(engine) + 3*sizeof(uint32_t); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1503 | r->presumed_offset = presumed_offset; |
| 1504 | addr = presumed_offset + r->delta; |
| 1505 | r++; |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1506 | *cs++ = MI_STORE_DWORD_IMM; |
| 1507 | *cs++ = addr; |
| 1508 | *cs++ = addr >> 32; |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1509 | *cs++ = current_seqno(wrk, engine); |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1510 | offset += 4 * sizeof(uint32_t); |
Chris Wilson | 02b0f8c | 2017-05-09 21:26:46 +0100 | [diff] [blame] | 1511 | |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1512 | *cs++ = MI_BATCH_BUFFER_END; |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1513 | |
Tvrtko Ursulin | c14a260 | 2017-06-07 11:40:43 +0100 | [diff] [blame] | 1514 | eb_set_engine(&eb, engine, wrk->flags); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1515 | eb.flags |= I915_EXEC_HANDLE_LUT; |
| 1516 | eb.flags |= I915_EXEC_NO_RELOC; |
| 1517 | |
Chris Wilson | 5a6b975 | 2017-05-09 21:59:58 +0100 | [diff] [blame] | 1518 | eb.batch_start_offset = 128 * engine; |
| 1519 | |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1520 | gem_execbuf(fd, &eb); |
Chris Wilson | 7d1362a | 2017-05-09 13:41:01 +0100 | [diff] [blame] | 1521 | } |
| 1522 | } |
| 1523 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1524 | static void |
| 1525 | do_eb(struct workload *wrk, struct w_step *w, enum intel_engine_id engine, |
| 1526 | unsigned int flags) |
| 1527 | { |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1528 | uint32_t seqno = new_seqno(wrk, engine); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1529 | unsigned int i; |
| 1530 | |
| 1531 | eb_update_flags(w, engine, flags); |
| 1532 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1533 | if (flags & SEQNO) |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1534 | update_bb_seqno(w, engine, seqno); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1535 | if (flags & RT) |
Tvrtko Ursulin | e329adc | 2017-05-19 15:33:23 +0100 | [diff] [blame] | 1536 | update_bb_rt(w, engine, seqno); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1537 | |
| 1538 | w->eb.batch_start_offset = |
| 1539 | ALIGN(w->bb_sz - get_bb_sz(get_duration(w)), |
| 1540 | 2 * sizeof(uint32_t)); |
| 1541 | |
| 1542 | for (i = 0; i < w->fence_deps.nr; i++) { |
| 1543 | int tgt = w->idx + w->fence_deps.list[i]; |
| 1544 | |
| 1545 | /* TODO: fence merging needed to support multiple inputs */ |
| 1546 | igt_assert(i == 0); |
| 1547 | igt_assert(tgt >= 0 && tgt < w->idx); |
| 1548 | igt_assert(wrk->steps[tgt].emit_fence > 0); |
| 1549 | |
Arkadiusz Hiler | 200d0f5 | 2017-06-07 12:11:37 +0200 | [diff] [blame] | 1550 | w->eb.flags |= LOCAL_I915_EXEC_FENCE_IN; |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1551 | w->eb.rsvd2 = wrk->steps[tgt].emit_fence; |
| 1552 | } |
| 1553 | |
Arkadiusz Hiler | 200d0f5 | 2017-06-07 12:11:37 +0200 | [diff] [blame] | 1554 | if (w->eb.flags & LOCAL_I915_EXEC_FENCE_OUT) |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1555 | gem_execbuf_wr(fd, &w->eb); |
| 1556 | else |
| 1557 | gem_execbuf(fd, &w->eb); |
| 1558 | |
Arkadiusz Hiler | 200d0f5 | 2017-06-07 12:11:37 +0200 | [diff] [blame] | 1559 | if (w->eb.flags & LOCAL_I915_EXEC_FENCE_OUT) { |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1560 | w->emit_fence = w->eb.rsvd2 >> 32; |
| 1561 | igt_assert(w->emit_fence > 0); |
| 1562 | } |
| 1563 | } |
| 1564 | |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1565 | static bool sync_deps(struct workload *wrk, struct w_step *w) |
| 1566 | { |
| 1567 | bool synced = false; |
| 1568 | unsigned int i; |
| 1569 | |
| 1570 | for (i = 0; i < w->data_deps.nr; i++) { |
| 1571 | int dep_idx; |
| 1572 | |
| 1573 | igt_assert(w->data_deps.list[i] <= 0); |
| 1574 | |
| 1575 | if (!w->data_deps.list[i]) |
| 1576 | continue; |
| 1577 | |
| 1578 | dep_idx = w->idx + w->data_deps.list[i]; |
| 1579 | |
| 1580 | igt_assert(dep_idx >= 0 && dep_idx < w->idx); |
| 1581 | igt_assert(wrk->steps[dep_idx].type == BATCH); |
| 1582 | |
| 1583 | gem_sync(fd, wrk->steps[dep_idx].obj[0].handle); |
| 1584 | |
| 1585 | synced = true; |
| 1586 | } |
| 1587 | |
| 1588 | return synced; |
| 1589 | } |
| 1590 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1591 | static void *run_workload(void *data) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1592 | { |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1593 | struct workload *wrk = (struct workload *)data; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1594 | struct timespec t_start, t_end; |
| 1595 | struct w_step *w; |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1596 | bool last_sync = false; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1597 | int throttle = -1; |
| 1598 | int qd_throttle = -1; |
Chris Wilson | cb5479a | 2017-05-17 19:54:49 +0100 | [diff] [blame] | 1599 | int count; |
| 1600 | int i; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1601 | |
| 1602 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 1603 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1604 | hars_petruska_f54_1_random_seed((wrk->flags & SYNCEDCLIENTS) ? |
| 1605 | 0 : wrk->id); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1606 | |
Chris Wilson | dd7bc6d | 2017-05-10 11:29:09 +0100 | [diff] [blame] | 1607 | init_status_page(wrk, INIT_ALL); |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1608 | for (count = 0; wrk->run && (wrk->background || count < wrk->repeat); |
| 1609 | count++) { |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 1610 | unsigned int cur_seqno = wrk->sync_seqno; |
| 1611 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1612 | clock_gettime(CLOCK_MONOTONIC, &wrk->repeat_start); |
| 1613 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1614 | for (i = 0, w = wrk->steps; wrk->run && (i < wrk->nr_steps); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1615 | i++, w++) { |
| 1616 | enum intel_engine_id engine = w->engine; |
| 1617 | int do_sleep = 0; |
| 1618 | |
| 1619 | if (w->type == DELAY) { |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1620 | do_sleep = w->delay; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1621 | } else if (w->type == PERIOD) { |
| 1622 | struct timespec now; |
| 1623 | |
| 1624 | clock_gettime(CLOCK_MONOTONIC, &now); |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1625 | do_sleep = w->period - |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1626 | elapsed_us(&wrk->repeat_start, &now); |
| 1627 | if (do_sleep < 0) { |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1628 | if (verbose > 1) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1629 | printf("%u: Dropped period @ %u/%u (%dus late)!\n", |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1630 | wrk->id, count, i, do_sleep); |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1631 | continue; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1632 | } |
| 1633 | } else if (w->type == SYNC) { |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1634 | unsigned int s_idx = i + w->target; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1635 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 1636 | igt_assert(s_idx >= 0 && s_idx < i); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1637 | igt_assert(wrk->steps[s_idx].type == BATCH); |
| 1638 | gem_sync(fd, wrk->steps[s_idx].obj[0].handle); |
| 1639 | continue; |
| 1640 | } else if (w->type == THROTTLE) { |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1641 | throttle = w->throttle; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1642 | continue; |
| 1643 | } else if (w->type == QD_THROTTLE) { |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1644 | qd_throttle = w->throttle; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1645 | continue; |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 1646 | } else if (w->type == SW_FENCE) { |
| 1647 | igt_assert(w->emit_fence < 0); |
| 1648 | w->emit_fence = |
| 1649 | sw_sync_timeline_create_fence(wrk->sync_timeline, |
| 1650 | cur_seqno + w->idx); |
| 1651 | igt_assert(w->emit_fence > 0); |
| 1652 | continue; |
| 1653 | } else if (w->type == SW_FENCE_SIGNAL) { |
| 1654 | int tgt = w->idx + w->target; |
| 1655 | int inc; |
| 1656 | |
| 1657 | igt_assert(tgt >= 0 && tgt < i); |
| 1658 | igt_assert(wrk->steps[tgt].type == SW_FENCE); |
| 1659 | cur_seqno += wrk->steps[tgt].idx; |
| 1660 | inc = cur_seqno - wrk->sync_seqno; |
| 1661 | sw_sync_timeline_inc(wrk->sync_timeline, inc); |
| 1662 | continue; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1663 | } |
| 1664 | |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1665 | if (do_sleep || w->type == PERIOD) { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1666 | usleep(do_sleep); |
| 1667 | continue; |
| 1668 | } |
| 1669 | |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1670 | igt_assert(w->type == BATCH); |
| 1671 | |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1672 | if ((wrk->flags & DEPSYNC) && engine == VCS) |
| 1673 | last_sync = sync_deps(wrk, w); |
| 1674 | |
| 1675 | if (last_sync && (wrk->flags & HEARTBEAT)) |
| 1676 | init_status_page(wrk, 0); |
| 1677 | |
| 1678 | last_sync = false; |
| 1679 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1680 | wrk->nr_bb[engine]++; |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1681 | if (engine == VCS && wrk->balancer) { |
| 1682 | engine = wrk->balancer->balance(wrk->balancer, |
| 1683 | wrk, w); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1684 | wrk->nr_bb[engine]++; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1685 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1686 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1687 | if (throttle > 0) |
| 1688 | w_sync_to(wrk, w, i - throttle); |
| 1689 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1690 | do_eb(wrk, w, engine, wrk->flags); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1691 | |
Chris Wilson | 12e2def | 2017-05-09 22:50:19 +0100 | [diff] [blame] | 1692 | if (w->request != -1) { |
| 1693 | igt_list_del(&w->rq_link); |
| 1694 | wrk->nrequest[w->request]--; |
| 1695 | } |
| 1696 | w->request = engine; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1697 | igt_list_add_tail(&w->rq_link, &wrk->requests[engine]); |
| 1698 | wrk->nrequest[engine]++; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1699 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1700 | if (!wrk->run) |
| 1701 | break; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1702 | |
Tvrtko Ursulin | 3e622a8 | 2017-05-16 11:46:50 +0100 | [diff] [blame] | 1703 | if (w->sync) { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1704 | gem_sync(fd, w->obj[0].handle); |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1705 | last_sync = true; |
Chris Wilson | 75b2b1a | 2017-05-10 11:03:57 +0100 | [diff] [blame] | 1706 | } |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1707 | |
Chris Wilson | 3b72346 | 2017-05-09 16:35:22 +0100 | [diff] [blame] | 1708 | if (qd_throttle > 0) { |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1709 | while (wrk->nrequest[engine] > qd_throttle) { |
| 1710 | struct w_step *s; |
| 1711 | |
| 1712 | s = igt_list_first_entry(&wrk->requests[engine], |
| 1713 | s, rq_link); |
| 1714 | |
| 1715 | gem_sync(fd, s->obj[0].handle); |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1716 | last_sync = true; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1717 | |
Chris Wilson | 12e2def | 2017-05-09 22:50:19 +0100 | [diff] [blame] | 1718 | s->request = -1; |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1719 | igt_list_del(&s->rq_link); |
| 1720 | wrk->nrequest[engine]--; |
| 1721 | } |
| 1722 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1723 | } |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1724 | |
Tvrtko Ursulin | 987abfb | 2017-05-16 13:49:21 +0100 | [diff] [blame] | 1725 | if (wrk->sync_timeline) { |
| 1726 | int inc; |
| 1727 | |
| 1728 | inc = wrk->nr_steps - (cur_seqno - wrk->sync_seqno); |
| 1729 | sw_sync_timeline_inc(wrk->sync_timeline, inc); |
| 1730 | wrk->sync_seqno += wrk->nr_steps; |
| 1731 | } |
| 1732 | |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1733 | /* Cleanup all fences instantiated in this iteration. */ |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1734 | for (i = 0, w = wrk->steps; wrk->run && (i < wrk->nr_steps); |
Tvrtko Ursulin | a47419f | 2017-05-16 10:27:45 +0100 | [diff] [blame] | 1735 | i++, w++) { |
| 1736 | if (w->emit_fence > 0) { |
| 1737 | close(w->emit_fence); |
| 1738 | w->emit_fence = -1; |
| 1739 | } |
| 1740 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1741 | } |
| 1742 | |
Chris Wilson | 5be0563 | 2017-05-09 10:53:39 +0100 | [diff] [blame] | 1743 | for (i = 0; i < NUM_ENGINES; i++) { |
| 1744 | if (!wrk->nrequest[i]) |
| 1745 | continue; |
| 1746 | |
| 1747 | w = igt_list_last_entry(&wrk->requests[i], w, rq_link); |
| 1748 | gem_sync(fd, w->obj[0].handle); |
| 1749 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1750 | |
| 1751 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1752 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1753 | if (wrk->print_stats) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 1754 | double t = elapsed(&t_start, &t_end); |
| 1755 | |
Chris Wilson | cb5479a | 2017-05-17 19:54:49 +0100 | [diff] [blame] | 1756 | printf("%c%u: %.3fs elapsed (%d cycles, %.3f workloads/s).", |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1757 | wrk->background ? ' ' : '*', wrk->id, |
| 1758 | t, count, count / t); |
| 1759 | if (wrk->balancer) |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1760 | printf(" %lu (%lu + %lu) total VCS batches.", |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 1761 | wrk->nr_bb[VCS], wrk->nr_bb[VCS1], wrk->nr_bb[VCS2]); |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1762 | if (wrk->balancer && wrk->balancer->get_qd) |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1763 | printf(" Average queue depths %.3f, %.3f.", |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 1764 | (double)wrk->qd_sum[VCS1] / wrk->nr_bb[VCS], |
| 1765 | (double)wrk->qd_sum[VCS2] / wrk->nr_bb[VCS]); |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1766 | putchar('\n'); |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 1767 | } |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 1768 | |
| 1769 | return NULL; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1770 | } |
| 1771 | |
| 1772 | static void fini_workload(struct workload *wrk) |
| 1773 | { |
| 1774 | free(wrk->steps); |
| 1775 | free(wrk); |
| 1776 | } |
| 1777 | |
| 1778 | static unsigned long calibrate_nop(unsigned int tolerance_pct) |
| 1779 | { |
| 1780 | const uint32_t bbe = 0xa << 23; |
| 1781 | unsigned int loops = 17; |
| 1782 | unsigned int usecs = nop_calibration_us; |
| 1783 | struct drm_i915_gem_exec_object2 obj = {}; |
| 1784 | struct drm_i915_gem_execbuffer2 eb = |
| 1785 | { .buffer_count = 1, .buffers_ptr = (uintptr_t)&obj}; |
| 1786 | long size, last_size; |
| 1787 | struct timespec t_0, t_end; |
| 1788 | |
| 1789 | clock_gettime(CLOCK_MONOTONIC, &t_0); |
| 1790 | |
| 1791 | size = 256 * 1024; |
| 1792 | do { |
| 1793 | struct timespec t_start; |
| 1794 | |
| 1795 | obj.handle = gem_create(fd, size); |
| 1796 | gem_write(fd, obj.handle, size - sizeof(bbe), &bbe, |
| 1797 | sizeof(bbe)); |
| 1798 | gem_execbuf(fd, &eb); |
| 1799 | gem_sync(fd, obj.handle); |
| 1800 | |
| 1801 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 1802 | for (int loop = 0; loop < loops; loop++) |
| 1803 | gem_execbuf(fd, &eb); |
| 1804 | gem_sync(fd, obj.handle); |
| 1805 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1806 | |
| 1807 | gem_close(fd, obj.handle); |
| 1808 | |
| 1809 | last_size = size; |
| 1810 | size = loops * size / elapsed(&t_start, &t_end) / 1e6 * usecs; |
| 1811 | size = ALIGN(size, sizeof(uint32_t)); |
| 1812 | } while (elapsed(&t_0, &t_end) < 5 || |
| 1813 | abs(size - last_size) > (size * tolerance_pct / 100)); |
| 1814 | |
| 1815 | return size / sizeof(uint32_t); |
| 1816 | } |
| 1817 | |
| 1818 | static void print_help(void) |
| 1819 | { |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1820 | unsigned int i; |
| 1821 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1822 | puts( |
| 1823 | "Usage: gem_wsim [OPTIONS]\n" |
| 1824 | "\n" |
| 1825 | "Runs a simulated workload on the GPU.\n" |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1826 | "When ran without arguments performs a GPU calibration result of which needs to\n" |
| 1827 | "be provided when running the simulation in subsequent invocations.\n" |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1828 | "\n" |
| 1829 | "Options:\n" |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1830 | " -h This text.\n" |
| 1831 | " -q Be quiet - do not output anything to stdout.\n" |
| 1832 | " -n <n> Nop calibration value.\n" |
| 1833 | " -t <n> Nop calibration tolerance percentage.\n" |
| 1834 | " Use when there is a difficulty obtaining calibration with the\n" |
| 1835 | " default settings.\n" |
Tvrtko Ursulin | a71597a | 2017-05-19 13:05:44 +0100 | [diff] [blame] | 1836 | " -p <n> Context priority to use for the following workload on the\n" |
| 1837 | " command line.\n" |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1838 | " -w <desc|path> Filename or a workload descriptor.\n" |
| 1839 | " Can be given multiple times.\n" |
| 1840 | " -W <desc|path> Filename or a master workload descriptor.\n" |
| 1841 | " Only one master workload can be optinally specified in which\n" |
| 1842 | " case all other workloads become background ones and run as\n" |
| 1843 | " long as the master.\n" |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1844 | " -a <desc|path> Append a workload to all other workloads.\n" |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1845 | " -r <n> How many times to emit the workload.\n" |
| 1846 | " -c <n> Fork N clients emitting the workload simultaneously.\n" |
| 1847 | " -x Swap VCS1 and VCS2 engines in every other client.\n" |
| 1848 | " -b <n> Load balancing to use.\n" |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1849 | " Available load balancers are:" |
| 1850 | ); |
| 1851 | |
| 1852 | for (i = 0; i < ARRAY_SIZE(all_balancers); i++) { |
| 1853 | igt_assert(all_balancers[i].desc); |
| 1854 | printf( |
| 1855 | " %s (%u): %s\n", |
| 1856 | all_balancers[i].name, all_balancers[i].id, |
| 1857 | all_balancers[i].desc); |
| 1858 | } |
| 1859 | puts( |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1860 | " Balancers can be specified either as names or as their id\n" |
| 1861 | " number as listed above.\n" |
| 1862 | " -2 Remap VCS2 to BCS.\n" |
| 1863 | " -R Round-robin initial VCS assignment per client.\n" |
Tvrtko Ursulin | dd5a378 | 2017-06-05 08:00:52 +0100 | [diff] [blame] | 1864 | " -H Send heartbeat on synchronisation points with seqno based\n" |
| 1865 | " balancers. Gives better engine busyness view in some cases.\n" |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 1866 | " -S Synchronize the sequence of random batch durations between\n" |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 1867 | " clients.\n" |
| 1868 | " -G Global load balancing - a single load balancer will be shared\n" |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 1869 | " between all clients and there will be a single seqno domain.\n" |
| 1870 | " -d Sync between data dependencies in userspace." |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1871 | ); |
| 1872 | } |
| 1873 | |
| 1874 | static char *load_workload_descriptor(char *filename) |
| 1875 | { |
| 1876 | struct stat sbuf; |
| 1877 | char *buf; |
| 1878 | int infd, ret, i; |
| 1879 | ssize_t len; |
| 1880 | |
| 1881 | ret = stat(filename, &sbuf); |
| 1882 | if (ret || !S_ISREG(sbuf.st_mode)) |
| 1883 | return filename; |
| 1884 | |
| 1885 | igt_assert(sbuf.st_size < 1024 * 1024); /* Just so. */ |
| 1886 | buf = malloc(sbuf.st_size); |
| 1887 | igt_assert(buf); |
| 1888 | |
| 1889 | infd = open(filename, O_RDONLY); |
| 1890 | igt_assert(infd >= 0); |
| 1891 | len = read(infd, buf, sbuf.st_size); |
| 1892 | igt_assert(len == sbuf.st_size); |
| 1893 | close(infd); |
| 1894 | |
| 1895 | for (i = 0; i < len; i++) { |
| 1896 | if (buf[i] == '\n') |
| 1897 | buf[i] = ','; |
| 1898 | } |
| 1899 | |
| 1900 | len--; |
| 1901 | while (buf[len] == ',') |
| 1902 | buf[len--] = 0; |
| 1903 | |
| 1904 | return buf; |
| 1905 | } |
| 1906 | |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 1907 | static struct w_arg * |
| 1908 | add_workload_arg(struct w_arg *w_args, unsigned int nr_args, char *w_arg, int prio) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1909 | { |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 1910 | w_args = realloc(w_args, sizeof(*w_args) * nr_args); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1911 | igt_assert(w_args); |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 1912 | w_args[nr_args - 1] = (struct w_arg) { w_arg, NULL, prio }; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1913 | |
| 1914 | return w_args; |
| 1915 | } |
| 1916 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1917 | static int find_balancer_by_name(char *name) |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 1918 | { |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 1919 | unsigned int i; |
| 1920 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1921 | for (i = 0; i < ARRAY_SIZE(all_balancers); i++) { |
| 1922 | if (!strcasecmp(name, all_balancers[i].name)) |
| 1923 | return all_balancers[i].id; |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 1924 | } |
| 1925 | |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 1926 | return -1; |
| 1927 | } |
| 1928 | |
| 1929 | static const struct workload_balancer *find_balancer_by_id(unsigned int id) |
| 1930 | { |
| 1931 | unsigned int i; |
| 1932 | |
| 1933 | for (i = 0; i < ARRAY_SIZE(all_balancers); i++) { |
| 1934 | if (id == all_balancers[i].id) |
| 1935 | return &all_balancers[i]; |
| 1936 | } |
| 1937 | |
| 1938 | return NULL; |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 1939 | } |
| 1940 | |
Chris Wilson | 474bcdd | 2017-05-09 15:13:09 +0100 | [diff] [blame] | 1941 | static void init_clocks(void) |
| 1942 | { |
| 1943 | struct timespec t_start, t_end; |
| 1944 | uint32_t rcs_start, rcs_end; |
| 1945 | double overhead, t; |
| 1946 | |
| 1947 | intel_register_access_init(intel_get_pci_device(), false, fd); |
| 1948 | |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 1949 | if (verbose <= 1) |
Chris Wilson | 474bcdd | 2017-05-09 15:13:09 +0100 | [diff] [blame] | 1950 | return; |
| 1951 | |
| 1952 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 1953 | for (int i = 0; i < 100; i++) |
| 1954 | rcs_start = *REG(RCS_TIMESTAMP); |
| 1955 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1956 | overhead = 2 * elapsed(&t_start, &t_end) / 100; |
| 1957 | |
| 1958 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 1959 | for (int i = 0; i < 100; i++) |
| 1960 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1961 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1962 | overhead += elapsed(&t_start, &t_end) / 100; |
| 1963 | |
| 1964 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 1965 | rcs_start = *REG(RCS_TIMESTAMP); |
| 1966 | usleep(100); |
| 1967 | rcs_end = *REG(RCS_TIMESTAMP); |
| 1968 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 1969 | |
| 1970 | t = elapsed(&t_start, &t_end) - overhead; |
| 1971 | printf("%d cycles in %.1fus, i.e. 1024 cycles takes %1.fus\n", |
| 1972 | rcs_end - rcs_start, 1e6*t, 1024e6 * t / (rcs_end - rcs_start)); |
| 1973 | } |
| 1974 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1975 | int main(int argc, char **argv) |
| 1976 | { |
| 1977 | unsigned int repeat = 1; |
| 1978 | unsigned int clients = 1; |
| 1979 | unsigned int flags = 0; |
| 1980 | struct timespec t_start, t_end; |
| 1981 | struct workload **w, **wrk = NULL; |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1982 | struct workload *app_w = NULL; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1983 | unsigned int nr_w_args = 0; |
| 1984 | int master_workload = -1; |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 1985 | char *append_workload_arg = NULL; |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 1986 | struct w_arg *w_args = NULL; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1987 | unsigned int tolerance_pct = 1; |
| 1988 | const struct workload_balancer *balancer = NULL; |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 1989 | char *endptr = NULL; |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 1990 | int prio = 0; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 1991 | double t; |
| 1992 | int i, c; |
| 1993 | |
Tvrtko Ursulin | 3153082 | 2017-05-11 17:10:15 +0100 | [diff] [blame] | 1994 | /* |
| 1995 | * Open the device via the low-level API so we can do the GPU quiesce |
| 1996 | * manually as close as possible in time to the start of the workload. |
| 1997 | * This minimizes the gap in engine utilization tracking when observed |
| 1998 | * via external tools like trace.pl. |
| 1999 | */ |
| 2000 | fd = __drm_open_driver(DRIVER_INTEL); |
| 2001 | igt_require(fd); |
| 2002 | |
Chris Wilson | 474bcdd | 2017-05-09 15:13:09 +0100 | [diff] [blame] | 2003 | init_clocks(); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2004 | |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 2005 | while ((c = getopt(argc, argv, "hqv2RSHxGdc:n:r:w:W:a:t:b:p:")) != -1) { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2006 | switch (c) { |
| 2007 | case 'W': |
| 2008 | if (master_workload >= 0) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2009 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2010 | fprintf(stderr, |
| 2011 | "Only one master workload can be given!\n"); |
| 2012 | return 1; |
| 2013 | } |
| 2014 | master_workload = nr_w_args; |
| 2015 | /* Fall through */ |
| 2016 | case 'w': |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2017 | w_args = add_workload_arg(w_args, ++nr_w_args, optarg, prio); |
| 2018 | break; |
| 2019 | case 'p': |
| 2020 | prio = atoi(optarg); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2021 | break; |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 2022 | case 'a': |
| 2023 | if (append_workload_arg) { |
| 2024 | if (verbose) |
| 2025 | fprintf(stderr, |
| 2026 | "Only one append workload can be given!\n"); |
| 2027 | return 1; |
| 2028 | } |
| 2029 | append_workload_arg = optarg; |
| 2030 | break; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2031 | case 'c': |
| 2032 | clients = strtol(optarg, NULL, 0); |
| 2033 | break; |
| 2034 | case 't': |
| 2035 | tolerance_pct = strtol(optarg, NULL, 0); |
| 2036 | break; |
| 2037 | case 'n': |
| 2038 | nop_calibration = strtol(optarg, NULL, 0); |
| 2039 | break; |
| 2040 | case 'r': |
| 2041 | repeat = strtol(optarg, NULL, 0); |
| 2042 | break; |
| 2043 | case 'q': |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2044 | verbose = 0; |
| 2045 | break; |
| 2046 | case 'v': |
| 2047 | verbose++; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2048 | break; |
| 2049 | case 'x': |
| 2050 | flags |= SWAPVCS; |
| 2051 | break; |
Tvrtko Ursulin | b56e171 | 2017-05-08 13:56:32 +0100 | [diff] [blame] | 2052 | case '2': |
| 2053 | flags |= VCS2REMAP; |
| 2054 | break; |
Tvrtko Ursulin | 7736d7e | 2017-05-09 09:21:03 +0100 | [diff] [blame] | 2055 | case 'R': |
| 2056 | flags |= INITVCSRR; |
| 2057 | break; |
Tvrtko Ursulin | 8540b91 | 2017-05-09 09:39:17 +0100 | [diff] [blame] | 2058 | case 'S': |
| 2059 | flags |= SYNCEDCLIENTS; |
| 2060 | break; |
Chris Wilson | 70d3814 | 2017-05-10 11:38:39 +0100 | [diff] [blame] | 2061 | case 'H': |
| 2062 | flags |= HEARTBEAT; |
| 2063 | break; |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 2064 | case 'G': |
| 2065 | flags |= GLOBAL_BALANCE; |
| 2066 | break; |
Tvrtko Ursulin | 6f2e3ba | 2017-06-05 11:16:59 +0100 | [diff] [blame] | 2067 | case 'd': |
| 2068 | flags |= DEPSYNC; |
| 2069 | break; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2070 | case 'b': |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 2071 | i = find_balancer_by_name(optarg); |
Tvrtko Ursulin | 0e0eca3 | 2017-05-09 10:01:22 +0100 | [diff] [blame] | 2072 | if (i < 0) { |
| 2073 | i = strtol(optarg, &endptr, 0); |
| 2074 | if (endptr && *endptr) |
| 2075 | i = -1; |
| 2076 | } |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 2077 | |
| 2078 | if (i >= 0) { |
| 2079 | balancer = find_balancer_by_id(i); |
| 2080 | if (balancer) { |
| 2081 | igt_assert(intel_gen(intel_get_drm_devid(fd)) >= balancer->min_gen); |
| 2082 | flags |= BALANCE | balancer->flags; |
| 2083 | } |
| 2084 | } |
| 2085 | |
| 2086 | if (!balancer) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2087 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2088 | fprintf(stderr, |
| 2089 | "Unknown balancing mode '%s'!\n", |
| 2090 | optarg); |
| 2091 | return 1; |
| 2092 | } |
| 2093 | break; |
| 2094 | case 'h': |
| 2095 | print_help(); |
| 2096 | return 0; |
| 2097 | default: |
| 2098 | return 1; |
| 2099 | } |
| 2100 | } |
| 2101 | |
Tvrtko Ursulin | dd5a378 | 2017-06-05 08:00:52 +0100 | [diff] [blame] | 2102 | if ((flags & HEARTBEAT) && !(flags & SEQNO)) { |
| 2103 | if (verbose) |
| 2104 | fprintf(stderr, "Heartbeat needs a seqno based balancer!\n"); |
| 2105 | return 1; |
| 2106 | } |
| 2107 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2108 | if (!nop_calibration) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2109 | if (verbose > 1) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2110 | printf("Calibrating nop delay with %u%% tolerance...\n", |
| 2111 | tolerance_pct); |
| 2112 | nop_calibration = calibrate_nop(tolerance_pct); |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2113 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2114 | printf("Nop calibration for %uus delay is %lu.\n", |
| 2115 | nop_calibration_us, nop_calibration); |
| 2116 | |
| 2117 | return 0; |
| 2118 | } |
| 2119 | |
| 2120 | if (!nr_w_args) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2121 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2122 | fprintf(stderr, "No workload descriptor(s)!\n"); |
| 2123 | return 1; |
| 2124 | } |
| 2125 | |
| 2126 | if (nr_w_args > 1 && clients > 1) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2127 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2128 | fprintf(stderr, |
| 2129 | "Cloned clients cannot be combined with multiple workloads!\n"); |
| 2130 | return 1; |
| 2131 | } |
| 2132 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 2133 | if ((flags & GLOBAL_BALANCE) && !balancer) { |
| 2134 | if (verbose) |
| 2135 | fprintf(stderr, |
| 2136 | "Balancer not specified in global balancing mode!\n"); |
| 2137 | return 1; |
| 2138 | } |
| 2139 | |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 2140 | if (append_workload_arg) { |
| 2141 | append_workload_arg = load_workload_descriptor(append_workload_arg); |
| 2142 | if (!append_workload_arg) { |
| 2143 | if (verbose) |
| 2144 | fprintf(stderr, |
| 2145 | "Failed to load append workload descriptor!\n"); |
| 2146 | return 1; |
| 2147 | } |
| 2148 | } |
| 2149 | |
| 2150 | if (append_workload_arg) { |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2151 | struct w_arg arg = { NULL, append_workload_arg, 0 }; |
| 2152 | app_w = parse_workload(&arg, flags, NULL); |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 2153 | if (!app_w) { |
| 2154 | if (verbose) |
| 2155 | fprintf(stderr, |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2156 | "Failed to parse append workload!\n"); |
Tvrtko Ursulin | 2e10dc9 | 2017-05-16 07:56:20 +0100 | [diff] [blame] | 2157 | return 1; |
| 2158 | } |
| 2159 | } |
| 2160 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2161 | wrk = calloc(nr_w_args, sizeof(*wrk)); |
| 2162 | igt_assert(wrk); |
| 2163 | |
| 2164 | for (i = 0; i < nr_w_args; i++) { |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2165 | w_args[i].desc = load_workload_descriptor(w_args[i].filename); |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 2166 | |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2167 | if (!w_args[i].desc) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2168 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2169 | fprintf(stderr, |
| 2170 | "Failed to load workload descriptor %u!\n", |
| 2171 | i); |
| 2172 | return 1; |
| 2173 | } |
| 2174 | |
Chris Wilson | 8910738 | 2017-05-17 19:55:25 +0100 | [diff] [blame] | 2175 | wrk[i] = parse_workload(&w_args[i], flags, app_w); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2176 | if (!wrk[i]) { |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2177 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2178 | fprintf(stderr, |
| 2179 | "Failed to parse workload %u!\n", i); |
| 2180 | return 1; |
| 2181 | } |
| 2182 | } |
| 2183 | |
Tvrtko Ursulin | 251b130 | 2017-05-10 11:11:10 +0100 | [diff] [blame] | 2184 | if (nr_w_args > 1) |
| 2185 | clients = nr_w_args; |
| 2186 | |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2187 | if (verbose > 1) { |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2188 | printf("Using %lu nop calibration for %uus delay.\n", |
| 2189 | nop_calibration, nop_calibration_us); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2190 | printf("%u client%s.\n", clients, clients > 1 ? "s" : ""); |
| 2191 | if (flags & SWAPVCS) |
| 2192 | printf("Swapping VCS rings between clients.\n"); |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 2193 | if (flags & GLOBAL_BALANCE) |
| 2194 | printf("Using %s balancer in global mode.\n", |
| 2195 | balancer->name); |
| 2196 | else if (balancer) |
Tvrtko Ursulin | f39a7c5 | 2017-05-10 12:06:05 +0100 | [diff] [blame] | 2197 | printf("Using %s balancer.\n", balancer->name); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2198 | } |
| 2199 | |
| 2200 | if (master_workload >= 0 && clients == 1) |
| 2201 | master_workload = -1; |
| 2202 | |
| 2203 | w = calloc(clients, sizeof(struct workload *)); |
| 2204 | igt_assert(w); |
| 2205 | |
| 2206 | for (i = 0; i < clients; i++) { |
| 2207 | unsigned int flags_ = flags; |
| 2208 | |
| 2209 | w[i] = clone_workload(wrk[nr_w_args > 1 ? i : 0]); |
| 2210 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2211 | if (flags & SWAPVCS && i & 1) |
| 2212 | flags_ &= ~SWAPVCS; |
| 2213 | |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 2214 | if (flags & GLOBAL_BALANCE) { |
| 2215 | w[i]->balancer = &global_balancer; |
| 2216 | w[i]->global_wrk = w[0]; |
| 2217 | w[i]->global_balancer = balancer; |
| 2218 | } else { |
| 2219 | w[i]->balancer = balancer; |
| 2220 | } |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2221 | |
| 2222 | w[i]->flags = flags; |
| 2223 | w[i]->repeat = repeat; |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2224 | w[i]->background = master_workload >= 0 && i != master_workload; |
| 2225 | w[i]->print_stats = verbose > 1 || |
| 2226 | (verbose > 0 && master_workload == i); |
Tvrtko Ursulin | 1c6c53c | 2017-05-22 10:30:45 +0100 | [diff] [blame] | 2227 | |
| 2228 | prepare_workload(i, w[i], flags_); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2229 | } |
| 2230 | |
Tvrtko Ursulin | 3153082 | 2017-05-11 17:10:15 +0100 | [diff] [blame] | 2231 | gem_quiescent_gpu(fd); |
| 2232 | |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2233 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
| 2234 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2235 | for (i = 0; i < clients; i++) { |
| 2236 | int ret; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2237 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2238 | ret = pthread_create(&w[i]->thread, NULL, run_workload, w[i]); |
| 2239 | igt_assert_eq(ret, 0); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2240 | } |
| 2241 | |
| 2242 | if (master_workload >= 0) { |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2243 | int ret = pthread_join(w[master_workload]->thread, NULL); |
| 2244 | |
| 2245 | igt_assert(ret == 0); |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2246 | |
| 2247 | for (i = 0; i < clients; i++) |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2248 | w[i]->run = false; |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2249 | } |
| 2250 | |
Tvrtko Ursulin | 255d1fc | 2017-05-19 15:13:48 +0100 | [diff] [blame] | 2251 | for (i = 0; i < clients; i++) { |
| 2252 | if (master_workload != i) { |
| 2253 | int ret = pthread_join(w[i]->thread, NULL); |
| 2254 | igt_assert(ret == 0); |
| 2255 | } |
| 2256 | } |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2257 | |
| 2258 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
| 2259 | |
| 2260 | t = elapsed(&t_start, &t_end); |
Chris Wilson | d099f7d | 2017-05-09 14:22:21 +0100 | [diff] [blame] | 2261 | if (verbose) |
Tvrtko Ursulin | 054eb1a | 2017-03-30 14:32:29 +0100 | [diff] [blame] | 2262 | printf("%.3fs elapsed (%.3f workloads/s)\n", |
| 2263 | t, clients * repeat / t); |
| 2264 | |
| 2265 | for (i = 0; i < clients; i++) |
| 2266 | fini_workload(w[i]); |
| 2267 | free(w); |
| 2268 | for (i = 0; i < nr_w_args; i++) |
| 2269 | fini_workload(wrk[i]); |
| 2270 | free(w_args); |
| 2271 | |
| 2272 | return 0; |
| 2273 | } |