Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 1 | #include <stdint.h> |
| 2 | #include <stdbool.h> |
| 3 | #include <linux/perf_event.h> |
| 4 | #include <sys/types.h> |
| 5 | #include <sys/mman.h> |
| 6 | #include <sys/ioctl.h> |
| 7 | #include <unistd.h> |
| 8 | #include <stdlib.h> |
| 9 | #include <stdio.h> |
| 10 | #include <string.h> |
| 11 | #include <fcntl.h> |
| 12 | #include <errno.h> |
| 13 | |
| 14 | #include "gpu-perf.h" |
| 15 | |
| 16 | #if defined(__i386__) |
| 17 | #define rmb() asm volatile("lock; addl $0,0(%%esp)" ::: "memory") |
| 18 | #endif |
| 19 | |
| 20 | #if defined(__x86_64__) |
| 21 | #define rmb() asm volatile("lfence" ::: "memory") |
| 22 | #endif |
| 23 | |
| 24 | #define TRACING_EVENT_PATH "/sys/kernel/debug/tracing/events" |
| 25 | #define N_PAGES 32 |
| 26 | |
| 27 | struct sample_event { |
| 28 | struct perf_event_header header; |
| 29 | uint32_t pid, tid; |
| 30 | uint64_t time; |
| 31 | uint64_t id; |
| 32 | uint32_t raw_size; |
| 33 | uint32_t raw[0]; |
| 34 | }; |
| 35 | |
| 36 | static int |
| 37 | perf_event_open(struct perf_event_attr *attr, |
| 38 | pid_t pid, |
| 39 | int cpu, |
| 40 | int group_fd, |
| 41 | unsigned long flags) |
| 42 | { |
| 43 | #ifndef __NR_perf_event_open |
| 44 | #if defined(__i386__) |
| 45 | #define __NR_perf_event_open 336 |
| 46 | #elif defined(__x86_64__) |
| 47 | #define __NR_perf_event_open 298 |
| 48 | #else |
| 49 | #define __NR_perf_event_open 0 |
| 50 | #endif |
| 51 | #endif |
| 52 | |
| 53 | attr->size = sizeof(*attr); |
| 54 | return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); |
| 55 | } |
| 56 | |
| 57 | static uint64_t tracepoint_id(const char *sys, const char *name) |
| 58 | { |
| 59 | char buf[1024]; |
| 60 | int fd, n; |
| 61 | |
| 62 | snprintf(buf, sizeof(buf), "%s/%s/%s/id", TRACING_EVENT_PATH, sys, name); |
| 63 | fd = open(buf, 0); |
| 64 | if (fd < 0) |
| 65 | return 0; |
| 66 | n = read(fd, buf, sizeof(buf)-1); |
| 67 | close(fd); |
| 68 | if (n < 0) |
| 69 | return 0; |
| 70 | |
| 71 | buf[n] = '\0'; |
| 72 | return strtoull(buf, 0, 0); |
| 73 | } |
| 74 | |
| 75 | static int perf_tracepoint_open(struct gpu_perf *gp, |
| 76 | const char *sys, const char *name, |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 77 | int (*func)(struct gpu_perf *, const void *)) |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 78 | { |
| 79 | struct perf_event_attr attr; |
| 80 | struct gpu_perf_sample *sample; |
| 81 | int n, *fd; |
| 82 | |
| 83 | memset(&attr, 0, sizeof (attr)); |
| 84 | |
| 85 | attr.type = PERF_TYPE_TRACEPOINT; |
| 86 | attr.config = tracepoint_id(sys, name); |
| 87 | if (attr.config == 0) |
| 88 | return ENOENT; |
| 89 | |
| 90 | attr.sample_period = 1; |
| 91 | attr.sample_type = (PERF_SAMPLE_TIME | PERF_SAMPLE_STREAM_ID | PERF_SAMPLE_TID | PERF_SAMPLE_RAW); |
| 92 | attr.read_format = PERF_FORMAT_ID; |
| 93 | |
| 94 | attr.exclude_guest = 1; |
| 95 | attr.mmap = gp->nr_events == 0; |
| 96 | attr.comm = gp->nr_events == 0; |
| 97 | |
| 98 | n = gp->nr_cpus * (gp->nr_events+1); |
| 99 | fd = realloc(gp->fd, n*sizeof(int)); |
| 100 | sample = realloc(gp->sample, n*sizeof(*gp->sample)); |
| 101 | if (fd == NULL || sample == NULL) |
| 102 | return ENOMEM; |
| 103 | gp->fd = fd; |
| 104 | gp->sample = sample; |
| 105 | |
| 106 | fd += gp->nr_events * gp->nr_cpus; |
| 107 | sample += gp->nr_events * gp->nr_cpus; |
| 108 | for (n = 0; n < gp->nr_cpus; n++) { |
| 109 | uint64_t track[2]; |
| 110 | |
| 111 | fd[n] = perf_event_open(&attr, -1, n, -1, 0); |
| 112 | if (fd[n] == -1) |
| 113 | return errno; |
| 114 | |
| 115 | /* read back the event to establish id->tracepoint */ |
| 116 | read(fd[n], track, sizeof(track)); |
| 117 | sample[n].id = track[1]; |
| 118 | sample[n].func = func; |
| 119 | |
| 120 | if (gp->nr_events) |
| 121 | ioctl(fd[n], PERF_EVENT_IOC_SET_OUTPUT, gp->fd[n]); |
| 122 | |
| 123 | } |
| 124 | |
| 125 | gp->nr_events++; |
| 126 | return 0; |
| 127 | } |
| 128 | |
| 129 | static int perf_mmap(struct gpu_perf *gp) |
| 130 | { |
| 131 | int size = (1 + N_PAGES) * gp->page_size; |
| 132 | int n; |
| 133 | |
| 134 | gp->map = malloc(sizeof(void *)*gp->nr_cpus); |
| 135 | if (gp->map == NULL) |
| 136 | return ENOMEM; |
| 137 | |
| 138 | for (n = 0; n < gp->nr_cpus; n++) { |
| 139 | gp->map[n] = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, gp->fd[n], 0); |
| 140 | if (gp->map[n] == (void *)-1) |
| 141 | goto err; |
| 142 | } |
| 143 | |
| 144 | return 0; |
| 145 | |
| 146 | err: |
| 147 | while (--n > 0) |
| 148 | munmap(gp->map[n], size); |
| 149 | free(gp->map); |
| 150 | gp->map = NULL; |
| 151 | return EINVAL; |
| 152 | } |
| 153 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 154 | static int seqno_start(struct gpu_perf *gp, const void *event) |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 155 | { |
| 156 | return 0; |
| 157 | } |
| 158 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 159 | static int seqno_end(struct gpu_perf *gp, const void *event) |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 160 | { |
| 161 | return 0; |
| 162 | } |
| 163 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 164 | static int flip_complete(struct gpu_perf *gp, const void *event) |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 165 | { |
| 166 | gp->flip_complete++; |
| 167 | return 1; |
| 168 | } |
| 169 | |
| 170 | void gpu_perf_init(struct gpu_perf *gp, unsigned flags) |
| 171 | { |
| 172 | memset(gp, 0, sizeof(*gp)); |
| 173 | gp->nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); |
| 174 | gp->page_size = getpagesize(); |
| 175 | |
| 176 | if (perf_tracepoint_open(gp, "i915", "i915_gem_ring_complete", seqno_end) == 0) |
| 177 | perf_tracepoint_open(gp, "i915", "i915_gem_ring_dispatch", seqno_start); |
| 178 | |
| 179 | perf_tracepoint_open(gp, "i915", "i915_flip_complete", flip_complete); |
| 180 | |
| 181 | if (gp->nr_events == 0) |
| 182 | return; |
| 183 | |
| 184 | if (perf_mmap(gp)) |
| 185 | return; |
| 186 | } |
| 187 | |
| 188 | static int process_sample(struct gpu_perf *gp, |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 189 | const struct perf_event_header *header) |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 190 | { |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 191 | const struct sample_event *sample = (const struct sample_event *)header; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 192 | int n, update = 0; |
| 193 | |
| 194 | /* hash me! */ |
| 195 | for (n = 0; n < gp->nr_cpus * gp->nr_events; n++) { |
| 196 | if (gp->sample[n].id != sample->id) |
| 197 | continue; |
| 198 | |
| 199 | update = 1; |
| 200 | if (gp->sample[n].func) |
| 201 | update = gp->sample[n].func(gp, sample); |
| 202 | break; |
| 203 | } |
| 204 | |
| 205 | return update; |
| 206 | } |
| 207 | |
| 208 | int gpu_perf_update(struct gpu_perf *gp) |
| 209 | { |
| 210 | const int size = N_PAGES * gp->page_size; |
| 211 | const int mask = size - 1; |
| 212 | uint8_t *buffer = NULL; |
| 213 | int buffer_size = 0; |
| 214 | int n, update = 0; |
| 215 | |
| 216 | if (gp->map == NULL) |
| 217 | return 0; |
| 218 | |
| 219 | for (n = 0; n < gp->nr_cpus; n++) { |
| 220 | struct perf_event_mmap_page *mmap = gp->map[n]; |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 221 | const uint8_t *data; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 222 | uint64_t head, tail; |
| 223 | |
| 224 | tail = mmap->data_tail; |
| 225 | head = mmap->data_head; |
| 226 | rmb(); |
| 227 | |
| 228 | if (head < tail) |
Chris Wilson | 3e430a8 | 2013-08-17 18:09:40 +0100 | [diff] [blame] | 229 | head += size; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 230 | |
| 231 | data = (uint8_t *)mmap + gp->page_size; |
| 232 | while (head - tail >= sizeof (struct perf_event_header)) { |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 233 | const struct perf_event_header *header; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 234 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 235 | header = (const struct perf_event_header *)(data + (tail & mask)); |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 236 | if (header->size > head - tail) |
| 237 | break; |
| 238 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 239 | if ((const uint8_t *)header + header->size > data + size) { |
| 240 | int before; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 241 | |
| 242 | if (header->size > buffer_size) { |
| 243 | uint8_t *b = realloc(buffer, header->size); |
| 244 | if (b == NULL) |
| 245 | break; |
| 246 | |
| 247 | buffer = b; |
| 248 | buffer_size = header->size; |
| 249 | } |
| 250 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 251 | before = data + size - (const uint8_t *)header; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 252 | |
Chris Wilson | 8cdb5bc | 2013-08-17 18:24:39 +0100 | [diff] [blame^] | 253 | memcpy(buffer, header, before); |
| 254 | memcpy(buffer + before, data, header->size - before); |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 255 | |
| 256 | header = (struct perf_event_header *)buffer; |
| 257 | } |
| 258 | |
| 259 | if (header->type == PERF_RECORD_SAMPLE) |
| 260 | update += process_sample(gp, header); |
| 261 | tail += header->size; |
| 262 | } |
| 263 | |
Chris Wilson | 3e430a8 | 2013-08-17 18:09:40 +0100 | [diff] [blame] | 264 | mmap->data_tail = tail & mask; |
Chris Wilson | cc45a9a | 2013-08-17 17:38:37 +0100 | [diff] [blame] | 265 | } |
| 266 | |
| 267 | free(buffer); |
| 268 | return update; |
| 269 | } |