| /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */ |
| |
| /* |
| * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org> |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| * |
| * Authors: |
| * Rob Clark <robclark@freedesktop.org> |
| */ |
| |
| #include "pipe/p_state.h" |
| #include "util/u_memory.h" |
| #include "util/u_inlines.h" |
| |
| #include "freedreno_query_hw.h" |
| #include "freedreno_context.h" |
| #include "freedreno_resource.h" |
| #include "freedreno_util.h" |
| |
| struct fd_hw_sample_period { |
| struct fd_hw_sample *start, *end; |
| struct list_head list; |
| }; |
| |
| /* maps query_type to sample provider idx: */ |
| static int pidx(unsigned query_type) |
| { |
| switch (query_type) { |
| case PIPE_QUERY_OCCLUSION_COUNTER: |
| return 0; |
| case PIPE_QUERY_OCCLUSION_PREDICATE: |
| return 1; |
| /* TODO currently queries only emitted in main pass (not in binning pass).. |
| * which is fine for occlusion query, but pretty much not anything else. |
| */ |
| case PIPE_QUERY_TIME_ELAPSED: |
| return 2; |
| case PIPE_QUERY_TIMESTAMP: |
| return 3; |
| default: |
| return -1; |
| } |
| } |
| |
| static struct fd_hw_sample * |
| get_sample(struct fd_batch *batch, struct fd_ringbuffer *ring, |
| unsigned query_type) |
| { |
| struct fd_context *ctx = batch->ctx; |
| struct fd_hw_sample *samp = NULL; |
| int idx = pidx(query_type); |
| |
| assume(idx >= 0); /* query never would have been created otherwise */ |
| |
| if (!batch->sample_cache[idx]) { |
| struct fd_hw_sample *new_samp = |
| ctx->sample_providers[idx]->get_sample(batch, ring); |
| fd_hw_sample_reference(ctx, &batch->sample_cache[idx], new_samp); |
| util_dynarray_append(&batch->samples, struct fd_hw_sample *, new_samp); |
| batch->needs_flush = true; |
| } |
| |
| fd_hw_sample_reference(ctx, &samp, batch->sample_cache[idx]); |
| |
| return samp; |
| } |
| |
| static void |
| clear_sample_cache(struct fd_batch *batch) |
| { |
| int i; |
| |
| for (i = 0; i < ARRAY_SIZE(batch->sample_cache); i++) |
| fd_hw_sample_reference(batch->ctx, &batch->sample_cache[i], NULL); |
| } |
| |
| static bool |
| is_active(struct fd_hw_query *hq, enum fd_render_stage stage) |
| { |
| return !!(hq->provider->active & stage); |
| } |
| |
| |
| static void |
| resume_query(struct fd_batch *batch, struct fd_hw_query *hq, |
| struct fd_ringbuffer *ring) |
| { |
| int idx = pidx(hq->provider->query_type); |
| DBG("%p", hq); |
| assert(idx >= 0); /* query never would have been created otherwise */ |
| assert(!hq->period); |
| batch->active_providers |= (1 << idx); |
| hq->period = slab_alloc_st(&batch->ctx->sample_period_pool); |
| list_inithead(&hq->period->list); |
| hq->period->start = get_sample(batch, ring, hq->base.type); |
| /* NOTE: slab_alloc_st() does not zero out the buffer: */ |
| hq->period->end = NULL; |
| } |
| |
| static void |
| pause_query(struct fd_batch *batch, struct fd_hw_query *hq, |
| struct fd_ringbuffer *ring) |
| { |
| int idx = pidx(hq->provider->query_type); |
| DBG("%p", hq); |
| assert(idx >= 0); /* query never would have been created otherwise */ |
| assert(hq->period && !hq->period->end); |
| assert(batch->active_providers & (1 << idx)); |
| hq->period->end = get_sample(batch, ring, hq->base.type); |
| list_addtail(&hq->period->list, &hq->periods); |
| hq->period = NULL; |
| } |
| |
| static void |
| destroy_periods(struct fd_context *ctx, struct fd_hw_query *hq) |
| { |
| struct fd_hw_sample_period *period, *s; |
| LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->periods, list) { |
| fd_hw_sample_reference(ctx, &period->start, NULL); |
| fd_hw_sample_reference(ctx, &period->end, NULL); |
| list_del(&period->list); |
| slab_free_st(&ctx->sample_period_pool, period); |
| } |
| } |
| |
| static void |
| fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q) |
| { |
| struct fd_hw_query *hq = fd_hw_query(q); |
| |
| DBG("%p: active=%d", q, q->active); |
| |
| destroy_periods(ctx, hq); |
| list_del(&hq->list); |
| |
| free(hq); |
| } |
| |
| static boolean |
| fd_hw_begin_query(struct fd_context *ctx, struct fd_query *q) |
| { |
| struct fd_batch *batch = ctx->batch; |
| struct fd_hw_query *hq = fd_hw_query(q); |
| |
| DBG("%p: active=%d", q, q->active); |
| |
| if (q->active) |
| return false; |
| |
| /* begin_query() should clear previous results: */ |
| destroy_periods(ctx, hq); |
| |
| if (batch && is_active(hq, batch->stage)) |
| resume_query(batch, hq, batch->draw); |
| |
| q->active = true; |
| |
| /* add to active list: */ |
| assert(list_empty(&hq->list)); |
| list_addtail(&hq->list, &ctx->active_queries); |
| |
| return true; |
| } |
| |
| static void |
| fd_hw_end_query(struct fd_context *ctx, struct fd_query *q) |
| { |
| struct fd_batch *batch = ctx->batch; |
| struct fd_hw_query *hq = fd_hw_query(q); |
| |
| /* there are a couple special cases, which don't have |
| * a matching ->begin_query(): |
| */ |
| if (skip_begin_query(q->type) && !q->active) { |
| fd_hw_begin_query(ctx, q); |
| } |
| |
| DBG("%p: active=%d", q, q->active); |
| |
| if (!q->active) |
| return; |
| |
| if (batch && is_active(hq, batch->stage)) |
| pause_query(batch, hq, batch->draw); |
| |
| q->active = false; |
| /* remove from active list: */ |
| list_delinit(&hq->list); |
| } |
| |
| /* helper to get ptr to specified sample: */ |
| static void * sampptr(struct fd_hw_sample *samp, uint32_t n, void *ptr) |
| { |
| return ((char *)ptr) + (samp->tile_stride * n) + samp->offset; |
| } |
| |
| static boolean |
| fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q, |
| boolean wait, union pipe_query_result *result) |
| { |
| struct fd_hw_query *hq = fd_hw_query(q); |
| const struct fd_hw_sample_provider *p = hq->provider; |
| struct fd_hw_sample_period *period; |
| |
| DBG("%p: wait=%d, active=%d", q, wait, q->active); |
| |
| if (q->active) |
| return false; |
| |
| util_query_clear_result(result, q->type); |
| |
| if (LIST_IS_EMPTY(&hq->periods)) |
| return true; |
| |
| assert(LIST_IS_EMPTY(&hq->list)); |
| assert(!hq->period); |
| |
| /* if !wait, then check the last sample (the one most likely to |
| * not be ready yet) and bail if it is not ready: |
| */ |
| if (!wait) { |
| int ret; |
| |
| period = LIST_ENTRY(struct fd_hw_sample_period, |
| hq->periods.prev, list); |
| |
| struct fd_resource *rsc = fd_resource(period->end->prsc); |
| |
| if (pending(rsc, false)) { |
| /* piglit spec@arb_occlusion_query@occlusion_query_conform |
| * test, and silly apps perhaps, get stuck in a loop trying |
| * to get query result forever with wait==false.. we don't |
| * wait to flush unnecessarily but we also don't want to |
| * spin forever: |
| */ |
| if (hq->no_wait_cnt++ > 5) |
| fd_batch_flush(rsc->write_batch, false); |
| return false; |
| } |
| |
| if (!rsc->bo) |
| return false; |
| |
| ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, |
| DRM_FREEDRENO_PREP_READ | DRM_FREEDRENO_PREP_NOSYNC); |
| if (ret) |
| return false; |
| |
| fd_bo_cpu_fini(rsc->bo); |
| } |
| |
| /* sum the result across all sample periods: */ |
| LIST_FOR_EACH_ENTRY(period, &hq->periods, list) { |
| struct fd_hw_sample *start = period->start; |
| struct fd_hw_sample *end = period->end; |
| unsigned i; |
| |
| /* start and end samples should be from same batch: */ |
| assert(start->prsc == end->prsc); |
| assert(start->num_tiles == end->num_tiles); |
| |
| struct fd_resource *rsc = fd_resource(start->prsc); |
| |
| if (rsc->write_batch) |
| fd_batch_flush(rsc->write_batch, true); |
| |
| /* some piglit tests at least do query with no draws, I guess: */ |
| if (!rsc->bo) |
| continue; |
| |
| fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, DRM_FREEDRENO_PREP_READ); |
| |
| void *ptr = fd_bo_map(rsc->bo); |
| |
| for (i = 0; i < start->num_tiles; i++) { |
| p->accumulate_result(ctx, sampptr(period->start, i, ptr), |
| sampptr(period->end, i, ptr), result); |
| } |
| |
| fd_bo_cpu_fini(rsc->bo); |
| } |
| |
| return true; |
| } |
| |
| static const struct fd_query_funcs hw_query_funcs = { |
| .destroy_query = fd_hw_destroy_query, |
| .begin_query = fd_hw_begin_query, |
| .end_query = fd_hw_end_query, |
| .get_query_result = fd_hw_get_query_result, |
| }; |
| |
| struct fd_query * |
| fd_hw_create_query(struct fd_context *ctx, unsigned query_type) |
| { |
| struct fd_hw_query *hq; |
| struct fd_query *q; |
| int idx = pidx(query_type); |
| |
| if ((idx < 0) || !ctx->sample_providers[idx]) |
| return NULL; |
| |
| hq = CALLOC_STRUCT(fd_hw_query); |
| if (!hq) |
| return NULL; |
| |
| DBG("%p: query_type=%u", hq, query_type); |
| |
| hq->provider = ctx->sample_providers[idx]; |
| |
| list_inithead(&hq->periods); |
| list_inithead(&hq->list); |
| |
| q = &hq->base; |
| q->funcs = &hw_query_funcs; |
| q->type = query_type; |
| |
| return q; |
| } |
| |
| struct fd_hw_sample * |
| fd_hw_sample_init(struct fd_batch *batch, uint32_t size) |
| { |
| struct fd_hw_sample *samp = slab_alloc_st(&batch->ctx->sample_pool); |
| pipe_reference_init(&samp->reference, 1); |
| samp->size = size; |
| debug_assert(util_is_power_of_two(size)); |
| batch->next_sample_offset = align(batch->next_sample_offset, size); |
| samp->offset = batch->next_sample_offset; |
| /* NOTE: slab_alloc_st() does not zero out the buffer: */ |
| samp->prsc = NULL; |
| samp->num_tiles = 0; |
| samp->tile_stride = 0; |
| batch->next_sample_offset += size; |
| |
| if (!batch->query_buf) { |
| struct pipe_screen *pscreen = &batch->ctx->screen->base; |
| struct pipe_resource templ = { |
| .target = PIPE_BUFFER, |
| .format = PIPE_FORMAT_R8_UNORM, |
| .bind = PIPE_BIND_QUERY_BUFFER, |
| .width0 = 0, /* create initially zero size buffer */ |
| .height0 = 1, |
| .depth0 = 1, |
| .array_size = 1, |
| .last_level = 0, |
| .nr_samples = 1, |
| }; |
| batch->query_buf = pscreen->resource_create(pscreen, &templ); |
| } |
| |
| pipe_resource_reference(&samp->prsc, batch->query_buf); |
| |
| return samp; |
| } |
| |
| void |
| __fd_hw_sample_destroy(struct fd_context *ctx, struct fd_hw_sample *samp) |
| { |
| pipe_resource_reference(&samp->prsc, NULL); |
| slab_free_st(&ctx->sample_pool, samp); |
| } |
| |
| /* called from gmem code once total storage requirements are known (ie. |
| * number of samples times number of tiles) |
| */ |
| void |
| fd_hw_query_prepare(struct fd_batch *batch, uint32_t num_tiles) |
| { |
| uint32_t tile_stride = batch->next_sample_offset; |
| |
| if (tile_stride > 0) |
| fd_resource_resize(batch->query_buf, tile_stride * num_tiles); |
| |
| batch->query_tile_stride = tile_stride; |
| |
| while (batch->samples.size > 0) { |
| struct fd_hw_sample *samp = |
| util_dynarray_pop(&batch->samples, struct fd_hw_sample *); |
| samp->num_tiles = num_tiles; |
| samp->tile_stride = tile_stride; |
| fd_hw_sample_reference(batch->ctx, &samp, NULL); |
| } |
| |
| /* reset things for next batch: */ |
| batch->next_sample_offset = 0; |
| } |
| |
| void |
| fd_hw_query_prepare_tile(struct fd_batch *batch, uint32_t n, |
| struct fd_ringbuffer *ring) |
| { |
| uint32_t tile_stride = batch->query_tile_stride; |
| uint32_t offset = tile_stride * n; |
| |
| /* bail if no queries: */ |
| if (tile_stride == 0) |
| return; |
| |
| fd_wfi(batch, ring); |
| OUT_PKT0 (ring, HW_QUERY_BASE_REG, 1); |
| OUT_RELOCW(ring, fd_resource(batch->query_buf)->bo, offset, 0, 0); |
| } |
| |
| void |
| fd_hw_query_set_stage(struct fd_batch *batch, struct fd_ringbuffer *ring, |
| enum fd_render_stage stage) |
| { |
| /* special case: internal blits (like mipmap level generation) |
| * go through normal draw path (via util_blitter_blit()).. but |
| * we need to ignore the FD_STAGE_DRAW which will be set, so we |
| * don't enable queries which should be paused during internal |
| * blits: |
| */ |
| if ((batch->stage == FD_STAGE_BLIT) && |
| (stage != FD_STAGE_NULL)) |
| return; |
| |
| if (stage != batch->stage) { |
| struct fd_hw_query *hq; |
| LIST_FOR_EACH_ENTRY(hq, &batch->ctx->active_queries, list) { |
| bool was_active = is_active(hq, batch->stage); |
| bool now_active = is_active(hq, stage); |
| |
| if (now_active && !was_active) |
| resume_query(batch, hq, ring); |
| else if (was_active && !now_active) |
| pause_query(batch, hq, ring); |
| } |
| } |
| clear_sample_cache(batch); |
| batch->stage = stage; |
| } |
| |
| /* call the provider->enable() for all the hw queries that were active |
| * in the current batch. This sets up perfctr selector regs statically |
| * for the duration of the batch. |
| */ |
| void |
| fd_hw_query_enable(struct fd_batch *batch, struct fd_ringbuffer *ring) |
| { |
| struct fd_context *ctx = batch->ctx; |
| for (int idx = 0; idx < MAX_HW_SAMPLE_PROVIDERS; idx++) { |
| if (batch->active_providers & (1 << idx)) { |
| assert(ctx->sample_providers[idx]); |
| if (ctx->sample_providers[idx]->enable) |
| ctx->sample_providers[idx]->enable(ctx, ring); |
| } |
| } |
| batch->active_providers = 0; /* clear it for next frame */ |
| } |
| |
| void |
| fd_hw_query_register_provider(struct pipe_context *pctx, |
| const struct fd_hw_sample_provider *provider) |
| { |
| struct fd_context *ctx = fd_context(pctx); |
| int idx = pidx(provider->query_type); |
| |
| assert((0 <= idx) && (idx < MAX_HW_SAMPLE_PROVIDERS)); |
| assert(!ctx->sample_providers[idx]); |
| |
| ctx->sample_providers[idx] = provider; |
| } |
| |
| void |
| fd_hw_query_init(struct pipe_context *pctx) |
| { |
| struct fd_context *ctx = fd_context(pctx); |
| |
| slab_create(&ctx->sample_pool, sizeof(struct fd_hw_sample), |
| 16); |
| slab_create(&ctx->sample_period_pool, sizeof(struct fd_hw_sample_period), |
| 16); |
| list_inithead(&ctx->active_queries); |
| } |
| |
| void |
| fd_hw_query_fini(struct pipe_context *pctx) |
| { |
| struct fd_context *ctx = fd_context(pctx); |
| |
| slab_destroy(&ctx->sample_pool); |
| slab_destroy(&ctx->sample_period_pool); |
| } |