| /********************************************************** |
| * Copyright 2008-2009 VMware, Inc. All rights reserved. |
| * |
| * Permission is hereby granted, free of charge, to any person |
| * obtaining a copy of this software and associated documentation |
| * files (the "Software"), to deal in the Software without |
| * restriction, including without limitation the rights to use, copy, |
| * modify, merge, publish, distribute, sublicense, and/or sell copies |
| * of the Software, and to permit persons to whom the Software is |
| * furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be |
| * included in all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| * |
| **********************************************************/ |
| |
| #include "util/u_math.h" |
| #include "util/u_memory.h" |
| #include "util/crc32.h" |
| |
| #include "svga_debug.h" |
| #include "svga_format.h" |
| #include "svga_winsys.h" |
| #include "svga_screen.h" |
| #include "svga_screen_cache.h" |
| #include "svga_context.h" |
| |
| |
| #define SVGA_SURFACE_CACHE_ENABLED 1 |
| |
| |
| /** |
| * Return the size of the surface described by the key (in bytes). |
| */ |
| static unsigned |
| surface_size(const struct svga_host_surface_cache_key *key) |
| { |
| unsigned bw, bh, bpb, total_size, i; |
| |
| assert(key->numMipLevels > 0); |
| assert(key->numFaces > 0); |
| assert(key->arraySize > 0); |
| |
| if (key->format == SVGA3D_BUFFER) { |
| /* Special case: we don't want to count vertex/index buffers |
| * against the cache size limit, so view them as zero-sized. |
| */ |
| return 0; |
| } |
| |
| svga_format_size(key->format, &bw, &bh, &bpb); |
| |
| total_size = 0; |
| |
| for (i = 0; i < key->numMipLevels; i++) { |
| unsigned w = u_minify(key->size.width, i); |
| unsigned h = u_minify(key->size.height, i); |
| unsigned d = u_minify(key->size.depth, i); |
| unsigned img_size = ((w + bw - 1) / bw) * ((h + bh - 1) / bh) * d * bpb; |
| total_size += img_size; |
| } |
| |
| total_size *= key->numFaces * key->arraySize * MAX2(1, key->sampleCount); |
| |
| return total_size; |
| } |
| |
| |
| /** |
| * Compute the bucket for this key. |
| */ |
| static inline unsigned |
| svga_screen_cache_bucket(const struct svga_host_surface_cache_key *key) |
| { |
| return util_hash_crc32(key, sizeof *key) % SVGA_HOST_SURFACE_CACHE_BUCKETS; |
| } |
| |
| |
| /** |
| * Search the cache for a surface that matches the key. If a match is |
| * found, remove it from the cache and return the surface pointer. |
| * Return NULL otherwise. |
| */ |
| static struct svga_winsys_surface * |
| svga_screen_cache_lookup(struct svga_screen *svgascreen, |
| const struct svga_host_surface_cache_key *key) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| struct svga_host_surface_cache_entry *entry; |
| struct svga_winsys_surface *handle = NULL; |
| struct list_head *curr, *next; |
| unsigned bucket; |
| unsigned tries = 0; |
| |
| assert(key->cachable); |
| |
| bucket = svga_screen_cache_bucket(key); |
| |
| mtx_lock(&cache->mutex); |
| |
| curr = cache->bucket[bucket].next; |
| next = curr->next; |
| while (curr != &cache->bucket[bucket]) { |
| ++tries; |
| |
| entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, bucket_head); |
| |
| assert(entry->handle); |
| |
| /* If the key matches and the fence is signalled (the surface is no |
| * longer needed) the lookup was successful. We found a surface that |
| * can be reused. |
| * We unlink the surface from the cache entry and we add the entry to |
| * the 'empty' list. |
| */ |
| if (memcmp(&entry->key, key, sizeof *key) == 0 && |
| sws->fence_signalled(sws, entry->fence, 0) == 0) { |
| unsigned surf_size; |
| |
| assert(sws->surface_is_flushed(sws, entry->handle)); |
| |
| handle = entry->handle; /* Reference is transfered here. */ |
| entry->handle = NULL; |
| |
| /* Remove from hash table */ |
| LIST_DEL(&entry->bucket_head); |
| |
| /* remove from LRU list */ |
| LIST_DEL(&entry->head); |
| |
| /* Add the cache entry (but not the surface!) to the empty list */ |
| LIST_ADD(&entry->head, &cache->empty); |
| |
| /* update the cache size */ |
| surf_size = surface_size(&entry->key); |
| assert(surf_size <= cache->total_size); |
| if (surf_size > cache->total_size) |
| cache->total_size = 0; /* should never happen, but be safe */ |
| else |
| cache->total_size -= surf_size; |
| |
| break; |
| } |
| |
| curr = next; |
| next = curr->next; |
| } |
| |
| mtx_unlock(&cache->mutex); |
| |
| if (SVGA_DEBUG & DEBUG_DMA) |
| debug_printf("%s: cache %s after %u tries (bucket %d)\n", __FUNCTION__, |
| handle ? "hit" : "miss", tries, bucket); |
| |
| return handle; |
| } |
| |
| |
| /** |
| * Free the least recently used entries in the surface cache until the |
| * cache size is <= the target size OR there are no unused entries left |
| * to discard. We don't do any flushing to try to free up additional |
| * surfaces. |
| */ |
| static void |
| svga_screen_cache_shrink(struct svga_screen *svgascreen, |
| unsigned target_size) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| struct svga_host_surface_cache_entry *entry = NULL, *next_entry; |
| |
| /* Walk over the list of unused buffers in reverse order: from oldest |
| * to newest. |
| */ |
| LIST_FOR_EACH_ENTRY_SAFE_REV(entry, next_entry, &cache->unused, head) { |
| if (entry->key.format != SVGA3D_BUFFER) { |
| /* we don't want to discard vertex/index buffers */ |
| |
| cache->total_size -= surface_size(&entry->key); |
| |
| assert(entry->handle); |
| sws->surface_reference(sws, &entry->handle, NULL); |
| |
| LIST_DEL(&entry->bucket_head); |
| LIST_DEL(&entry->head); |
| LIST_ADD(&entry->head, &cache->empty); |
| |
| if (cache->total_size <= target_size) { |
| /* all done */ |
| break; |
| } |
| } |
| } |
| } |
| |
| |
| /** |
| * Add a surface to the cache. This is done when the driver deletes |
| * the surface. Note: transfers a handle reference. |
| */ |
| static void |
| svga_screen_cache_add(struct svga_screen *svgascreen, |
| const struct svga_host_surface_cache_key *key, |
| struct svga_winsys_surface **p_handle) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| struct svga_host_surface_cache_entry *entry = NULL; |
| struct svga_winsys_surface *handle = *p_handle; |
| unsigned surf_size; |
| |
| assert(key->cachable); |
| |
| if (!handle) |
| return; |
| |
| surf_size = surface_size(key); |
| |
| *p_handle = NULL; |
| mtx_lock(&cache->mutex); |
| |
| if (surf_size >= SVGA_HOST_SURFACE_CACHE_BYTES) { |
| /* this surface is too large to cache, just free it */ |
| sws->surface_reference(sws, &handle, NULL); |
| mtx_unlock(&cache->mutex); |
| return; |
| } |
| |
| if (cache->total_size + surf_size > SVGA_HOST_SURFACE_CACHE_BYTES) { |
| /* Adding this surface would exceed the cache size. |
| * Try to discard least recently used entries until we hit the |
| * new target cache size. |
| */ |
| unsigned target_size = SVGA_HOST_SURFACE_CACHE_BYTES - surf_size; |
| |
| svga_screen_cache_shrink(svgascreen, target_size); |
| |
| if (cache->total_size > target_size) { |
| /* we weren't able to shrink the cache as much as we wanted so |
| * just discard this surface. |
| */ |
| sws->surface_reference(sws, &handle, NULL); |
| mtx_unlock(&cache->mutex); |
| return; |
| } |
| } |
| |
| if (!LIST_IS_EMPTY(&cache->empty)) { |
| /* An empty entry has no surface associated with it. |
| * Use the first empty entry. |
| */ |
| entry = LIST_ENTRY(struct svga_host_surface_cache_entry, |
| cache->empty.next, head); |
| |
| /* Remove from LRU list */ |
| LIST_DEL(&entry->head); |
| } |
| else if (!LIST_IS_EMPTY(&cache->unused)) { |
| /* free the last used buffer and reuse its entry */ |
| entry = LIST_ENTRY(struct svga_host_surface_cache_entry, |
| cache->unused.prev, head); |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "unref sid %p (make space)\n", entry->handle); |
| |
| cache->total_size -= surface_size(&entry->key); |
| |
| sws->surface_reference(sws, &entry->handle, NULL); |
| |
| /* Remove from hash table */ |
| LIST_DEL(&entry->bucket_head); |
| |
| /* Remove from LRU list */ |
| LIST_DEL(&entry->head); |
| } |
| |
| if (entry) { |
| assert(entry->handle == NULL); |
| entry->handle = handle; |
| memcpy(&entry->key, key, sizeof entry->key); |
| |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "cache sid %p\n", entry->handle); |
| LIST_ADD(&entry->head, &cache->validated); |
| |
| cache->total_size += surf_size; |
| } |
| else { |
| /* Couldn't cache the buffer -- this really shouldn't happen */ |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "unref sid %p (couldn't find space)\n", handle); |
| sws->surface_reference(sws, &handle, NULL); |
| } |
| |
| mtx_unlock(&cache->mutex); |
| } |
| |
| |
| /** |
| * Called during the screen flush to move all buffers not in a validate list |
| * into the unused list. |
| */ |
| void |
| svga_screen_cache_flush(struct svga_screen *svgascreen, |
| struct svga_context *svga, |
| struct pipe_fence_handle *fence) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| struct svga_host_surface_cache_entry *entry; |
| struct list_head *curr, *next; |
| unsigned bucket; |
| |
| mtx_lock(&cache->mutex); |
| |
| /* Loop over entries in the invalidated list */ |
| curr = cache->invalidated.next; |
| next = curr->next; |
| while (curr != &cache->invalidated) { |
| entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, head); |
| |
| assert(entry->handle); |
| |
| if (sws->surface_is_flushed(sws, entry->handle)) { |
| /* remove entry from the invalidated list */ |
| LIST_DEL(&entry->head); |
| |
| sws->fence_reference(sws, &entry->fence, fence); |
| |
| /* Add entry to the unused list */ |
| LIST_ADD(&entry->head, &cache->unused); |
| |
| /* Add entry to the hash table bucket */ |
| bucket = svga_screen_cache_bucket(&entry->key); |
| LIST_ADD(&entry->bucket_head, &cache->bucket[bucket]); |
| } |
| |
| curr = next; |
| next = curr->next; |
| } |
| |
| curr = cache->validated.next; |
| next = curr->next; |
| while (curr != &cache->validated) { |
| entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, head); |
| |
| assert(entry->handle); |
| |
| if (sws->surface_is_flushed(sws, entry->handle)) { |
| /* remove entry from the validated list */ |
| LIST_DEL(&entry->head); |
| |
| /* It is now safe to invalidate the surface content. |
| * It will be done using the current context. |
| */ |
| if (svga->swc->surface_invalidate(svga->swc, entry->handle) != PIPE_OK) { |
| MAYBE_UNUSED enum pipe_error ret; |
| |
| /* Even though surface invalidation here is done after the command |
| * buffer is flushed, it is still possible that it will |
| * fail because there might be just enough of this command that is |
| * filling up the command buffer, so in this case we will call |
| * the winsys flush directly to flush the buffer. |
| * Note, we don't want to call svga_context_flush() here because |
| * this function itself is called inside svga_context_flush(). |
| */ |
| svga->swc->flush(svga->swc, NULL); |
| ret = svga->swc->surface_invalidate(svga->swc, entry->handle); |
| assert(ret == PIPE_OK); |
| } |
| |
| /* add the entry to the invalidated list */ |
| LIST_ADD(&entry->head, &cache->invalidated); |
| } |
| |
| curr = next; |
| next = curr->next; |
| } |
| |
| mtx_unlock(&cache->mutex); |
| } |
| |
| |
| /** |
| * Free all the surfaces in the cache. |
| * Called when destroying the svga screen object. |
| */ |
| void |
| svga_screen_cache_cleanup(struct svga_screen *svgascreen) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| unsigned i; |
| |
| for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i) { |
| if (cache->entries[i].handle) { |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "unref sid %p (shutdown)\n", cache->entries[i].handle); |
| sws->surface_reference(sws, &cache->entries[i].handle, NULL); |
| |
| cache->total_size -= surface_size(&cache->entries[i].key); |
| } |
| |
| if (cache->entries[i].fence) |
| sws->fence_reference(sws, &cache->entries[i].fence, NULL); |
| } |
| |
| mtx_destroy(&cache->mutex); |
| } |
| |
| |
| enum pipe_error |
| svga_screen_cache_init(struct svga_screen *svgascreen) |
| { |
| struct svga_host_surface_cache *cache = &svgascreen->cache; |
| unsigned i; |
| |
| assert(cache->total_size == 0); |
| |
| (void) mtx_init(&cache->mutex, mtx_plain); |
| |
| for (i = 0; i < SVGA_HOST_SURFACE_CACHE_BUCKETS; ++i) |
| LIST_INITHEAD(&cache->bucket[i]); |
| |
| LIST_INITHEAD(&cache->unused); |
| |
| LIST_INITHEAD(&cache->validated); |
| |
| LIST_INITHEAD(&cache->invalidated); |
| |
| LIST_INITHEAD(&cache->empty); |
| for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i) |
| LIST_ADDTAIL(&cache->entries[i].head, &cache->empty); |
| |
| return PIPE_OK; |
| } |
| |
| |
| /** |
| * Allocate a new host-side surface. If the surface is marked as cachable, |
| * first try re-using a surface in the cache of freed surfaces. Otherwise, |
| * allocate a new surface. |
| * \param bind_flags bitmask of PIPE_BIND_x flags |
| * \param usage one of PIPE_USAGE_x values |
| * \param validated return True if the surface is a reused surface |
| */ |
| struct svga_winsys_surface * |
| svga_screen_surface_create(struct svga_screen *svgascreen, |
| unsigned bind_flags, enum pipe_resource_usage usage, |
| boolean *validated, |
| struct svga_host_surface_cache_key *key) |
| { |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| struct svga_winsys_surface *handle = NULL; |
| boolean cachable = SVGA_SURFACE_CACHE_ENABLED && key->cachable; |
| |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "%s sz %dx%dx%d mips %d faces %d arraySize %d cachable %d\n", |
| __FUNCTION__, |
| key->size.width, |
| key->size.height, |
| key->size.depth, |
| key->numMipLevels, |
| key->numFaces, |
| key->arraySize, |
| key->cachable); |
| |
| if (cachable) { |
| /* Try to re-cycle a previously freed, cached surface */ |
| if (key->format == SVGA3D_BUFFER) { |
| SVGA3dSurfaceFlags hint_flag; |
| |
| /* For buffers, round the buffer size up to the nearest power |
| * of two to increase the probability of cache hits. Keep |
| * texture surface dimensions unchanged. |
| */ |
| uint32_t size = 1; |
| while (size < key->size.width) |
| size <<= 1; |
| key->size.width = size; |
| |
| /* Determine whether the buffer is static or dynamic. |
| * This is a bit of a heuristic which can be tuned as needed. |
| */ |
| if (usage == PIPE_USAGE_DEFAULT || |
| usage == PIPE_USAGE_IMMUTABLE) { |
| hint_flag = SVGA3D_SURFACE_HINT_STATIC; |
| } |
| else if (bind_flags & PIPE_BIND_INDEX_BUFFER) { |
| /* Index buffers don't change too often. Mark them as static. |
| */ |
| hint_flag = SVGA3D_SURFACE_HINT_STATIC; |
| } |
| else { |
| /* Since we're reusing buffers we're effectively transforming all |
| * of them into dynamic buffers. |
| * |
| * It would be nice to not cache long lived static buffers. But there |
| * is no way to detect the long lived from short lived ones yet. A |
| * good heuristic would be buffer size. |
| */ |
| hint_flag = SVGA3D_SURFACE_HINT_DYNAMIC; |
| } |
| |
| key->flags &= ~(SVGA3D_SURFACE_HINT_STATIC | |
| SVGA3D_SURFACE_HINT_DYNAMIC); |
| key->flags |= hint_flag; |
| } |
| |
| handle = svga_screen_cache_lookup(svgascreen, key); |
| if (handle) { |
| if (key->format == SVGA3D_BUFFER) |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "reuse sid %p sz %d (buffer)\n", handle, |
| key->size.width); |
| else |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| "reuse sid %p sz %dx%dx%d mips %d faces %d arraySize %d\n", handle, |
| key->size.width, |
| key->size.height, |
| key->size.depth, |
| key->numMipLevels, |
| key->numFaces, |
| key->arraySize); |
| *validated = TRUE; |
| } |
| } |
| |
| if (!handle) { |
| /* Unable to recycle surface, allocate a new one */ |
| unsigned usage = 0; |
| |
| if (!key->cachable) |
| usage |= SVGA_SURFACE_USAGE_SHARED; |
| if (key->scanout) |
| usage |= SVGA_SURFACE_USAGE_SCANOUT; |
| |
| handle = sws->surface_create(sws, |
| key->flags, |
| key->format, |
| usage, |
| key->size, |
| key->numFaces * key->arraySize, |
| key->numMipLevels, |
| key->sampleCount); |
| if (handle) |
| SVGA_DBG(DEBUG_CACHE|DEBUG_DMA, |
| " CREATE sid %p sz %dx%dx%d\n", |
| handle, |
| key->size.width, |
| key->size.height, |
| key->size.depth); |
| |
| *validated = FALSE; |
| } |
| |
| return handle; |
| } |
| |
| |
| /** |
| * Release a surface. We don't actually free the surface- we put |
| * it into the cache of freed surfaces (if it's cachable). |
| */ |
| void |
| svga_screen_surface_destroy(struct svga_screen *svgascreen, |
| const struct svga_host_surface_cache_key *key, |
| struct svga_winsys_surface **p_handle) |
| { |
| struct svga_winsys_screen *sws = svgascreen->sws; |
| |
| /* We only set the cachable flag for surfaces of which we are the |
| * exclusive owner. So just hold onto our existing reference in |
| * that case. |
| */ |
| if (SVGA_SURFACE_CACHE_ENABLED && key->cachable) { |
| svga_screen_cache_add(svgascreen, key, p_handle); |
| } |
| else { |
| SVGA_DBG(DEBUG_DMA, |
| "unref sid %p (uncachable)\n", *p_handle); |
| sws->surface_reference(sws, p_handle, NULL); |
| } |
| } |
| |
| |
| /** |
| * Print/dump the contents of the screen cache. For debugging. |
| */ |
| void |
| svga_screen_cache_dump(const struct svga_screen *svgascreen) |
| { |
| const struct svga_host_surface_cache *cache = &svgascreen->cache; |
| unsigned bucket; |
| unsigned count = 0; |
| |
| debug_printf("svga3d surface cache:\n"); |
| for (bucket = 0; bucket < SVGA_HOST_SURFACE_CACHE_BUCKETS; bucket++) { |
| struct list_head *curr; |
| curr = cache->bucket[bucket].next; |
| while (curr && curr != &cache->bucket[bucket]) { |
| struct svga_host_surface_cache_entry *entry = |
| LIST_ENTRY(struct svga_host_surface_cache_entry, |
| curr, bucket_head); |
| if (entry->key.format == SVGA3D_BUFFER) { |
| debug_printf(" %p: buffer %u bytes\n", |
| entry->handle, |
| entry->key.size.width); |
| } |
| else { |
| debug_printf(" %p: %u x %u x %u format %u\n", |
| entry->handle, |
| entry->key.size.width, |
| entry->key.size.height, |
| entry->key.size.depth, |
| entry->key.format); |
| } |
| curr = curr->next; |
| count++; |
| } |
| } |
| |
| debug_printf("%u surfaces, %u bytes\n", count, cache->total_size); |
| } |