Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2015 Google Inc. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license that can be |
| 5 | * found in the LICENSE file. |
| 6 | */ |
| 7 | |
| 8 | #include "GrVkMemory.h" |
| 9 | |
| 10 | #include "GrVkGpu.h" |
| 11 | #include "GrVkUtil.h" |
| 12 | |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 13 | static bool get_valid_memory_type_index(const VkPhysicalDeviceMemoryProperties& physDevMemProps, |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 14 | uint32_t typeBits, |
| 15 | VkMemoryPropertyFlags requestedMemFlags, |
| 16 | uint32_t* typeIndex) { |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 17 | for (uint32_t i = 0; i < physDevMemProps.memoryTypeCount; ++i) { |
| 18 | if (typeBits & (1 << i)) { |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 19 | uint32_t supportedFlags = physDevMemProps.memoryTypes[i].propertyFlags & |
| 20 | requestedMemFlags; |
| 21 | if (supportedFlags == requestedMemFlags) { |
| 22 | *typeIndex = i; |
| 23 | return true; |
| 24 | } |
| 25 | } |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 26 | } |
| 27 | return false; |
| 28 | } |
| 29 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 30 | static GrVkGpu::Heap buffer_type_to_heap(GrVkBuffer::Type type) { |
| 31 | const GrVkGpu::Heap kBufferToHeap[]{ |
| 32 | GrVkGpu::kVertexBuffer_Heap, |
| 33 | GrVkGpu::kIndexBuffer_Heap, |
| 34 | GrVkGpu::kUniformBuffer_Heap, |
| 35 | GrVkGpu::kCopyReadBuffer_Heap, |
| 36 | GrVkGpu::kCopyWriteBuffer_Heap, |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 37 | }; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 38 | GR_STATIC_ASSERT(0 == GrVkBuffer::kVertex_Type); |
| 39 | GR_STATIC_ASSERT(1 == GrVkBuffer::kIndex_Type); |
| 40 | GR_STATIC_ASSERT(2 == GrVkBuffer::kUniform_Type); |
| 41 | GR_STATIC_ASSERT(3 == GrVkBuffer::kCopyRead_Type); |
| 42 | GR_STATIC_ASSERT(4 == GrVkBuffer::kCopyWrite_Type); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 43 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 44 | return kBufferToHeap[type]; |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | bool GrVkMemory::AllocAndBindBufferMemory(const GrVkGpu* gpu, |
| 48 | VkBuffer buffer, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 49 | GrVkBuffer::Type type, |
jvanverth | a584de9 | 2016-06-30 09:10:52 -0700 | [diff] [blame] | 50 | bool dynamic, |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 51 | GrVkAlloc* alloc) { |
jvanverth | e50f3e7 | 2016-03-28 07:03:06 -0700 | [diff] [blame] | 52 | const GrVkInterface* iface = gpu->vkInterface(); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 53 | VkDevice device = gpu->device(); |
| 54 | |
| 55 | VkMemoryRequirements memReqs; |
jvanverth | e50f3e7 | 2016-03-28 07:03:06 -0700 | [diff] [blame] | 56 | GR_VK_CALL(iface, GetBufferMemoryRequirements(device, buffer, &memReqs)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 57 | |
jvanverth | 7378ac8 | 2016-06-14 08:32:44 -0700 | [diff] [blame] | 58 | uint32_t typeIndex = 0; |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 59 | const VkPhysicalDeviceMemoryProperties& phDevMemProps = gpu->physicalDeviceMemoryProperties(); |
| 60 | if (dynamic) { |
| 61 | // try to get cached and ideally non-coherent memory first |
| 62 | if (!get_valid_memory_type_index(phDevMemProps, |
| 63 | memReqs.memoryTypeBits, |
| 64 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | |
| 65 | VK_MEMORY_PROPERTY_HOST_CACHED_BIT, |
| 66 | &typeIndex)) { |
| 67 | // some sort of host-visible memory type should always be available for dynamic buffers |
| 68 | SkASSERT_RELEASE(get_valid_memory_type_index(phDevMemProps, |
| 69 | memReqs.memoryTypeBits, |
| 70 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, |
| 71 | &typeIndex)); |
| 72 | } |
| 73 | |
| 74 | VkMemoryPropertyFlags mpf = phDevMemProps.memoryTypes[typeIndex].propertyFlags; |
| 75 | alloc->fFlags = mpf & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT ? 0x0 |
| 76 | : GrVkAlloc::kNoncoherent_Flag; |
| 77 | } else { |
| 78 | // device-local memory should always be available for static buffers |
| 79 | SkASSERT_RELEASE(get_valid_memory_type_index(phDevMemProps, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 80 | memReqs.memoryTypeBits, |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 81 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 82 | &typeIndex)); |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 83 | alloc->fFlags = 0x0; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | GrVkHeap* heap = gpu->getHeap(buffer_type_to_heap(type)); |
| 87 | |
| 88 | if (!heap->alloc(memReqs.size, memReqs.alignment, typeIndex, alloc)) { |
| 89 | SkDebugf("Failed to alloc buffer\n"); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 90 | return false; |
| 91 | } |
| 92 | |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 93 | // Bind buffer |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 94 | VkResult err = GR_VK_CALL(iface, BindBufferMemory(device, buffer, |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 95 | alloc->fMemory, alloc->fOffset)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 96 | if (err) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 97 | SkASSERT_RELEASE(heap->free(*alloc)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 98 | return false; |
| 99 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 100 | |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 101 | return true; |
| 102 | } |
| 103 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 104 | void GrVkMemory::FreeBufferMemory(const GrVkGpu* gpu, GrVkBuffer::Type type, |
| 105 | const GrVkAlloc& alloc) { |
| 106 | |
| 107 | GrVkHeap* heap = gpu->getHeap(buffer_type_to_heap(type)); |
| 108 | SkASSERT_RELEASE(heap->free(alloc)); |
| 109 | } |
| 110 | |
| 111 | // for debugging |
| 112 | static uint64_t gTotalImageMemory = 0; |
| 113 | static uint64_t gTotalImageMemoryFullPage = 0; |
| 114 | |
| 115 | const VkDeviceSize kMaxSmallImageSize = 16 * 1024; |
| 116 | const VkDeviceSize kMinVulkanPageSize = 16 * 1024; |
| 117 | |
| 118 | static VkDeviceSize align_size(VkDeviceSize size, VkDeviceSize alignment) { |
| 119 | return (size + alignment - 1) & ~(alignment - 1); |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 120 | } |
| 121 | |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 122 | bool GrVkMemory::AllocAndBindImageMemory(const GrVkGpu* gpu, |
| 123 | VkImage image, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 124 | bool linearTiling, |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 125 | GrVkAlloc* alloc) { |
jvanverth | e50f3e7 | 2016-03-28 07:03:06 -0700 | [diff] [blame] | 126 | const GrVkInterface* iface = gpu->vkInterface(); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 127 | VkDevice device = gpu->device(); |
| 128 | |
| 129 | VkMemoryRequirements memReqs; |
jvanverth | e50f3e7 | 2016-03-28 07:03:06 -0700 | [diff] [blame] | 130 | GR_VK_CALL(iface, GetImageMemoryRequirements(device, image, &memReqs)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 131 | |
jvanverth | 7378ac8 | 2016-06-14 08:32:44 -0700 | [diff] [blame] | 132 | uint32_t typeIndex = 0; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 133 | GrVkHeap* heap; |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 134 | const VkPhysicalDeviceMemoryProperties& phDevMemProps = gpu->physicalDeviceMemoryProperties(); |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 135 | if (linearTiling) { |
| 136 | VkMemoryPropertyFlags desiredMemProps = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 137 | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 138 | if (!get_valid_memory_type_index(phDevMemProps, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 139 | memReqs.memoryTypeBits, |
| 140 | desiredMemProps, |
| 141 | &typeIndex)) { |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 142 | // some sort of host-visible memory type should always be available |
| 143 | SkASSERT_RELEASE(get_valid_memory_type_index(phDevMemProps, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 144 | memReqs.memoryTypeBits, |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 145 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 146 | &typeIndex)); |
| 147 | } |
| 148 | heap = gpu->getHeap(GrVkGpu::kLinearImage_Heap); |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 149 | VkMemoryPropertyFlags mpf = phDevMemProps.memoryTypes[typeIndex].propertyFlags; |
| 150 | alloc->fFlags = mpf & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT ? 0x0 |
| 151 | : GrVkAlloc::kNoncoherent_Flag; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 152 | } else { |
| 153 | // this memory type should always be available |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 154 | SkASSERT_RELEASE(get_valid_memory_type_index(phDevMemProps, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 155 | memReqs.memoryTypeBits, |
| 156 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, |
| 157 | &typeIndex)); |
| 158 | if (memReqs.size <= kMaxSmallImageSize) { |
| 159 | heap = gpu->getHeap(GrVkGpu::kSmallOptimalImage_Heap); |
| 160 | } else { |
| 161 | heap = gpu->getHeap(GrVkGpu::kOptimalImage_Heap); |
| 162 | } |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 163 | alloc->fFlags = 0x0; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 164 | } |
| 165 | |
| 166 | if (!heap->alloc(memReqs.size, memReqs.alignment, typeIndex, alloc)) { |
| 167 | SkDebugf("Failed to alloc image\n"); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 168 | return false; |
| 169 | } |
| 170 | |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 171 | // Bind image |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 172 | VkResult err = GR_VK_CALL(iface, BindImageMemory(device, image, |
| 173 | alloc->fMemory, alloc->fOffset)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 174 | if (err) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 175 | SkASSERT_RELEASE(heap->free(*alloc)); |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 176 | return false; |
| 177 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 178 | |
| 179 | gTotalImageMemory += alloc->fSize; |
| 180 | |
| 181 | VkDeviceSize pageAlignedSize = align_size(alloc->fSize, kMinVulkanPageSize); |
| 182 | gTotalImageMemoryFullPage += pageAlignedSize; |
| 183 | |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 184 | return true; |
| 185 | } |
| 186 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 187 | void GrVkMemory::FreeImageMemory(const GrVkGpu* gpu, bool linearTiling, |
| 188 | const GrVkAlloc& alloc) { |
| 189 | GrVkHeap* heap; |
| 190 | if (linearTiling) { |
| 191 | heap = gpu->getHeap(GrVkGpu::kLinearImage_Heap); |
| 192 | } else if (alloc.fSize <= kMaxSmallImageSize) { |
| 193 | heap = gpu->getHeap(GrVkGpu::kSmallOptimalImage_Heap); |
| 194 | } else { |
| 195 | heap = gpu->getHeap(GrVkGpu::kOptimalImage_Heap); |
| 196 | } |
| 197 | if (!heap->free(alloc)) { |
| 198 | // must be an adopted allocation |
| 199 | GR_VK_CALL(gpu->vkInterface(), FreeMemory(gpu->device(), alloc.fMemory, nullptr)); |
| 200 | } else { |
| 201 | gTotalImageMemory -= alloc.fSize; |
| 202 | VkDeviceSize pageAlignedSize = align_size(alloc.fSize, kMinVulkanPageSize); |
| 203 | gTotalImageMemoryFullPage -= pageAlignedSize; |
| 204 | } |
jvanverth | 1e305ba | 2016-06-01 09:39:15 -0700 | [diff] [blame] | 205 | } |
| 206 | |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 207 | VkPipelineStageFlags GrVkMemory::LayoutToPipelineStageFlags(const VkImageLayout layout) { |
| 208 | if (VK_IMAGE_LAYOUT_GENERAL == layout) { |
| 209 | return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; |
| 210 | } else if (VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL == layout || |
| 211 | VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL == layout) { |
| 212 | return VK_PIPELINE_STAGE_TRANSFER_BIT; |
| 213 | } else if (VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL == layout || |
| 214 | VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL == layout || |
| 215 | VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL == layout || |
| 216 | VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL == layout) { |
| 217 | return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; |
| 218 | } else if (VK_IMAGE_LAYOUT_PREINITIALIZED == layout) { |
| 219 | return VK_PIPELINE_STAGE_HOST_BIT; |
| 220 | } |
| 221 | |
| 222 | SkASSERT(VK_IMAGE_LAYOUT_UNDEFINED == layout); |
| 223 | return VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; |
| 224 | } |
| 225 | |
| 226 | VkAccessFlags GrVkMemory::LayoutToSrcAccessMask(const VkImageLayout layout) { |
| 227 | // Currently we assume we will never being doing any explict shader writes (this doesn't include |
| 228 | // color attachment or depth/stencil writes). So we will ignore the |
| 229 | // VK_MEMORY_OUTPUT_SHADER_WRITE_BIT. |
| 230 | |
| 231 | // We can only directly access the host memory if we are in preinitialized or general layout, |
| 232 | // and the image is linear. |
| 233 | // TODO: Add check for linear here so we are not always adding host to general, and we should |
| 234 | // only be in preinitialized if we are linear |
| 235 | VkAccessFlags flags = 0;; |
| 236 | if (VK_IMAGE_LAYOUT_GENERAL == layout) { |
| 237 | flags = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | |
egdaniel | 19ff103 | 2016-08-31 10:13:08 -0700 | [diff] [blame] | 238 | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | |
| 239 | VK_ACCESS_TRANSFER_WRITE_BIT | |
| 240 | VK_ACCESS_TRANSFER_READ_BIT | |
| 241 | VK_ACCESS_SHADER_READ_BIT | |
| 242 | VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 243 | } else if (VK_IMAGE_LAYOUT_PREINITIALIZED == layout) { |
egdaniel | c2fde8b | 2016-06-24 10:29:02 -0700 | [diff] [blame] | 244 | flags = VK_ACCESS_HOST_WRITE_BIT; |
Greg Daniel | 164a9f0 | 2016-02-22 09:56:40 -0500 | [diff] [blame] | 245 | } else if (VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL == layout) { |
| 246 | flags = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; |
| 247 | } else if (VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL == layout) { |
| 248 | flags = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; |
| 249 | } else if (VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL == layout) { |
| 250 | flags = VK_ACCESS_TRANSFER_WRITE_BIT; |
| 251 | } else if (VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL == layout) { |
| 252 | flags = VK_ACCESS_TRANSFER_READ_BIT; |
| 253 | } else if (VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL == layout) { |
| 254 | flags = VK_ACCESS_SHADER_READ_BIT; |
| 255 | } |
| 256 | return flags; |
| 257 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 258 | |
jvanverth | 9d54afc | 2016-09-20 09:20:03 -0700 | [diff] [blame^] | 259 | void GrVkMemory::FlushMappedAlloc(const GrVkGpu* gpu, const GrVkAlloc& alloc) { |
| 260 | if (alloc.fFlags & GrVkAlloc::kNoncoherent_Flag) { |
| 261 | VkMappedMemoryRange mappedMemoryRange; |
| 262 | memset(&mappedMemoryRange, 0, sizeof(VkMappedMemoryRange)); |
| 263 | mappedMemoryRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; |
| 264 | mappedMemoryRange.memory = alloc.fMemory; |
| 265 | mappedMemoryRange.offset = alloc.fOffset; |
| 266 | mappedMemoryRange.size = alloc.fSize; |
| 267 | GR_VK_CALL(gpu->vkInterface(), FlushMappedMemoryRanges(gpu->device(), |
| 268 | 1, &mappedMemoryRange)); |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | void GrVkMemory::InvalidateMappedAlloc(const GrVkGpu* gpu, const GrVkAlloc& alloc) { |
| 273 | if (alloc.fFlags & GrVkAlloc::kNoncoherent_Flag) { |
| 274 | VkMappedMemoryRange mappedMemoryRange; |
| 275 | memset(&mappedMemoryRange, 0, sizeof(VkMappedMemoryRange)); |
| 276 | mappedMemoryRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; |
| 277 | mappedMemoryRange.memory = alloc.fMemory; |
| 278 | mappedMemoryRange.offset = alloc.fOffset; |
| 279 | mappedMemoryRange.size = alloc.fSize; |
| 280 | GR_VK_CALL(gpu->vkInterface(), InvalidateMappedMemoryRanges(gpu->device(), |
| 281 | 1, &mappedMemoryRange)); |
| 282 | } |
| 283 | } |
| 284 | |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 285 | bool GrVkFreeListAlloc::alloc(VkDeviceSize requestedSize, |
| 286 | VkDeviceSize* allocOffset, VkDeviceSize* allocSize) { |
| 287 | VkDeviceSize alignedSize = align_size(requestedSize, fAlignment); |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 288 | |
| 289 | // find the smallest block big enough for our allocation |
| 290 | FreeList::Iter iter = fFreeList.headIter(); |
| 291 | FreeList::Iter bestFitIter; |
| 292 | VkDeviceSize bestFitSize = fSize + 1; |
| 293 | VkDeviceSize secondLargestSize = 0; |
| 294 | VkDeviceSize secondLargestOffset = 0; |
| 295 | while (iter.get()) { |
| 296 | Block* block = iter.get(); |
| 297 | // need to adjust size to match desired alignment |
| 298 | SkASSERT(align_size(block->fOffset, fAlignment) - block->fOffset == 0); |
| 299 | if (block->fSize >= alignedSize && block->fSize < bestFitSize) { |
| 300 | bestFitIter = iter; |
| 301 | bestFitSize = block->fSize; |
| 302 | } |
| 303 | if (secondLargestSize < block->fSize && block->fOffset != fLargestBlockOffset) { |
| 304 | secondLargestSize = block->fSize; |
| 305 | secondLargestOffset = block->fOffset; |
| 306 | } |
| 307 | iter.next(); |
| 308 | } |
| 309 | SkASSERT(secondLargestSize <= fLargestBlockSize); |
| 310 | |
| 311 | Block* bestFit = bestFitIter.get(); |
| 312 | if (bestFit) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 313 | SkASSERT(align_size(bestFit->fOffset, fAlignment) == bestFit->fOffset); |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 314 | *allocOffset = bestFit->fOffset; |
| 315 | *allocSize = alignedSize; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 316 | // adjust or remove current block |
| 317 | VkDeviceSize originalBestFitOffset = bestFit->fOffset; |
| 318 | if (bestFit->fSize > alignedSize) { |
| 319 | bestFit->fOffset += alignedSize; |
| 320 | bestFit->fSize -= alignedSize; |
| 321 | if (fLargestBlockOffset == originalBestFitOffset) { |
| 322 | if (bestFit->fSize >= secondLargestSize) { |
| 323 | fLargestBlockSize = bestFit->fSize; |
| 324 | fLargestBlockOffset = bestFit->fOffset; |
| 325 | } else { |
| 326 | fLargestBlockSize = secondLargestSize; |
| 327 | fLargestBlockOffset = secondLargestOffset; |
| 328 | } |
| 329 | } |
| 330 | #ifdef SK_DEBUG |
| 331 | VkDeviceSize largestSize = 0; |
| 332 | iter = fFreeList.headIter(); |
| 333 | while (iter.get()) { |
| 334 | Block* block = iter.get(); |
| 335 | if (largestSize < block->fSize) { |
| 336 | largestSize = block->fSize; |
| 337 | } |
| 338 | iter.next(); |
| 339 | } |
caryclark | d656200 | 2016-07-27 12:02:07 -0700 | [diff] [blame] | 340 | SkASSERT(largestSize == fLargestBlockSize); |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 341 | #endif |
| 342 | } else { |
| 343 | SkASSERT(bestFit->fSize == alignedSize); |
| 344 | if (fLargestBlockOffset == originalBestFitOffset) { |
| 345 | fLargestBlockSize = secondLargestSize; |
| 346 | fLargestBlockOffset = secondLargestOffset; |
| 347 | } |
| 348 | fFreeList.remove(bestFit); |
| 349 | #ifdef SK_DEBUG |
| 350 | VkDeviceSize largestSize = 0; |
| 351 | iter = fFreeList.headIter(); |
| 352 | while (iter.get()) { |
| 353 | Block* block = iter.get(); |
| 354 | if (largestSize < block->fSize) { |
| 355 | largestSize = block->fSize; |
| 356 | } |
| 357 | iter.next(); |
| 358 | } |
| 359 | SkASSERT(largestSize == fLargestBlockSize); |
| 360 | #endif |
| 361 | } |
| 362 | fFreeSize -= alignedSize; |
egdaniel | 6e46eea | 2016-07-07 08:12:33 -0700 | [diff] [blame] | 363 | SkASSERT(*allocSize > 0); |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 364 | |
| 365 | return true; |
| 366 | } |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 367 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 368 | SkDebugf("Can't allocate %d bytes, %d bytes available, largest free block %d\n", alignedSize, fFreeSize, fLargestBlockSize); |
| 369 | |
| 370 | return false; |
| 371 | } |
| 372 | |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 373 | void GrVkFreeListAlloc::free(VkDeviceSize allocOffset, VkDeviceSize allocSize) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 374 | // find the block right after this allocation |
| 375 | FreeList::Iter iter = fFreeList.headIter(); |
jvanverth | d6f8034 | 2016-06-16 04:42:30 -0700 | [diff] [blame] | 376 | FreeList::Iter prev; |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 377 | while (iter.get() && iter.get()->fOffset < allocOffset) { |
jvanverth | d6f8034 | 2016-06-16 04:42:30 -0700 | [diff] [blame] | 378 | prev = iter; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 379 | iter.next(); |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 380 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 381 | // we have four cases: |
| 382 | // we exactly follow the previous one |
| 383 | Block* block; |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 384 | if (prev.get() && prev.get()->fOffset + prev.get()->fSize == allocOffset) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 385 | block = prev.get(); |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 386 | block->fSize += allocSize; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 387 | if (block->fOffset == fLargestBlockOffset) { |
| 388 | fLargestBlockSize = block->fSize; |
| 389 | } |
| 390 | // and additionally we may exactly precede the next one |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 391 | if (iter.get() && iter.get()->fOffset == allocOffset + allocSize) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 392 | block->fSize += iter.get()->fSize; |
| 393 | if (iter.get()->fOffset == fLargestBlockOffset) { |
| 394 | fLargestBlockOffset = block->fOffset; |
| 395 | fLargestBlockSize = block->fSize; |
| 396 | } |
| 397 | fFreeList.remove(iter.get()); |
| 398 | } |
| 399 | // or we only exactly proceed the next one |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 400 | } else if (iter.get() && iter.get()->fOffset == allocOffset + allocSize) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 401 | block = iter.get(); |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 402 | block->fSize += allocSize; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 403 | if (block->fOffset == fLargestBlockOffset) { |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 404 | fLargestBlockOffset = allocOffset; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 405 | fLargestBlockSize = block->fSize; |
| 406 | } |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 407 | block->fOffset = allocOffset; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 408 | // or we fall somewhere in between, with gaps |
| 409 | } else { |
| 410 | block = fFreeList.addBefore(iter); |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 411 | block->fOffset = allocOffset; |
| 412 | block->fSize = allocSize; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 413 | } |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 414 | fFreeSize += allocSize; |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 415 | if (block->fSize > fLargestBlockSize) { |
| 416 | fLargestBlockSize = block->fSize; |
| 417 | fLargestBlockOffset = block->fOffset; |
| 418 | } |
| 419 | |
| 420 | #ifdef SK_DEBUG |
| 421 | VkDeviceSize largestSize = 0; |
| 422 | iter = fFreeList.headIter(); |
| 423 | while (iter.get()) { |
| 424 | Block* block = iter.get(); |
| 425 | if (largestSize < block->fSize) { |
| 426 | largestSize = block->fSize; |
| 427 | } |
| 428 | iter.next(); |
| 429 | } |
| 430 | SkASSERT(fLargestBlockSize == largestSize); |
| 431 | #endif |
| 432 | } |
| 433 | |
jvanverth | 82356cc | 2016-07-07 07:16:42 -0700 | [diff] [blame] | 434 | GrVkSubHeap::GrVkSubHeap(const GrVkGpu* gpu, uint32_t memoryTypeIndex, |
| 435 | VkDeviceSize size, VkDeviceSize alignment) |
| 436 | : INHERITED(size, alignment) |
| 437 | , fGpu(gpu) |
| 438 | , fMemoryTypeIndex(memoryTypeIndex) { |
| 439 | |
| 440 | VkMemoryAllocateInfo allocInfo = { |
| 441 | VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // sType |
| 442 | NULL, // pNext |
| 443 | size, // allocationSize |
| 444 | memoryTypeIndex, // memoryTypeIndex |
| 445 | }; |
| 446 | |
| 447 | VkResult err = GR_VK_CALL(gpu->vkInterface(), AllocateMemory(gpu->device(), |
| 448 | &allocInfo, |
| 449 | nullptr, |
| 450 | &fAlloc)); |
| 451 | if (VK_SUCCESS != err) { |
| 452 | this->reset(); |
| 453 | } |
| 454 | } |
| 455 | |
| 456 | GrVkSubHeap::~GrVkSubHeap() { |
| 457 | const GrVkInterface* iface = fGpu->vkInterface(); |
| 458 | GR_VK_CALL(iface, FreeMemory(fGpu->device(), fAlloc, nullptr)); |
| 459 | } |
| 460 | |
| 461 | bool GrVkSubHeap::alloc(VkDeviceSize size, GrVkAlloc* alloc) { |
| 462 | alloc->fMemory = fAlloc; |
| 463 | return INHERITED::alloc(size, &alloc->fOffset, &alloc->fSize); |
| 464 | } |
| 465 | |
| 466 | void GrVkSubHeap::free(const GrVkAlloc& alloc) { |
| 467 | SkASSERT(alloc.fMemory == fAlloc); |
| 468 | |
| 469 | INHERITED::free(alloc.fOffset, alloc.fSize); |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 470 | } |
| 471 | |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 472 | bool GrVkHeap::subAlloc(VkDeviceSize size, VkDeviceSize alignment, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 473 | uint32_t memoryTypeIndex, GrVkAlloc* alloc) { |
| 474 | VkDeviceSize alignedSize = align_size(size, alignment); |
| 475 | |
jvanverth | 6dc3af4 | 2016-06-16 14:05:09 -0700 | [diff] [blame] | 476 | // if requested is larger than our subheap allocation, just alloc directly |
| 477 | if (alignedSize > fSubHeapSize) { |
| 478 | VkMemoryAllocateInfo allocInfo = { |
| 479 | VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // sType |
| 480 | NULL, // pNext |
| 481 | size, // allocationSize |
| 482 | memoryTypeIndex, // memoryTypeIndex |
| 483 | }; |
| 484 | |
| 485 | VkResult err = GR_VK_CALL(fGpu->vkInterface(), AllocateMemory(fGpu->device(), |
| 486 | &allocInfo, |
| 487 | nullptr, |
| 488 | &alloc->fMemory)); |
| 489 | if (VK_SUCCESS != err) { |
| 490 | return false; |
| 491 | } |
| 492 | alloc->fOffset = 0; |
| 493 | alloc->fSize = 0; // hint that this is not a subheap allocation |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 494 | |
jvanverth | 6dc3af4 | 2016-06-16 14:05:09 -0700 | [diff] [blame] | 495 | return true; |
| 496 | } |
| 497 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 498 | // first try to find a subheap that fits our allocation request |
| 499 | int bestFitIndex = -1; |
| 500 | VkDeviceSize bestFitSize = 0x7FFFFFFF; |
| 501 | for (auto i = 0; i < fSubHeaps.count(); ++i) { |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 502 | if (fSubHeaps[i]->memoryTypeIndex() == memoryTypeIndex && |
| 503 | fSubHeaps[i]->alignment() == alignment) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 504 | VkDeviceSize heapSize = fSubHeaps[i]->largestBlockSize(); |
jvanverth | d6f8034 | 2016-06-16 04:42:30 -0700 | [diff] [blame] | 505 | if (heapSize >= alignedSize && heapSize < bestFitSize) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 506 | bestFitIndex = i; |
| 507 | bestFitSize = heapSize; |
| 508 | } |
| 509 | } |
| 510 | } |
| 511 | |
| 512 | if (bestFitIndex >= 0) { |
| 513 | SkASSERT(fSubHeaps[bestFitIndex]->alignment() == alignment); |
| 514 | if (fSubHeaps[bestFitIndex]->alloc(size, alloc)) { |
| 515 | fUsedSize += alloc->fSize; |
| 516 | return true; |
| 517 | } |
| 518 | return false; |
jvanverth | 6dc3af4 | 2016-06-16 14:05:09 -0700 | [diff] [blame] | 519 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 520 | |
| 521 | // need to allocate a new subheap |
| 522 | SkAutoTDelete<GrVkSubHeap>& subHeap = fSubHeaps.push_back(); |
| 523 | subHeap.reset(new GrVkSubHeap(fGpu, memoryTypeIndex, fSubHeapSize, alignment)); |
jvanverth | 6dc3af4 | 2016-06-16 14:05:09 -0700 | [diff] [blame] | 524 | // try to recover from failed allocation by only allocating what we need |
| 525 | if (subHeap->size() == 0) { |
| 526 | VkDeviceSize alignedSize = align_size(size, alignment); |
| 527 | subHeap.reset(new GrVkSubHeap(fGpu, memoryTypeIndex, alignedSize, alignment)); |
| 528 | if (subHeap->size() == 0) { |
| 529 | return false; |
| 530 | } |
| 531 | } |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 532 | fAllocSize += fSubHeapSize; |
| 533 | if (subHeap->alloc(size, alloc)) { |
| 534 | fUsedSize += alloc->fSize; |
| 535 | return true; |
| 536 | } |
| 537 | |
| 538 | return false; |
| 539 | } |
| 540 | |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 541 | bool GrVkHeap::singleAlloc(VkDeviceSize size, VkDeviceSize alignment, |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 542 | uint32_t memoryTypeIndex, GrVkAlloc* alloc) { |
| 543 | VkDeviceSize alignedSize = align_size(size, alignment); |
| 544 | |
| 545 | // first try to find an unallocated subheap that fits our allocation request |
| 546 | int bestFitIndex = -1; |
| 547 | VkDeviceSize bestFitSize = 0x7FFFFFFF; |
| 548 | for (auto i = 0; i < fSubHeaps.count(); ++i) { |
egdaniel | 6e90d42 | 2016-08-10 08:29:53 -0700 | [diff] [blame] | 549 | if (fSubHeaps[i]->memoryTypeIndex() == memoryTypeIndex && |
| 550 | fSubHeaps[i]->alignment() == alignment && |
| 551 | fSubHeaps[i]->unallocated()) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 552 | VkDeviceSize heapSize = fSubHeaps[i]->size(); |
jvanverth | d6f8034 | 2016-06-16 04:42:30 -0700 | [diff] [blame] | 553 | if (heapSize >= alignedSize && heapSize < bestFitSize) { |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 554 | bestFitIndex = i; |
| 555 | bestFitSize = heapSize; |
| 556 | } |
| 557 | } |
| 558 | } |
| 559 | |
| 560 | if (bestFitIndex >= 0) { |
| 561 | SkASSERT(fSubHeaps[bestFitIndex]->alignment() == alignment); |
| 562 | if (fSubHeaps[bestFitIndex]->alloc(size, alloc)) { |
| 563 | fUsedSize += alloc->fSize; |
| 564 | return true; |
| 565 | } |
| 566 | return false; |
| 567 | } |
| 568 | |
| 569 | // need to allocate a new subheap |
| 570 | SkAutoTDelete<GrVkSubHeap>& subHeap = fSubHeaps.push_back(); |
| 571 | subHeap.reset(new GrVkSubHeap(fGpu, memoryTypeIndex, alignedSize, alignment)); |
| 572 | fAllocSize += alignedSize; |
| 573 | if (subHeap->alloc(size, alloc)) { |
| 574 | fUsedSize += alloc->fSize; |
| 575 | return true; |
| 576 | } |
| 577 | |
| 578 | return false; |
| 579 | } |
| 580 | |
| 581 | bool GrVkHeap::free(const GrVkAlloc& alloc) { |
jvanverth | 6dc3af4 | 2016-06-16 14:05:09 -0700 | [diff] [blame] | 582 | // a size of 0 means we're using the system heap |
| 583 | if (0 == alloc.fSize) { |
| 584 | const GrVkInterface* iface = fGpu->vkInterface(); |
| 585 | GR_VK_CALL(iface, FreeMemory(fGpu->device(), alloc.fMemory, nullptr)); |
| 586 | return true; |
| 587 | } |
| 588 | |
jvanverth | 6b6ffc4 | 2016-06-13 14:28:07 -0700 | [diff] [blame] | 589 | for (auto i = 0; i < fSubHeaps.count(); ++i) { |
| 590 | if (fSubHeaps[i]->memory() == alloc.fMemory) { |
| 591 | fSubHeaps[i]->free(alloc); |
| 592 | fUsedSize -= alloc.fSize; |
| 593 | return true; |
| 594 | } |
| 595 | } |
| 596 | |
| 597 | return false; |
| 598 | } |
| 599 | |
| 600 | |