Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. |
| 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice shall be included in |
| 12 | * all copies or substantial portions of the Software. |
| 13 | * |
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 | * OTHER DEALINGS IN THE SOFTWARE. |
| 21 | */ |
| 22 | |
| 23 | #include "amdgpu_amdkfd.h" |
Jammy Zhou | 2f7d10b | 2015-07-22 11:29:01 +0800 | [diff] [blame] | 24 | #include "amd_shared.h" |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 25 | #include <drm/drmP.h> |
| 26 | #include "amdgpu.h" |
| 27 | #include <linux/module.h> |
| 28 | |
| 29 | const struct kfd2kgd_calls *kfd2kgd; |
| 30 | const struct kgd2kfd_calls *kgd2kfd; |
| 31 | bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); |
| 32 | |
Oded Gabbay | efb1c65 | 2016-02-09 13:30:12 +0200 | [diff] [blame] | 33 | int amdgpu_amdkfd_init(void) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 34 | { |
Oded Gabbay | efb1c65 | 2016-02-09 13:30:12 +0200 | [diff] [blame] | 35 | int ret; |
| 36 | |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 37 | #if defined(CONFIG_HSA_AMD_MODULE) |
Oded Gabbay | efb1c65 | 2016-02-09 13:30:12 +0200 | [diff] [blame] | 38 | int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 39 | |
| 40 | kgd2kfd_init_p = symbol_request(kgd2kfd_init); |
| 41 | |
| 42 | if (kgd2kfd_init_p == NULL) |
Oded Gabbay | efb1c65 | 2016-02-09 13:30:12 +0200 | [diff] [blame] | 43 | return -ENOENT; |
| 44 | |
| 45 | ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd); |
| 46 | if (ret) { |
| 47 | symbol_put(kgd2kfd_init); |
| 48 | kgd2kfd = NULL; |
| 49 | } |
| 50 | |
| 51 | #elif defined(CONFIG_HSA_AMD) |
| 52 | ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd); |
| 53 | if (ret) |
| 54 | kgd2kfd = NULL; |
| 55 | |
| 56 | #else |
| 57 | ret = -ENOENT; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 58 | #endif |
Oded Gabbay | efb1c65 | 2016-02-09 13:30:12 +0200 | [diff] [blame] | 59 | |
| 60 | return ret; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 61 | } |
| 62 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 63 | bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 64 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 65 | switch (adev->asic_type) { |
Alex Deucher | 41548ef | 2015-07-31 17:20:14 -0400 | [diff] [blame] | 66 | #ifdef CONFIG_DRM_AMDGPU_CIK |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 67 | case CHIP_KAVERI: |
Oded Gabbay | 32c22e9 | 2015-06-12 21:38:22 +0300 | [diff] [blame] | 68 | kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions(); |
| 69 | break; |
Alex Deucher | 41548ef | 2015-07-31 17:20:14 -0400 | [diff] [blame] | 70 | #endif |
Ben Goz | ff758a1 | 2014-10-07 14:43:07 +0300 | [diff] [blame] | 71 | case CHIP_CARRIZO: |
| 72 | kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); |
| 73 | break; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 74 | default: |
| 75 | return false; |
| 76 | } |
| 77 | |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 78 | return true; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 79 | } |
| 80 | |
| 81 | void amdgpu_amdkfd_fini(void) |
| 82 | { |
| 83 | if (kgd2kfd) { |
| 84 | kgd2kfd->exit(); |
| 85 | symbol_put(kgd2kfd_init); |
| 86 | } |
| 87 | } |
| 88 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 89 | void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 90 | { |
| 91 | if (kgd2kfd) |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 92 | adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev, |
| 93 | adev->pdev, kfd2kgd); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 94 | } |
| 95 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 96 | void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 97 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 98 | if (adev->kfd) { |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 99 | struct kgd2kfd_shared_resources gpu_resources = { |
| 100 | .compute_vmid_bitmap = 0xFF00, |
| 101 | |
| 102 | .first_compute_pipe = 1, |
| 103 | .compute_pipe_count = 4 - 1, |
| 104 | }; |
| 105 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 106 | amdgpu_doorbell_get_kfd_info(adev, |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 107 | &gpu_resources.doorbell_physical_address, |
| 108 | &gpu_resources.doorbell_aperture_size, |
| 109 | &gpu_resources.doorbell_start_offset); |
| 110 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 111 | kgd2kfd->device_init(adev->kfd, &gpu_resources); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 112 | } |
| 113 | } |
| 114 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 115 | void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 116 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 117 | if (adev->kfd) { |
| 118 | kgd2kfd->device_exit(adev->kfd); |
| 119 | adev->kfd = NULL; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 120 | } |
| 121 | } |
| 122 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 123 | void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 124 | const void *ih_ring_entry) |
| 125 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 126 | if (adev->kfd) |
| 127 | kgd2kfd->interrupt(adev->kfd, ih_ring_entry); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 128 | } |
| 129 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 130 | void amdgpu_amdkfd_suspend(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 131 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 132 | if (adev->kfd) |
| 133 | kgd2kfd->suspend(adev->kfd); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 134 | } |
| 135 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 136 | int amdgpu_amdkfd_resume(struct amdgpu_device *adev) |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 137 | { |
| 138 | int r = 0; |
| 139 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 140 | if (adev->kfd) |
| 141 | r = kgd2kfd->resume(adev->kfd); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 142 | |
| 143 | return r; |
| 144 | } |
| 145 | |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 146 | int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, |
| 147 | void **mem_obj, uint64_t *gpu_addr, |
| 148 | void **cpu_ptr) |
| 149 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 150 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 151 | struct kgd_mem **mem = (struct kgd_mem **) mem_obj; |
| 152 | int r; |
| 153 | |
| 154 | BUG_ON(kgd == NULL); |
| 155 | BUG_ON(gpu_addr == NULL); |
| 156 | BUG_ON(cpu_ptr == NULL); |
| 157 | |
| 158 | *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL); |
| 159 | if ((*mem) == NULL) |
| 160 | return -ENOMEM; |
| 161 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 162 | r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT, |
Christian König | 72d7668 | 2015-09-03 17:34:59 +0200 | [diff] [blame] | 163 | AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 164 | if (r) { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 165 | dev_err(adev->dev, |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 166 | "failed to allocate BO for amdkfd (%d)\n", r); |
| 167 | return r; |
| 168 | } |
| 169 | |
| 170 | /* map the buffer */ |
| 171 | r = amdgpu_bo_reserve((*mem)->bo, true); |
| 172 | if (r) { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 173 | dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 174 | goto allocate_mem_reserve_bo_failed; |
| 175 | } |
| 176 | |
| 177 | r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT, |
| 178 | &(*mem)->gpu_addr); |
| 179 | if (r) { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 180 | dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 181 | goto allocate_mem_pin_bo_failed; |
| 182 | } |
| 183 | *gpu_addr = (*mem)->gpu_addr; |
| 184 | |
| 185 | r = amdgpu_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr); |
| 186 | if (r) { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 187 | dev_err(adev->dev, |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 188 | "(%d) failed to map bo to kernel for amdkfd\n", r); |
| 189 | goto allocate_mem_kmap_bo_failed; |
| 190 | } |
| 191 | *cpu_ptr = (*mem)->cpu_ptr; |
| 192 | |
| 193 | amdgpu_bo_unreserve((*mem)->bo); |
| 194 | |
| 195 | return 0; |
| 196 | |
| 197 | allocate_mem_kmap_bo_failed: |
| 198 | amdgpu_bo_unpin((*mem)->bo); |
| 199 | allocate_mem_pin_bo_failed: |
| 200 | amdgpu_bo_unreserve((*mem)->bo); |
| 201 | allocate_mem_reserve_bo_failed: |
| 202 | amdgpu_bo_unref(&(*mem)->bo); |
| 203 | |
| 204 | return r; |
| 205 | } |
| 206 | |
| 207 | void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) |
| 208 | { |
| 209 | struct kgd_mem *mem = (struct kgd_mem *) mem_obj; |
| 210 | |
| 211 | BUG_ON(mem == NULL); |
| 212 | |
| 213 | amdgpu_bo_reserve(mem->bo, true); |
| 214 | amdgpu_bo_kunmap(mem->bo); |
| 215 | amdgpu_bo_unpin(mem->bo); |
| 216 | amdgpu_bo_unreserve(mem->bo); |
| 217 | amdgpu_bo_unref(&(mem->bo)); |
| 218 | kfree(mem); |
| 219 | } |
| 220 | |
| 221 | uint64_t get_vmem_size(struct kgd_dev *kgd) |
| 222 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 223 | struct amdgpu_device *adev = |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 224 | (struct amdgpu_device *)kgd; |
| 225 | |
| 226 | BUG_ON(kgd == NULL); |
| 227 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 228 | return adev->mc.real_vram_size; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 229 | } |
| 230 | |
| 231 | uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) |
| 232 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 233 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 234 | |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 235 | if (adev->gfx.funcs->get_gpu_clock_counter) |
| 236 | return adev->gfx.funcs->get_gpu_clock_counter(adev); |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 237 | return 0; |
| 238 | } |
| 239 | |
| 240 | uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) |
| 241 | { |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 242 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 243 | |
| 244 | /* The sclk is in quantas of 10kHz */ |
Andres Rodriguez | dc102c4 | 2017-02-01 17:02:13 -0500 | [diff] [blame^] | 245 | return adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100; |
Oded Gabbay | 130e037 | 2015-06-12 21:35:14 +0300 | [diff] [blame] | 246 | } |