blob: 3317d1536f4fc352247756e3c650d72c9236916b [file] [log] [blame]
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
Chunming Zhou0875dc92016-06-12 15:41:58 +080028#include <linux/kthread.h>
Alex Deucherd38ceaf2015-04-20 16:55:21 -040029#include <linux/console.h>
30#include <linux/slab.h>
Alex Deucherd38ceaf2015-04-20 16:55:21 -040031#include <drm/drmP.h>
32#include <drm/drm_crtc_helper.h>
Harry Wentland45622362017-09-12 15:58:20 -040033#include <drm/drm_atomic_helper.h>
Alex Deucherd38ceaf2015-04-20 16:55:21 -040034#include <drm/amdgpu_drm.h>
35#include <linux/vgaarb.h>
36#include <linux/vga_switcheroo.h>
37#include <linux/efi.h>
38#include "amdgpu.h"
Tom St Denisf4b373f2016-05-31 08:02:27 -040039#include "amdgpu_trace.h"
Alex Deucherd38ceaf2015-04-20 16:55:21 -040040#include "amdgpu_i2c.h"
41#include "atom.h"
42#include "amdgpu_atombios.h"
Alex Deuchera5bde2f2016-09-23 16:23:41 -040043#include "amdgpu_atomfirmware.h"
Alex Deucherd0dd7f02015-11-11 19:45:06 -050044#include "amd_pcie.h"
Ken Wang33f34802016-01-21 17:29:41 +080045#ifdef CONFIG_DRM_AMDGPU_SI
46#include "si.h"
47#endif
Alex Deuchera2e73f52015-04-20 17:09:27 -040048#ifdef CONFIG_DRM_AMDGPU_CIK
49#include "cik.h"
50#endif
Alex Deucheraaa36a92015-04-20 17:31:14 -040051#include "vi.h"
Ken Wang460826e2017-03-06 14:53:16 -050052#include "soc15.h"
Alex Deucherd38ceaf2015-04-20 16:55:21 -040053#include "bif/bif_4_1_d.h"
Emily Deng9accf2f2016-08-10 16:01:25 +080054#include <linux/pci.h>
Monk Liubec86372016-09-14 19:38:08 +080055#include <linux/firmware.h>
Gavin Wan89041942017-06-23 13:55:15 -040056#include "amdgpu_vf_error.h"
Alex Deucherd38ceaf2015-04-20 16:55:21 -040057
Yong Zhaoba997702015-11-09 17:21:45 -050058#include "amdgpu_amdkfd.h"
Rex Zhud2f52ac2017-09-22 17:47:27 +080059#include "amdgpu_pm.h"
Alex Deucherd38ceaf2015-04-20 16:55:21 -040060
Alex Deuchere2a75f82017-04-27 16:58:01 -040061MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
Alex Deucher3f76dce2017-09-01 16:20:53 -040062MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
Alex Deucher2d2e5e72017-05-09 12:27:35 -040063MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
Alex Deuchere2a75f82017-04-27 16:58:01 -040064
Shirish S2dc80b02017-05-25 10:05:25 +053065#define AMDGPU_RESUME_MS 2000
66
Alex Deucherd38ceaf2015-04-20 16:55:21 -040067static const char *amdgpu_asic_name[] = {
Ken Wangda69c1612016-01-21 19:08:55 +080068 "TAHITI",
69 "PITCAIRN",
70 "VERDE",
71 "OLAND",
72 "HAINAN",
Alex Deucherd38ceaf2015-04-20 16:55:21 -040073 "BONAIRE",
74 "KAVERI",
75 "KABINI",
76 "HAWAII",
77 "MULLINS",
78 "TOPAZ",
79 "TONGA",
David Zhang48299f92015-07-08 01:05:16 +080080 "FIJI",
Alex Deucherd38ceaf2015-04-20 16:55:21 -040081 "CARRIZO",
Samuel Li139f4912015-10-08 14:50:27 -040082 "STONEY",
Flora Cui2cc0c0b2016-03-14 18:33:29 -040083 "POLARIS10",
84 "POLARIS11",
Junwei Zhangc4642a42016-12-14 15:32:28 -050085 "POLARIS12",
Leo Liu48ff1082017-11-09 13:18:24 -050086 "VEGAM",
Ken Wangd4196f02016-03-09 09:28:32 +080087 "VEGA10",
Feifei Xu8fab8062017-10-19 17:04:54 +080088 "VEGA12",
Feifei Xu956fcdd2018-04-20 12:27:54 +080089 "VEGA20",
Chunming Zhou2ca8a5d2016-12-07 17:31:19 +080090 "RAVEN",
Alex Deucherd38ceaf2015-04-20 16:55:21 -040091 "LAST",
92};
93
Alex Deucher5494d862018-03-09 15:14:11 -050094static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
95
Alex Deuchere3ecdff2018-03-15 17:39:45 -050096/**
97 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
98 *
99 * @dev: drm_device pointer
100 *
101 * Returns true if the device is a dGPU with HG/PX power control,
102 * otherwise return false.
103 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400104bool amdgpu_device_is_px(struct drm_device *dev)
105{
106 struct amdgpu_device *adev = dev->dev_private;
107
Jammy Zhou2f7d10b2015-07-22 11:29:01 +0800108 if (adev->flags & AMD_IS_PX)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400109 return true;
110 return false;
111}
112
113/*
114 * MMIO register access helper functions.
115 */
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500116/**
117 * amdgpu_mm_rreg - read a memory mapped IO register
118 *
119 * @adev: amdgpu_device pointer
120 * @reg: dword aligned register offset
121 * @acc_flags: access flags which require special behavior
122 *
123 * Returns the 32 bit value from the offset specified.
124 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400125uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
Monk Liu15d72fd2017-01-25 15:07:40 +0800126 uint32_t acc_flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400127{
Tom St Denisf4b373f2016-05-31 08:02:27 -0400128 uint32_t ret;
129
pding43ca8ef2017-10-13 15:38:35 +0800130 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
Xiangliang Yubc992ba2017-01-12 14:29:34 +0800131 return amdgpu_virt_kiq_rreg(adev, reg);
Xiangliang Yubc992ba2017-01-12 14:29:34 +0800132
Monk Liu15d72fd2017-01-25 15:07:40 +0800133 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
Tom St Denisf4b373f2016-05-31 08:02:27 -0400134 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400135 else {
136 unsigned long flags;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400137
138 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
139 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
140 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
141 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400142 }
Tom St Denisf4b373f2016-05-31 08:02:27 -0400143 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
144 return ret;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400145}
146
Monk Liu421a2a32018-01-04 18:13:20 +0800147/*
148 * MMIO register read with bytes helper functions
149 * @offset:bytes offset from MMIO start
150 *
151*/
152
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500153/**
154 * amdgpu_mm_rreg8 - read a memory mapped IO register
155 *
156 * @adev: amdgpu_device pointer
157 * @offset: byte aligned register offset
158 *
159 * Returns the 8 bit value from the offset specified.
160 */
Monk Liu421a2a32018-01-04 18:13:20 +0800161uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
162 if (offset < adev->rmmio_size)
163 return (readb(adev->rmmio + offset));
164 BUG();
165}
166
167/*
168 * MMIO register write with bytes helper functions
169 * @offset:bytes offset from MMIO start
170 * @value: the value want to be written to the register
171 *
172*/
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500173/**
174 * amdgpu_mm_wreg8 - read a memory mapped IO register
175 *
176 * @adev: amdgpu_device pointer
177 * @offset: byte aligned register offset
178 * @value: 8 bit value to write
179 *
180 * Writes the value specified to the offset specified.
181 */
Monk Liu421a2a32018-01-04 18:13:20 +0800182void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
183 if (offset < adev->rmmio_size)
184 writeb(value, adev->rmmio + offset);
185 else
186 BUG();
187}
188
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500189/**
190 * amdgpu_mm_wreg - write to a memory mapped IO register
191 *
192 * @adev: amdgpu_device pointer
193 * @reg: dword aligned register offset
194 * @v: 32 bit value to write to the register
195 * @acc_flags: access flags which require special behavior
196 *
197 * Writes the value specified to the offset specified.
198 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400199void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
Monk Liu15d72fd2017-01-25 15:07:40 +0800200 uint32_t acc_flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400201{
Tom St Denisf4b373f2016-05-31 08:02:27 -0400202 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
Monk Liu4e99a442016-03-31 13:26:59 +0800203
Ken Wang47ed4e12017-07-04 13:11:52 +0800204 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
205 adev->last_mm_index = v;
206 }
207
pding43ca8ef2017-10-13 15:38:35 +0800208 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
Xiangliang Yubc992ba2017-01-12 14:29:34 +0800209 return amdgpu_virt_kiq_wreg(adev, reg, v);
Xiangliang Yubc992ba2017-01-12 14:29:34 +0800210
Monk Liu15d72fd2017-01-25 15:07:40 +0800211 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400212 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
213 else {
214 unsigned long flags;
215
216 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
217 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
218 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
219 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
220 }
Ken Wang47ed4e12017-07-04 13:11:52 +0800221
222 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
223 udelay(500);
224 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400225}
226
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500227/**
228 * amdgpu_io_rreg - read an IO register
229 *
230 * @adev: amdgpu_device pointer
231 * @reg: dword aligned register offset
232 *
233 * Returns the 32 bit value from the offset specified.
234 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400235u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
236{
237 if ((reg * 4) < adev->rio_mem_size)
238 return ioread32(adev->rio_mem + (reg * 4));
239 else {
240 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
241 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
242 }
243}
244
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500245/**
246 * amdgpu_io_wreg - write to an IO register
247 *
248 * @adev: amdgpu_device pointer
249 * @reg: dword aligned register offset
250 * @v: 32 bit value to write to the register
251 *
252 * Writes the value specified to the offset specified.
253 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400254void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
255{
Ken Wang47ed4e12017-07-04 13:11:52 +0800256 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
257 adev->last_mm_index = v;
258 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400259
260 if ((reg * 4) < adev->rio_mem_size)
261 iowrite32(v, adev->rio_mem + (reg * 4));
262 else {
263 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
264 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
265 }
Ken Wang47ed4e12017-07-04 13:11:52 +0800266
267 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
268 udelay(500);
269 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400270}
271
272/**
273 * amdgpu_mm_rdoorbell - read a doorbell dword
274 *
275 * @adev: amdgpu_device pointer
276 * @index: doorbell index
277 *
278 * Returns the value in the doorbell aperture at the
279 * requested doorbell index (CIK).
280 */
281u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
282{
283 if (index < adev->doorbell.num_doorbells) {
284 return readl(adev->doorbell.ptr + index);
285 } else {
286 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
287 return 0;
288 }
289}
290
291/**
292 * amdgpu_mm_wdoorbell - write a doorbell dword
293 *
294 * @adev: amdgpu_device pointer
295 * @index: doorbell index
296 * @v: value to write
297 *
298 * Writes @v to the doorbell aperture at the
299 * requested doorbell index (CIK).
300 */
301void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
302{
303 if (index < adev->doorbell.num_doorbells) {
304 writel(v, adev->doorbell.ptr + index);
305 } else {
306 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
307 }
308}
309
310/**
Ken Wang832be402016-03-18 15:23:08 +0800311 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
312 *
313 * @adev: amdgpu_device pointer
314 * @index: doorbell index
315 *
316 * Returns the value in the doorbell aperture at the
317 * requested doorbell index (VEGA10+).
318 */
319u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
320{
321 if (index < adev->doorbell.num_doorbells) {
322 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
323 } else {
324 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
325 return 0;
326 }
327}
328
329/**
330 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
331 *
332 * @adev: amdgpu_device pointer
333 * @index: doorbell index
334 * @v: value to write
335 *
336 * Writes @v to the doorbell aperture at the
337 * requested doorbell index (VEGA10+).
338 */
339void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
340{
341 if (index < adev->doorbell.num_doorbells) {
342 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
343 } else {
344 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
345 }
346}
347
348/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400349 * amdgpu_invalid_rreg - dummy reg read function
350 *
351 * @adev: amdgpu device pointer
352 * @reg: offset of register
353 *
354 * Dummy register read function. Used for register blocks
355 * that certain asics don't have (all asics).
356 * Returns the value in the register.
357 */
358static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
359{
360 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
361 BUG();
362 return 0;
363}
364
365/**
366 * amdgpu_invalid_wreg - dummy reg write function
367 *
368 * @adev: amdgpu device pointer
369 * @reg: offset of register
370 * @v: value to write to the register
371 *
372 * Dummy register read function. Used for register blocks
373 * that certain asics don't have (all asics).
374 */
375static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
376{
377 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
378 reg, v);
379 BUG();
380}
381
382/**
383 * amdgpu_block_invalid_rreg - dummy reg read function
384 *
385 * @adev: amdgpu device pointer
386 * @block: offset of instance
387 * @reg: offset of register
388 *
389 * Dummy register read function. Used for register blocks
390 * that certain asics don't have (all asics).
391 * Returns the value in the register.
392 */
393static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
394 uint32_t block, uint32_t reg)
395{
396 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
397 reg, block);
398 BUG();
399 return 0;
400}
401
402/**
403 * amdgpu_block_invalid_wreg - dummy reg write function
404 *
405 * @adev: amdgpu device pointer
406 * @block: offset of instance
407 * @reg: offset of register
408 * @v: value to write to the register
409 *
410 * Dummy register read function. Used for register blocks
411 * that certain asics don't have (all asics).
412 */
413static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
414 uint32_t block,
415 uint32_t reg, uint32_t v)
416{
417 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
418 reg, block, v);
419 BUG();
420}
421
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500422/**
423 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
424 *
425 * @adev: amdgpu device pointer
426 *
427 * Allocates a scratch page of VRAM for use by various things in the
428 * driver.
429 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500430static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400431{
Christian Königa4a02772017-07-27 17:24:36 +0200432 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
433 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
434 &adev->vram_scratch.robj,
435 &adev->vram_scratch.gpu_addr,
436 (void **)&adev->vram_scratch.ptr);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400437}
438
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500439/**
440 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
441 *
442 * @adev: amdgpu device pointer
443 *
444 * Frees the VRAM scratch page.
445 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500446static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400447{
Christian König078af1a2017-07-27 17:43:00 +0200448 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400449}
450
451/**
Alex Deucher9c3f2b52017-12-14 16:20:19 -0500452 * amdgpu_device_program_register_sequence - program an array of registers.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400453 *
454 * @adev: amdgpu_device pointer
455 * @registers: pointer to the register array
456 * @array_size: size of the register array
457 *
458 * Programs an array or registers with and and or masks.
459 * This is a helper for setting golden registers.
460 */
Alex Deucher9c3f2b52017-12-14 16:20:19 -0500461void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
462 const u32 *registers,
463 const u32 array_size)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400464{
465 u32 tmp, reg, and_mask, or_mask;
466 int i;
467
468 if (array_size % 3)
469 return;
470
471 for (i = 0; i < array_size; i +=3) {
472 reg = registers[i + 0];
473 and_mask = registers[i + 1];
474 or_mask = registers[i + 2];
475
476 if (and_mask == 0xffffffff) {
477 tmp = or_mask;
478 } else {
479 tmp = RREG32(reg);
480 tmp &= ~and_mask;
481 tmp |= or_mask;
482 }
483 WREG32(reg, tmp);
484 }
485}
486
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500487/**
488 * amdgpu_device_pci_config_reset - reset the GPU
489 *
490 * @adev: amdgpu_device pointer
491 *
492 * Resets the GPU using the pci config reset sequence.
493 * Only applicable to asics prior to vega10.
494 */
Alex Deucher8111c382017-12-14 16:22:53 -0500495void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400496{
497 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
498}
499
500/*
501 * GPU doorbell aperture helpers function.
502 */
503/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500504 * amdgpu_device_doorbell_init - Init doorbell driver information.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400505 *
506 * @adev: amdgpu_device pointer
507 *
508 * Init doorbell driver information (CIK)
509 * Returns 0 on success, error on failure.
510 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500511static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400512{
Christian König705e5192017-06-08 11:15:16 +0200513 /* No doorbell on SI hardware generation */
514 if (adev->asic_type < CHIP_BONAIRE) {
515 adev->doorbell.base = 0;
516 adev->doorbell.size = 0;
517 adev->doorbell.num_doorbells = 0;
518 adev->doorbell.ptr = NULL;
519 return 0;
520 }
521
Christian Königd6895ad2017-02-28 10:36:43 +0100522 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
523 return -EINVAL;
524
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400525 /* doorbell bar mapping */
526 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
527 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
528
Christian Königedf600d2016-05-03 15:54:54 +0200529 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400530 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
531 if (adev->doorbell.num_doorbells == 0)
532 return -EINVAL;
533
Christian König8972e5d2017-03-06 13:34:57 +0100534 adev->doorbell.ptr = ioremap(adev->doorbell.base,
535 adev->doorbell.num_doorbells *
536 sizeof(u32));
537 if (adev->doorbell.ptr == NULL)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400538 return -ENOMEM;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400539
540 return 0;
541}
542
543/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500544 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400545 *
546 * @adev: amdgpu_device pointer
547 *
548 * Tear down doorbell driver information (CIK)
549 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500550static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400551{
552 iounmap(adev->doorbell.ptr);
553 adev->doorbell.ptr = NULL;
554}
555
Alex Deucher22cb0162017-12-14 16:27:11 -0500556
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400557
558/*
Alex Deucher06ec9072017-12-14 15:02:39 -0500559 * amdgpu_device_wb_*()
Alex Xie455a7bc2017-05-08 21:36:03 -0400560 * Writeback is the method by which the GPU updates special pages in memory
Alex Xieea81a172017-05-08 13:41:11 -0400561 * with the status of certain GPU events (fences, ring pointers,etc.).
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400562 */
563
564/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500565 * amdgpu_device_wb_fini - Disable Writeback and free memory
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400566 *
567 * @adev: amdgpu_device pointer
568 *
569 * Disables Writeback and frees the Writeback memory (all asics).
570 * Used at driver shutdown.
571 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500572static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400573{
574 if (adev->wb.wb_obj) {
Alex Deuchera76ed482016-10-21 15:30:36 -0400575 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
576 &adev->wb.gpu_addr,
577 (void **)&adev->wb.wb);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400578 adev->wb.wb_obj = NULL;
579 }
580}
581
582/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500583 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400584 *
585 * @adev: amdgpu_device pointer
586 *
Alex Xie455a7bc2017-05-08 21:36:03 -0400587 * Initializes writeback and allocates writeback memory (all asics).
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400588 * Used at driver startup.
589 * Returns 0 on success or an -error on failure.
590 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500591static int amdgpu_device_wb_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400592{
593 int r;
594
595 if (adev->wb.wb_obj == NULL) {
Alex Deucher97407b62017-07-28 12:14:15 -0400596 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
597 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
Alex Deuchera76ed482016-10-21 15:30:36 -0400598 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
599 &adev->wb.wb_obj, &adev->wb.gpu_addr,
600 (void **)&adev->wb.wb);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400601 if (r) {
602 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
603 return r;
604 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400605
606 adev->wb.num_wb = AMDGPU_MAX_WB;
607 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
608
609 /* clear wb memory */
Monk Liu73469582017-12-29 17:06:41 +0800610 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400611 }
612
613 return 0;
614}
615
616/**
Alex Deucher131b4b32017-12-14 16:03:43 -0500617 * amdgpu_device_wb_get - Allocate a wb entry
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400618 *
619 * @adev: amdgpu_device pointer
620 * @wb: wb index
621 *
622 * Allocate a wb slot for use by the driver (all asics).
623 * Returns 0 on success or -EINVAL on failure.
624 */
Alex Deucher131b4b32017-12-14 16:03:43 -0500625int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400626{
627 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
Alex Deucher97407b62017-07-28 12:14:15 -0400628
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400629 if (offset < adev->wb.num_wb) {
630 __set_bit(offset, adev->wb.used);
Monk Liu63ae07c2017-10-17 19:18:56 +0800631 *wb = offset << 3; /* convert to dw offset */
Monk Liu0915fdb2017-06-19 10:19:41 -0400632 return 0;
633 } else {
634 return -EINVAL;
635 }
636}
637
Ken Wang70142852016-03-18 15:08:49 +0800638/**
Alex Deucher131b4b32017-12-14 16:03:43 -0500639 * amdgpu_device_wb_free - Free a wb entry
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400640 *
641 * @adev: amdgpu_device pointer
642 * @wb: wb index
643 *
644 * Free a wb slot allocated for use by the driver (all asics)
645 */
Alex Deucher131b4b32017-12-14 16:03:43 -0500646void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400647{
Monk Liu73469582017-12-29 17:06:41 +0800648 wb >>= 3;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400649 if (wb < adev->wb.num_wb)
Monk Liu73469582017-12-29 17:06:41 +0800650 __clear_bit(wb, adev->wb.used);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400651}
652
653/**
Alex Deucher2543e282017-12-14 16:33:36 -0500654 * amdgpu_device_vram_location - try to find VRAM location
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500655 *
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400656 * @adev: amdgpu device structure holding all necessary informations
657 * @mc: memory controller structure holding memory informations
658 * @base: base address at which to put VRAM
659 *
Alex Xie455a7bc2017-05-08 21:36:03 -0400660 * Function will try to place VRAM at base address provided
Christian König3d647c82017-11-16 19:36:10 +0100661 * as parameter.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400662 */
Alex Deucher2543e282017-12-14 16:33:36 -0500663void amdgpu_device_vram_location(struct amdgpu_device *adev,
Christian König770d13b2018-01-12 14:52:22 +0100664 struct amdgpu_gmc *mc, u64 base)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400665{
666 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
667
668 mc->vram_start = base;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400669 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
670 if (limit && limit < mc->real_vram_size)
671 mc->real_vram_size = limit;
672 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
673 mc->mc_vram_size >> 20, mc->vram_start,
674 mc->vram_end, mc->real_vram_size >> 20);
675}
676
677/**
Alex Deucher2543e282017-12-14 16:33:36 -0500678 * amdgpu_device_gart_location - try to find GTT location
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500679 *
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400680 * @adev: amdgpu device structure holding all necessary informations
681 * @mc: memory controller structure holding memory informations
682 *
683 * Function will place try to place GTT before or after VRAM.
684 *
685 * If GTT size is bigger than space left then we ajust GTT size.
686 * Thus function will never fails.
687 *
688 * FIXME: when reducing GTT size align new size on power of 2.
689 */
Alex Deucher2543e282017-12-14 16:33:36 -0500690void amdgpu_device_gart_location(struct amdgpu_device *adev,
Christian König770d13b2018-01-12 14:52:22 +0100691 struct amdgpu_gmc *mc)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400692{
693 u64 size_af, size_bf;
694
Rex Zhu7951e372018-04-13 16:13:41 +0800695 mc->gart_size += adev->pm.smu_prv_buffer_size;
696
Christian König770d13b2018-01-12 14:52:22 +0100697 size_af = adev->gmc.mc_mask - mc->vram_end;
Christian Königed21c042017-07-06 22:26:05 +0200698 size_bf = mc->vram_start;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400699 if (size_bf > size_af) {
Christian König6f02a692017-07-07 11:56:59 +0200700 if (mc->gart_size > size_bf) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400701 dev_warn(adev->dev, "limiting GTT\n");
Christian König6f02a692017-07-07 11:56:59 +0200702 mc->gart_size = size_bf;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400703 }
Christian König6f02a692017-07-07 11:56:59 +0200704 mc->gart_start = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400705 } else {
Christian König6f02a692017-07-07 11:56:59 +0200706 if (mc->gart_size > size_af) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400707 dev_warn(adev->dev, "limiting GTT\n");
Christian König6f02a692017-07-07 11:56:59 +0200708 mc->gart_size = size_af;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400709 }
Christian Königb98f1b92017-11-16 20:12:51 +0100710 /* VCE doesn't like it when BOs cross a 4GB segment, so align
711 * the GART base on a 4GB boundary as well.
712 */
713 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400714 }
Christian König6f02a692017-07-07 11:56:59 +0200715 mc->gart_end = mc->gart_start + mc->gart_size - 1;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400716 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
Christian König6f02a692017-07-07 11:56:59 +0200717 mc->gart_size >> 20, mc->gart_start, mc->gart_end);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400718}
719
Christian Königd6895ad2017-02-28 10:36:43 +0100720/**
721 * amdgpu_device_resize_fb_bar - try to resize FB BAR
722 *
723 * @adev: amdgpu_device pointer
724 *
725 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
726 * to fail, but if any of the BARs is not accessible after the size we abort
727 * driver loading by returning -ENODEV.
728 */
729int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
730{
Christian König770d13b2018-01-12 14:52:22 +0100731 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
Christian Königd6895ad2017-02-28 10:36:43 +0100732 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
Christian König31b8ada2017-11-15 20:07:38 +0100733 struct pci_bus *root;
734 struct resource *res;
735 unsigned i;
Christian Königd6895ad2017-02-28 10:36:43 +0100736 u16 cmd;
737 int r;
738
pding0c03b912017-11-07 11:02:00 +0800739 /* Bypass for VF */
740 if (amdgpu_sriov_vf(adev))
741 return 0;
742
Christian König31b8ada2017-11-15 20:07:38 +0100743 /* Check if the root BUS has 64bit memory resources */
744 root = adev->pdev->bus;
745 while (root->parent)
746 root = root->parent;
747
748 pci_bus_for_each_resource(root, res, i) {
Christian König0ebb7c52018-01-07 10:18:57 +0100749 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
Christian König31b8ada2017-11-15 20:07:38 +0100750 res->start > 0x100000000ull)
751 break;
752 }
753
754 /* Trying to resize is pointless without a root hub window above 4GB */
755 if (!res)
756 return 0;
757
Christian Königd6895ad2017-02-28 10:36:43 +0100758 /* Disable memory decoding while we change the BAR addresses and size */
759 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
760 pci_write_config_word(adev->pdev, PCI_COMMAND,
761 cmd & ~PCI_COMMAND_MEMORY);
762
763 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
Alex Deucher06ec9072017-12-14 15:02:39 -0500764 amdgpu_device_doorbell_fini(adev);
Christian Königd6895ad2017-02-28 10:36:43 +0100765 if (adev->asic_type >= CHIP_BONAIRE)
766 pci_release_resource(adev->pdev, 2);
767
768 pci_release_resource(adev->pdev, 0);
769
770 r = pci_resize_resource(adev->pdev, 0, rbar_size);
771 if (r == -ENOSPC)
772 DRM_INFO("Not enough PCI address space for a large BAR.");
773 else if (r && r != -ENOTSUPP)
774 DRM_ERROR("Problem resizing BAR0 (%d).", r);
775
776 pci_assign_unassigned_bus_resources(adev->pdev->bus);
777
778 /* When the doorbell or fb BAR isn't available we have no chance of
779 * using the device.
780 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500781 r = amdgpu_device_doorbell_init(adev);
Christian Königd6895ad2017-02-28 10:36:43 +0100782 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
783 return -ENODEV;
784
785 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
786
787 return 0;
788}
Horace Chena05502e2017-09-29 14:41:57 +0800789
790/*
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400791 * GPU helpers function.
792 */
793/**
Alex Deucher39c640c2017-12-15 16:22:11 -0500794 * amdgpu_device_need_post - check if the hw need post or not
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400795 *
796 * @adev: amdgpu_device pointer
797 *
Jim Quc836fec2017-02-10 15:59:59 +0800798 * Check if the asic has been initialized (all asics) at driver startup
799 * or post is needed if hw reset is performed.
800 * Returns true if need or false if not.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400801 */
Alex Deucher39c640c2017-12-15 16:22:11 -0500802bool amdgpu_device_need_post(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400803{
804 uint32_t reg;
805
Monk Liubec86372016-09-14 19:38:08 +0800806 if (amdgpu_sriov_vf(adev))
807 return false;
808
809 if (amdgpu_passthrough(adev)) {
Monk Liu1da2c322016-11-11 11:24:29 +0800810 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
811 * some old smc fw still need driver do vPost otherwise gpu hang, while
812 * those smc fw version above 22.15 doesn't have this flaw, so we force
813 * vpost executed for smc version below 22.15
Monk Liubec86372016-09-14 19:38:08 +0800814 */
815 if (adev->asic_type == CHIP_FIJI) {
816 int err;
817 uint32_t fw_ver;
818 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
819 /* force vPost if error occured */
820 if (err)
821 return true;
822
823 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
Monk Liu1da2c322016-11-11 11:24:29 +0800824 if (fw_ver < 0x00160e00)
825 return true;
Monk Liubec86372016-09-14 19:38:08 +0800826 }
Monk Liubec86372016-09-14 19:38:08 +0800827 }
pding91fe77e2017-10-19 09:38:39 +0800828
829 if (adev->has_hw_reset) {
830 adev->has_hw_reset = false;
831 return true;
832 }
833
834 /* bios scratch used on CIK+ */
835 if (adev->asic_type >= CHIP_BONAIRE)
836 return amdgpu_atombios_scratch_need_asic_init(adev);
837
838 /* check MEM_SIZE for older asics */
839 reg = amdgpu_asic_get_config_memsize(adev);
840
841 if ((reg != 0) && (reg != 0xffffffff))
842 return false;
843
844 return true;
Monk Liubec86372016-09-14 19:38:08 +0800845}
846
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400847/* if we get transitioned to only one device, take VGA back */
848/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500849 * amdgpu_device_vga_set_decode - enable/disable vga decode
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400850 *
851 * @cookie: amdgpu_device pointer
852 * @state: enable/disable vga decode
853 *
854 * Enable/disable vga decode (all asics).
855 * Returns VGA resource flags.
856 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500857static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400858{
859 struct amdgpu_device *adev = cookie;
860 amdgpu_asic_set_vga_state(adev, state);
861 if (state)
862 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
863 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
864 else
865 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
866}
867
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500868/**
869 * amdgpu_device_check_block_size - validate the vm block size
870 *
871 * @adev: amdgpu_device pointer
872 *
873 * Validates the vm block size specified via module parameter.
874 * The vm block size defines number of bits in page table versus page directory,
875 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
876 * page table and the remaining bits are in the page directory.
877 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500878static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
Chunming Zhoua1adf8b2017-03-27 11:36:57 +0800879{
880 /* defines number of bits in page table versus page directory,
881 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
882 * page table and the remaining bits are in the page directory */
Junwei Zhangbab4fee2017-04-05 13:54:56 +0800883 if (amdgpu_vm_block_size == -1)
884 return;
Chunming Zhoua1adf8b2017-03-27 11:36:57 +0800885
Junwei Zhangbab4fee2017-04-05 13:54:56 +0800886 if (amdgpu_vm_block_size < 9) {
Chunming Zhoua1adf8b2017-03-27 11:36:57 +0800887 dev_warn(adev->dev, "VM page table size (%d) too small\n",
888 amdgpu_vm_block_size);
Christian König97489122017-11-27 16:22:05 +0100889 amdgpu_vm_block_size = -1;
Chunming Zhoua1adf8b2017-03-27 11:36:57 +0800890 }
Chunming Zhoua1adf8b2017-03-27 11:36:57 +0800891}
892
Alex Deuchere3ecdff2018-03-15 17:39:45 -0500893/**
894 * amdgpu_device_check_vm_size - validate the vm size
895 *
896 * @adev: amdgpu_device pointer
897 *
898 * Validates the vm size in GB specified via module parameter.
899 * The VM size is the size of the GPU virtual memory space in GB.
900 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500901static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
Zhang, Jerry83ca1452017-03-29 16:08:31 +0800902{
Alex Deucher64dab072017-06-15 18:20:09 -0400903 /* no need to check the default value */
904 if (amdgpu_vm_size == -1)
905 return;
906
Zhang, Jerry83ca1452017-03-29 16:08:31 +0800907 if (amdgpu_vm_size < 1) {
908 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
909 amdgpu_vm_size);
Christian Königf3368122017-11-23 12:57:18 +0100910 amdgpu_vm_size = -1;
Zhang, Jerry83ca1452017-03-29 16:08:31 +0800911 }
Zhang, Jerry83ca1452017-03-29 16:08:31 +0800912}
913
Rex Zhu7951e372018-04-13 16:13:41 +0800914static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
915{
916 struct sysinfo si;
917 bool is_os_64 = (sizeof(void *) == 8) ? true : false;
918 uint64_t total_memory;
919 uint64_t dram_size_seven_GB = 0x1B8000000;
920 uint64_t dram_size_three_GB = 0xB8000000;
921
922 if (amdgpu_smu_memory_pool_size == 0)
923 return;
924
925 if (!is_os_64) {
926 DRM_WARN("Not 64-bit OS, feature not supported\n");
927 goto def_value;
928 }
929 si_meminfo(&si);
930 total_memory = (uint64_t)si.totalram * si.mem_unit;
931
932 if ((amdgpu_smu_memory_pool_size == 1) ||
933 (amdgpu_smu_memory_pool_size == 2)) {
934 if (total_memory < dram_size_three_GB)
935 goto def_value1;
936 } else if ((amdgpu_smu_memory_pool_size == 4) ||
937 (amdgpu_smu_memory_pool_size == 8)) {
938 if (total_memory < dram_size_seven_GB)
939 goto def_value1;
940 } else {
941 DRM_WARN("Smu memory pool size not supported\n");
942 goto def_value;
943 }
944 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
945
946 return;
947
948def_value1:
949 DRM_WARN("No enough system memory\n");
950def_value:
951 adev->pm.smu_prv_buffer_size = 0;
952}
953
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400954/**
Alex Deucher06ec9072017-12-14 15:02:39 -0500955 * amdgpu_device_check_arguments - validate module params
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400956 *
957 * @adev: amdgpu_device pointer
958 *
959 * Validates certain module parameters and updates
960 * the associated values used by the driver (all asics).
961 */
Alex Deucher06ec9072017-12-14 15:02:39 -0500962static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400963{
Chunming Zhou5b011232015-12-10 17:34:33 +0800964 if (amdgpu_sched_jobs < 4) {
965 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
966 amdgpu_sched_jobs);
967 amdgpu_sched_jobs = 4;
Alex Deucher76117502017-06-21 12:31:41 -0400968 } else if (!is_power_of_2(amdgpu_sched_jobs)){
Chunming Zhou5b011232015-12-10 17:34:33 +0800969 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
970 amdgpu_sched_jobs);
971 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
972 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400973
Alex Deucher83e74db2017-08-21 11:58:25 -0400974 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
Christian Königf9321cc2017-07-07 13:44:05 +0200975 /* gart size must be greater or equal to 32M */
976 dev_warn(adev->dev, "gart size (%d) too small\n",
977 amdgpu_gart_size);
Alex Deucher83e74db2017-08-21 11:58:25 -0400978 amdgpu_gart_size = -1;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400979 }
980
Christian König36d38372017-07-07 13:17:45 +0200981 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400982 /* gtt size must be greater or equal to 32M */
Christian König36d38372017-07-07 13:17:45 +0200983 dev_warn(adev->dev, "gtt size (%d) too small\n",
984 amdgpu_gtt_size);
985 amdgpu_gtt_size = -1;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400986 }
987
Roger Hed07f14b2017-08-15 16:05:59 +0800988 /* valid range is between 4 and 9 inclusive */
989 if (amdgpu_vm_fragment_size != -1 &&
990 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
991 dev_warn(adev->dev, "valid range is between 4 and 9\n");
992 amdgpu_vm_fragment_size = -1;
993 }
994
Rex Zhu7951e372018-04-13 16:13:41 +0800995 amdgpu_device_check_smu_prv_buffer_size(adev);
996
Alex Deucher06ec9072017-12-14 15:02:39 -0500997 amdgpu_device_check_vm_size(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400998
Alex Deucher06ec9072017-12-14 15:02:39 -0500999 amdgpu_device_check_block_size(adev);
Christian König6a7f76e2016-08-24 15:51:49 +02001000
jimqu526bae32016-11-07 09:53:10 +08001001 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
Alex Deucher76117502017-06-21 12:31:41 -04001002 !is_power_of_2(amdgpu_vram_page_split))) {
Christian König6a7f76e2016-08-24 15:51:49 +02001003 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1004 amdgpu_vram_page_split);
1005 amdgpu_vram_page_split = 1024;
1006 }
Andrey Grodzovsky88546952017-12-13 14:36:53 -05001007
1008 if (amdgpu_lockup_timeout == 0) {
1009 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
1010 amdgpu_lockup_timeout = 10000;
1011 }
Alex Deucher19aede72018-03-09 15:06:35 -05001012
1013 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001014}
1015
1016/**
1017 * amdgpu_switcheroo_set_state - set switcheroo state
1018 *
1019 * @pdev: pci dev pointer
Lukas Wunner16944672015-09-05 11:17:35 +02001020 * @state: vga_switcheroo state
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001021 *
1022 * Callback for the switcheroo driver. Suspends or resumes the
1023 * the asics before or after it is powered up using ACPI methods.
1024 */
1025static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1026{
1027 struct drm_device *dev = pci_get_drvdata(pdev);
1028
1029 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1030 return;
1031
1032 if (state == VGA_SWITCHEROO_ON) {
Joe Perches7ca85292017-02-28 04:55:52 -08001033 pr_info("amdgpu: switched on\n");
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001034 /* don't suspend or resume card normally */
1035 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1036
Alex Deucher810ddc32016-08-23 13:25:49 -04001037 amdgpu_device_resume(dev, true, true);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001038
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001039 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1040 drm_kms_helper_poll_enable(dev);
1041 } else {
Joe Perches7ca85292017-02-28 04:55:52 -08001042 pr_info("amdgpu: switched off\n");
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001043 drm_kms_helper_poll_disable(dev);
1044 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
Alex Deucher810ddc32016-08-23 13:25:49 -04001045 amdgpu_device_suspend(dev, true, true);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001046 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1047 }
1048}
1049
1050/**
1051 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1052 *
1053 * @pdev: pci dev pointer
1054 *
1055 * Callback for the switcheroo driver. Check of the switcheroo
1056 * state can be changed.
1057 * Returns true if the state can be changed, false if not.
1058 */
1059static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1060{
1061 struct drm_device *dev = pci_get_drvdata(pdev);
1062
1063 /*
1064 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1065 * locking inversion with the driver load path. And the access here is
1066 * completely racy anyway. So don't bother with locking for now.
1067 */
1068 return dev->open_count == 0;
1069}
1070
1071static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1072 .set_gpu_state = amdgpu_switcheroo_set_state,
1073 .reprobe = NULL,
1074 .can_switch = amdgpu_switcheroo_can_switch,
1075};
1076
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001077/**
1078 * amdgpu_device_ip_set_clockgating_state - set the CG state
1079 *
1080 * @adev: amdgpu_device pointer
1081 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1082 * @state: clockgating state (gate or ungate)
1083 *
1084 * Sets the requested clockgating state for all instances of
1085 * the hardware IP specified.
1086 * Returns the error code from the last instance.
1087 */
Rex Zhu43fa5612018-03-28 13:42:45 -05001088int amdgpu_device_ip_set_clockgating_state(void *dev,
Alex Deucher2990a1f2017-12-15 16:18:00 -05001089 enum amd_ip_block_type block_type,
1090 enum amd_clockgating_state state)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001091{
Rex Zhu43fa5612018-03-28 13:42:45 -05001092 struct amdgpu_device *adev = dev;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001093 int i, r = 0;
1094
1095 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001096 if (!adev->ip_blocks[i].status.valid)
Alex Deucher9ecbe7f2016-06-23 11:53:12 -04001097 continue;
Rex Zhuc7228652017-02-22 15:33:46 +08001098 if (adev->ip_blocks[i].version->type != block_type)
1099 continue;
1100 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1101 continue;
1102 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1103 (void *)adev, state);
1104 if (r)
1105 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1106 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001107 }
1108 return r;
1109}
1110
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001111/**
1112 * amdgpu_device_ip_set_powergating_state - set the PG state
1113 *
1114 * @adev: amdgpu_device pointer
1115 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1116 * @state: powergating state (gate or ungate)
1117 *
1118 * Sets the requested powergating state for all instances of
1119 * the hardware IP specified.
1120 * Returns the error code from the last instance.
1121 */
Rex Zhu43fa5612018-03-28 13:42:45 -05001122int amdgpu_device_ip_set_powergating_state(void *dev,
Alex Deucher2990a1f2017-12-15 16:18:00 -05001123 enum amd_ip_block_type block_type,
1124 enum amd_powergating_state state)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001125{
Rex Zhu43fa5612018-03-28 13:42:45 -05001126 struct amdgpu_device *adev = dev;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001127 int i, r = 0;
1128
1129 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001130 if (!adev->ip_blocks[i].status.valid)
Alex Deucher9ecbe7f2016-06-23 11:53:12 -04001131 continue;
Rex Zhuc7228652017-02-22 15:33:46 +08001132 if (adev->ip_blocks[i].version->type != block_type)
1133 continue;
1134 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1135 continue;
1136 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1137 (void *)adev, state);
1138 if (r)
1139 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1140 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001141 }
1142 return r;
1143}
1144
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001145/**
1146 * amdgpu_device_ip_get_clockgating_state - get the CG state
1147 *
1148 * @adev: amdgpu_device pointer
1149 * @flags: clockgating feature flags
1150 *
1151 * Walks the list of IPs on the device and updates the clockgating
1152 * flags for each IP.
1153 * Updates @flags with the feature flags for each hardware IP where
1154 * clockgating is enabled.
1155 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001156void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1157 u32 *flags)
Huang Rui6cb2d4e2017-01-05 18:44:41 +08001158{
1159 int i;
1160
1161 for (i = 0; i < adev->num_ip_blocks; i++) {
1162 if (!adev->ip_blocks[i].status.valid)
1163 continue;
1164 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1165 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1166 }
1167}
1168
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001169/**
1170 * amdgpu_device_ip_wait_for_idle - wait for idle
1171 *
1172 * @adev: amdgpu_device pointer
1173 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1174 *
1175 * Waits for the request hardware IP to be idle.
1176 * Returns 0 for success or a negative error code on failure.
1177 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001178int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1179 enum amd_ip_block_type block_type)
Alex Deucher5dbbb602016-06-23 11:41:04 -04001180{
1181 int i, r;
1182
1183 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001184 if (!adev->ip_blocks[i].status.valid)
Alex Deucher9ecbe7f2016-06-23 11:53:12 -04001185 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001186 if (adev->ip_blocks[i].version->type == block_type) {
1187 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
Alex Deucher5dbbb602016-06-23 11:41:04 -04001188 if (r)
1189 return r;
1190 break;
1191 }
1192 }
1193 return 0;
1194
1195}
1196
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001197/**
1198 * amdgpu_device_ip_is_idle - is the hardware IP idle
1199 *
1200 * @adev: amdgpu_device pointer
1201 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1202 *
1203 * Check if the hardware IP is idle or not.
1204 * Returns true if it the IP is idle, false if not.
1205 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001206bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1207 enum amd_ip_block_type block_type)
Alex Deucher5dbbb602016-06-23 11:41:04 -04001208{
1209 int i;
1210
1211 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001212 if (!adev->ip_blocks[i].status.valid)
Alex Deucher9ecbe7f2016-06-23 11:53:12 -04001213 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001214 if (adev->ip_blocks[i].version->type == block_type)
1215 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
Alex Deucher5dbbb602016-06-23 11:41:04 -04001216 }
1217 return true;
1218
1219}
1220
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001221/**
1222 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1223 *
1224 * @adev: amdgpu_device pointer
1225 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1226 *
1227 * Returns a pointer to the hardware IP block structure
1228 * if it exists for the asic, otherwise NULL.
1229 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001230struct amdgpu_ip_block *
1231amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1232 enum amd_ip_block_type type)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001233{
1234 int i;
1235
1236 for (i = 0; i < adev->num_ip_blocks; i++)
Alex Deuchera1255102016-10-13 17:41:13 -04001237 if (adev->ip_blocks[i].version->type == type)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001238 return &adev->ip_blocks[i];
1239
1240 return NULL;
1241}
1242
1243/**
Alex Deucher2990a1f2017-12-15 16:18:00 -05001244 * amdgpu_device_ip_block_version_cmp
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001245 *
1246 * @adev: amdgpu_device pointer
yanyang15fc3aee2015-05-22 14:39:35 -04001247 * @type: enum amd_ip_block_type
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001248 * @major: major version
1249 * @minor: minor version
1250 *
1251 * return 0 if equal or greater
1252 * return 1 if smaller or the ip_block doesn't exist
1253 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001254int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1255 enum amd_ip_block_type type,
1256 u32 major, u32 minor)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001257{
Alex Deucher2990a1f2017-12-15 16:18:00 -05001258 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001259
Alex Deuchera1255102016-10-13 17:41:13 -04001260 if (ip_block && ((ip_block->version->major > major) ||
1261 ((ip_block->version->major == major) &&
1262 (ip_block->version->minor >= minor))))
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001263 return 0;
1264
1265 return 1;
1266}
1267
Alex Deuchera1255102016-10-13 17:41:13 -04001268/**
Alex Deucher2990a1f2017-12-15 16:18:00 -05001269 * amdgpu_device_ip_block_add
Alex Deuchera1255102016-10-13 17:41:13 -04001270 *
1271 * @adev: amdgpu_device pointer
1272 * @ip_block_version: pointer to the IP to add
1273 *
1274 * Adds the IP block driver information to the collection of IPs
1275 * on the asic.
1276 */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001277int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1278 const struct amdgpu_ip_block_version *ip_block_version)
Alex Deuchera1255102016-10-13 17:41:13 -04001279{
1280 if (!ip_block_version)
1281 return -EINVAL;
1282
Shaoyun Liue966a722018-02-01 16:45:26 -05001283 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
Huang Ruia0bae352017-05-03 09:52:06 +08001284 ip_block_version->funcs->name);
1285
Alex Deuchera1255102016-10-13 17:41:13 -04001286 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1287
1288 return 0;
1289}
1290
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001291/**
1292 * amdgpu_device_enable_virtual_display - enable virtual display feature
1293 *
1294 * @adev: amdgpu_device pointer
1295 *
1296 * Enabled the virtual display feature if the user has enabled it via
1297 * the module parameter virtual_display. This feature provides a virtual
1298 * display hardware on headless boards or in virtualized environments.
1299 * This function parses and validates the configuration string specified by
1300 * the user and configues the virtual display configuration (number of
1301 * virtual connectors, crtcs, etc.) specified.
1302 */
Alex Deucher483ef982016-09-30 12:43:04 -04001303static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
Emily Deng9accf2f2016-08-10 16:01:25 +08001304{
1305 adev->enable_virtual_display = false;
1306
1307 if (amdgpu_virtual_display) {
1308 struct drm_device *ddev = adev->ddev;
1309 const char *pci_address_name = pci_name(ddev->pdev);
Emily Deng0f663562016-09-30 13:02:18 -04001310 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
Emily Deng9accf2f2016-08-10 16:01:25 +08001311
1312 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1313 pciaddstr_tmp = pciaddstr;
Emily Deng0f663562016-09-30 13:02:18 -04001314 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1315 pciaddname = strsep(&pciaddname_tmp, ",");
Yintian Tao967de2a2017-01-22 15:16:51 +08001316 if (!strcmp("all", pciaddname)
1317 || !strcmp(pci_address_name, pciaddname)) {
Emily Deng0f663562016-09-30 13:02:18 -04001318 long num_crtc;
1319 int res = -1;
1320
Emily Deng9accf2f2016-08-10 16:01:25 +08001321 adev->enable_virtual_display = true;
Emily Deng0f663562016-09-30 13:02:18 -04001322
1323 if (pciaddname_tmp)
1324 res = kstrtol(pciaddname_tmp, 10,
1325 &num_crtc);
1326
1327 if (!res) {
1328 if (num_crtc < 1)
1329 num_crtc = 1;
1330 if (num_crtc > 6)
1331 num_crtc = 6;
1332 adev->mode_info.num_crtc = num_crtc;
1333 } else {
1334 adev->mode_info.num_crtc = 1;
1335 }
Emily Deng9accf2f2016-08-10 16:01:25 +08001336 break;
1337 }
1338 }
1339
Emily Deng0f663562016-09-30 13:02:18 -04001340 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1341 amdgpu_virtual_display, pci_address_name,
1342 adev->enable_virtual_display, adev->mode_info.num_crtc);
Emily Deng9accf2f2016-08-10 16:01:25 +08001343
1344 kfree(pciaddstr);
1345 }
1346}
1347
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001348/**
1349 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1350 *
1351 * @adev: amdgpu_device pointer
1352 *
1353 * Parses the asic configuration parameters specified in the gpu info
1354 * firmware and makes them availale to the driver for use in configuring
1355 * the asic.
1356 * Returns 0 on success, -EINVAL on failure.
1357 */
Alex Deuchere2a75f82017-04-27 16:58:01 -04001358static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1359{
Alex Deuchere2a75f82017-04-27 16:58:01 -04001360 const char *chip_name;
1361 char fw_name[30];
1362 int err;
1363 const struct gpu_info_firmware_header_v1_0 *hdr;
1364
Huang Ruiab4fe3e2017-06-05 22:11:59 +08001365 adev->firmware.gpu_info_fw = NULL;
1366
Alex Deuchere2a75f82017-04-27 16:58:01 -04001367 switch (adev->asic_type) {
1368 case CHIP_TOPAZ:
1369 case CHIP_TONGA:
1370 case CHIP_FIJI:
Alex Deuchere2a75f82017-04-27 16:58:01 -04001371 case CHIP_POLARIS10:
Leo Liucc07f182017-11-09 13:19:58 -05001372 case CHIP_POLARIS11:
Alex Deuchere2a75f82017-04-27 16:58:01 -04001373 case CHIP_POLARIS12:
Leo Liucc07f182017-11-09 13:19:58 -05001374 case CHIP_VEGAM:
Alex Deuchere2a75f82017-04-27 16:58:01 -04001375 case CHIP_CARRIZO:
1376 case CHIP_STONEY:
1377#ifdef CONFIG_DRM_AMDGPU_SI
1378 case CHIP_VERDE:
1379 case CHIP_TAHITI:
1380 case CHIP_PITCAIRN:
1381 case CHIP_OLAND:
1382 case CHIP_HAINAN:
1383#endif
1384#ifdef CONFIG_DRM_AMDGPU_CIK
1385 case CHIP_BONAIRE:
1386 case CHIP_HAWAII:
1387 case CHIP_KAVERI:
1388 case CHIP_KABINI:
1389 case CHIP_MULLINS:
1390#endif
Feifei Xu27c0bc72018-05-17 10:01:19 -05001391 case CHIP_VEGA20:
Alex Deuchere2a75f82017-04-27 16:58:01 -04001392 default:
1393 return 0;
1394 case CHIP_VEGA10:
1395 chip_name = "vega10";
1396 break;
Alex Deucher3f76dce2017-09-01 16:20:53 -04001397 case CHIP_VEGA12:
1398 chip_name = "vega12";
1399 break;
Alex Deucher2d2e5e72017-05-09 12:27:35 -04001400 case CHIP_RAVEN:
1401 chip_name = "raven";
1402 break;
Alex Deuchere2a75f82017-04-27 16:58:01 -04001403 }
1404
1405 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
Huang Ruiab4fe3e2017-06-05 22:11:59 +08001406 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
Alex Deuchere2a75f82017-04-27 16:58:01 -04001407 if (err) {
1408 dev_err(adev->dev,
1409 "Failed to load gpu_info firmware \"%s\"\n",
1410 fw_name);
1411 goto out;
1412 }
Huang Ruiab4fe3e2017-06-05 22:11:59 +08001413 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
Alex Deuchere2a75f82017-04-27 16:58:01 -04001414 if (err) {
1415 dev_err(adev->dev,
1416 "Failed to validate gpu_info firmware \"%s\"\n",
1417 fw_name);
1418 goto out;
1419 }
1420
Huang Ruiab4fe3e2017-06-05 22:11:59 +08001421 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
Alex Deuchere2a75f82017-04-27 16:58:01 -04001422 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1423
1424 switch (hdr->version_major) {
1425 case 1:
1426 {
1427 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
Huang Ruiab4fe3e2017-06-05 22:11:59 +08001428 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
Alex Deuchere2a75f82017-04-27 16:58:01 -04001429 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1430
Alex Deucherb5ab16b2017-05-11 19:09:49 -04001431 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1432 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1433 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1434 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
Alex Deuchere2a75f82017-04-27 16:58:01 -04001435 adev->gfx.config.max_texture_channel_caches =
Alex Deucherb5ab16b2017-05-11 19:09:49 -04001436 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1437 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1438 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1439 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1440 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
Alex Deuchere2a75f82017-04-27 16:58:01 -04001441 adev->gfx.config.double_offchip_lds_buf =
Alex Deucherb5ab16b2017-05-11 19:09:49 -04001442 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1443 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
Hawking Zhang51fd0372017-06-09 22:30:52 +08001444 adev->gfx.cu_info.max_waves_per_simd =
1445 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1446 adev->gfx.cu_info.max_scratch_slots_per_cu =
1447 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1448 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
Alex Deuchere2a75f82017-04-27 16:58:01 -04001449 break;
1450 }
1451 default:
1452 dev_err(adev->dev,
1453 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1454 err = -EINVAL;
1455 goto out;
1456 }
1457out:
Alex Deuchere2a75f82017-04-27 16:58:01 -04001458 return err;
1459}
1460
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001461/**
1462 * amdgpu_device_ip_early_init - run early init for hardware IPs
1463 *
1464 * @adev: amdgpu_device pointer
1465 *
1466 * Early initialization pass for hardware IPs. The hardware IPs that make
1467 * up each asic are discovered each IP's early_init callback is run. This
1468 * is the first stage in initializing the asic.
1469 * Returns 0 on success, negative error code on failure.
1470 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001471static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001472{
Alex Deucheraaa36a92015-04-20 17:31:14 -04001473 int i, r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001474
Alex Deucher483ef982016-09-30 12:43:04 -04001475 amdgpu_device_enable_virtual_display(adev);
Emily Denga6be7572016-08-08 11:37:50 +08001476
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001477 switch (adev->asic_type) {
Alex Deucheraaa36a92015-04-20 17:31:14 -04001478 case CHIP_TOPAZ:
1479 case CHIP_TONGA:
David Zhang48299f92015-07-08 01:05:16 +08001480 case CHIP_FIJI:
Flora Cui2cc0c0b2016-03-14 18:33:29 -04001481 case CHIP_POLARIS10:
Leo Liu32cc7e52017-11-09 13:22:54 -05001482 case CHIP_POLARIS11:
Junwei Zhangc4642a42016-12-14 15:32:28 -05001483 case CHIP_POLARIS12:
Leo Liu32cc7e52017-11-09 13:22:54 -05001484 case CHIP_VEGAM:
Alex Deucheraaa36a92015-04-20 17:31:14 -04001485 case CHIP_CARRIZO:
Samuel Li39bb0c92015-10-08 16:31:43 -04001486 case CHIP_STONEY:
1487 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
Alex Deucheraaa36a92015-04-20 17:31:14 -04001488 adev->family = AMDGPU_FAMILY_CZ;
1489 else
1490 adev->family = AMDGPU_FAMILY_VI;
1491
1492 r = vi_set_ip_blocks(adev);
1493 if (r)
1494 return r;
1495 break;
Ken Wang33f34802016-01-21 17:29:41 +08001496#ifdef CONFIG_DRM_AMDGPU_SI
1497 case CHIP_VERDE:
1498 case CHIP_TAHITI:
1499 case CHIP_PITCAIRN:
1500 case CHIP_OLAND:
1501 case CHIP_HAINAN:
Ken Wang295d0da2016-05-24 21:02:53 +08001502 adev->family = AMDGPU_FAMILY_SI;
Ken Wang33f34802016-01-21 17:29:41 +08001503 r = si_set_ip_blocks(adev);
1504 if (r)
1505 return r;
1506 break;
1507#endif
Alex Deuchera2e73f52015-04-20 17:09:27 -04001508#ifdef CONFIG_DRM_AMDGPU_CIK
1509 case CHIP_BONAIRE:
1510 case CHIP_HAWAII:
1511 case CHIP_KAVERI:
1512 case CHIP_KABINI:
1513 case CHIP_MULLINS:
1514 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1515 adev->family = AMDGPU_FAMILY_CI;
1516 else
1517 adev->family = AMDGPU_FAMILY_KV;
1518
1519 r = cik_set_ip_blocks(adev);
1520 if (r)
1521 return r;
1522 break;
1523#endif
Alex Deuchere48a3cd2017-09-01 16:22:35 -04001524 case CHIP_VEGA10:
1525 case CHIP_VEGA12:
Feifei Xue4bd8172018-04-20 12:33:33 +08001526 case CHIP_VEGA20:
Alex Deuchere48a3cd2017-09-01 16:22:35 -04001527 case CHIP_RAVEN:
Chunming Zhou2ca8a5d2016-12-07 17:31:19 +08001528 if (adev->asic_type == CHIP_RAVEN)
1529 adev->family = AMDGPU_FAMILY_RV;
1530 else
1531 adev->family = AMDGPU_FAMILY_AI;
Ken Wang460826e2017-03-06 14:53:16 -05001532
1533 r = soc15_set_ip_blocks(adev);
1534 if (r)
1535 return r;
1536 break;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001537 default:
1538 /* FIXME: not supported yet */
1539 return -EINVAL;
1540 }
1541
Alex Deuchere2a75f82017-04-27 16:58:01 -04001542 r = amdgpu_device_parse_gpu_info_fw(adev);
1543 if (r)
1544 return r;
1545
pding18847342017-11-06 10:21:26 +08001546 amdgpu_amdkfd_device_probe(adev);
1547
Xiangliang Yu3149d9d2017-01-12 15:14:36 +08001548 if (amdgpu_sriov_vf(adev)) {
1549 r = amdgpu_virt_request_full_gpu(adev, true);
1550 if (r)
pding5ffa61c2017-10-30 14:07:24 +08001551 return -EAGAIN;
Xiangliang Yu3149d9d2017-01-12 15:14:36 +08001552 }
1553
Huang Rui00f54b92018-02-27 21:53:00 +08001554 adev->powerplay.pp_feature = amdgpu_pp_feature_mask;
1555
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001556 for (i = 0; i < adev->num_ip_blocks; i++) {
1557 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
Huang Ruied8cf002017-05-03 09:40:17 +08001558 DRM_ERROR("disabled ip block: %d <%s>\n",
1559 i, adev->ip_blocks[i].version->funcs->name);
Alex Deuchera1255102016-10-13 17:41:13 -04001560 adev->ip_blocks[i].status.valid = false;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001561 } else {
Alex Deuchera1255102016-10-13 17:41:13 -04001562 if (adev->ip_blocks[i].version->funcs->early_init) {
1563 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001564 if (r == -ENOENT) {
Alex Deuchera1255102016-10-13 17:41:13 -04001565 adev->ip_blocks[i].status.valid = false;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001566 } else if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001567 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1568 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001569 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001570 } else {
Alex Deuchera1255102016-10-13 17:41:13 -04001571 adev->ip_blocks[i].status.valid = true;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001572 }
Alex Deucher974e6b62015-07-10 13:59:44 -04001573 } else {
Alex Deuchera1255102016-10-13 17:41:13 -04001574 adev->ip_blocks[i].status.valid = true;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001575 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001576 }
1577 }
1578
Nicolai Hähnle395d1fb2016-06-02 12:32:07 +02001579 adev->cg_flags &= amdgpu_cg_mask;
1580 adev->pg_flags &= amdgpu_pg_mask;
1581
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001582 return 0;
1583}
1584
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001585/**
1586 * amdgpu_device_ip_init - run init for hardware IPs
1587 *
1588 * @adev: amdgpu_device pointer
1589 *
1590 * Main initialization pass for hardware IPs. The list of all the hardware
1591 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1592 * are run. sw_init initializes the software state associated with each IP
1593 * and hw_init initializes the hardware associated with each IP.
1594 * Returns 0 on success, negative error code on failure.
1595 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001596static int amdgpu_device_ip_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001597{
1598 int i, r;
1599
1600 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001601 if (!adev->ip_blocks[i].status.valid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001602 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001603 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001604 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001605 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1606 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001607 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001608 }
Alex Deuchera1255102016-10-13 17:41:13 -04001609 adev->ip_blocks[i].status.sw = true;
Shaoyun Liubfca0282018-02-01 17:37:50 -05001610
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001611 /* need to do gmc hw init early so we can allocate gpu mem */
Alex Deuchera1255102016-10-13 17:41:13 -04001612 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
Alex Deucher06ec9072017-12-14 15:02:39 -05001613 r = amdgpu_device_vram_scratch_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001614 if (r) {
1615 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001616 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001617 }
Alex Deuchera1255102016-10-13 17:41:13 -04001618 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001619 if (r) {
1620 DRM_ERROR("hw_init %d failed %d\n", i, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001621 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001622 }
Alex Deucher06ec9072017-12-14 15:02:39 -05001623 r = amdgpu_device_wb_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001624 if (r) {
Alex Deucher06ec9072017-12-14 15:02:39 -05001625 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001626 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001627 }
Alex Deuchera1255102016-10-13 17:41:13 -04001628 adev->ip_blocks[i].status.hw = true;
Monk Liu24936642017-01-09 15:54:32 +08001629
1630 /* right after GMC hw init, we create CSA */
1631 if (amdgpu_sriov_vf(adev)) {
1632 r = amdgpu_allocate_static_csa(adev);
1633 if (r) {
1634 DRM_ERROR("allocate CSA failed %d\n", r);
1635 return r;
1636 }
1637 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001638 }
1639 }
1640
1641 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001642 if (!adev->ip_blocks[i].status.sw)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001643 continue;
Shaoyun Liubfca0282018-02-01 17:37:50 -05001644 if (adev->ip_blocks[i].status.hw)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001645 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001646 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001647 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001648 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1649 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001650 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001651 }
Alex Deuchera1255102016-10-13 17:41:13 -04001652 adev->ip_blocks[i].status.hw = true;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001653 }
1654
pding18847342017-11-06 10:21:26 +08001655 amdgpu_amdkfd_device_init(adev);
pdingc6332b92017-11-06 11:21:55 +08001656
1657 if (amdgpu_sriov_vf(adev))
1658 amdgpu_virt_release_full_gpu(adev, true);
1659
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001660 return 0;
1661}
1662
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001663/**
1664 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1665 *
1666 * @adev: amdgpu_device pointer
1667 *
1668 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
1669 * this function before a GPU reset. If the value is retained after a
1670 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
1671 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001672static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
Chunming Zhou0c49e0b2017-05-15 14:20:00 +08001673{
1674 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1675}
1676
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001677/**
1678 * amdgpu_device_check_vram_lost - check if vram is valid
1679 *
1680 * @adev: amdgpu_device pointer
1681 *
1682 * Checks the reset magic value written to the gart pointer in VRAM.
1683 * The driver calls this after a GPU reset to see if the contents of
1684 * VRAM is lost or now.
1685 * returns true if vram is lost, false if not.
1686 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001687static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
Chunming Zhou0c49e0b2017-05-15 14:20:00 +08001688{
1689 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1690 AMDGPU_RESET_MAGIC_NUM);
1691}
1692
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001693/**
1694 * amdgpu_device_ip_late_set_cg_state - late init for clockgating
1695 *
1696 * @adev: amdgpu_device pointer
1697 *
1698 * Late initialization pass enabling clockgating for hardware IPs.
1699 * The list of all the hardware IPs that make up the asic is walked and the
1700 * set_clockgating_state callbacks are run. This stage is run late
1701 * in the init process.
1702 * Returns 0 on success, negative error code on failure.
1703 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001704static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
Shirish S2dc80b02017-05-25 10:05:25 +05301705{
1706 int i = 0, r;
1707
Shaoyun Liu4a2ba392018-02-05 16:41:33 -05001708 if (amdgpu_emu_mode == 1)
1709 return 0;
1710
Shirish S2c773de2018-04-16 12:17:57 +05301711 r = amdgpu_ib_ring_tests(adev);
1712 if (r)
1713 DRM_ERROR("ib ring test failed (%d).\n", r);
1714
Shirish S2dc80b02017-05-25 10:05:25 +05301715 for (i = 0; i < adev->num_ip_blocks; i++) {
1716 if (!adev->ip_blocks[i].status.valid)
1717 continue;
1718 /* skip CG for VCE/UVD, it's handled specially */
1719 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
Rex Zhu57716322018-03-12 19:50:38 +08001720 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
Rex Zhu34319b32018-05-16 20:06:53 +08001721 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
Rex Zhu57716322018-03-12 19:50:38 +08001722 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
Shirish S2dc80b02017-05-25 10:05:25 +05301723 /* enable clockgating to save power */
1724 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1725 AMD_CG_STATE_GATE);
1726 if (r) {
1727 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1728 adev->ip_blocks[i].version->funcs->name, r);
1729 return r;
1730 }
1731 }
1732 }
Huang Rui06b18f62018-06-01 14:41:04 +08001733
1734 if (adev->powerplay.pp_feature & PP_GFXOFF_MASK) {
1735 /* enable gfx powergating */
1736 amdgpu_device_ip_set_powergating_state(adev,
1737 AMD_IP_BLOCK_TYPE_GFX,
1738 AMD_PG_STATE_GATE);
1739 /* enable gfxoff */
1740 amdgpu_device_ip_set_powergating_state(adev,
1741 AMD_IP_BLOCK_TYPE_SMC,
1742 AMD_PG_STATE_GATE);
1743 }
1744
Shirish S2dc80b02017-05-25 10:05:25 +05301745 return 0;
1746}
1747
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001748/**
1749 * amdgpu_device_ip_late_init - run late init for hardware IPs
1750 *
1751 * @adev: amdgpu_device pointer
1752 *
1753 * Late initialization pass for hardware IPs. The list of all the hardware
1754 * IPs that make up the asic is walked and the late_init callbacks are run.
1755 * late_init covers any special initialization that an IP requires
1756 * after all of the have been initialized or something that needs to happen
1757 * late in the init process.
1758 * Returns 0 on success, negative error code on failure.
1759 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001760static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001761{
1762 int i = 0, r;
1763
1764 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001765 if (!adev->ip_blocks[i].status.valid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001766 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001767 if (adev->ip_blocks[i].version->funcs->late_init) {
1768 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001769 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001770 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1771 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001772 return r;
Alex Deucher2c1a2782015-12-07 17:02:53 -05001773 }
Alex Deuchera1255102016-10-13 17:41:13 -04001774 adev->ip_blocks[i].status.late_initialized = true;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001775 }
1776 }
1777
Shirish S2c773de2018-04-16 12:17:57 +05301778 queue_delayed_work(system_wq, &adev->late_init_work,
1779 msecs_to_jiffies(AMDGPU_RESUME_MS));
Shirish S2dc80b02017-05-25 10:05:25 +05301780
Alex Deucher06ec9072017-12-14 15:02:39 -05001781 amdgpu_device_fill_reset_magic(adev);
Chunming Zhou0c49e0b2017-05-15 14:20:00 +08001782
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001783 return 0;
1784}
1785
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001786/**
1787 * amdgpu_device_ip_fini - run fini for hardware IPs
1788 *
1789 * @adev: amdgpu_device pointer
1790 *
1791 * Main teardown pass for hardware IPs. The list of all the hardware
1792 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
1793 * are run. hw_fini tears down the hardware associated with each IP
1794 * and sw_fini tears down any software state associated with each IP.
1795 * Returns 0 on success, negative error code on failure.
1796 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001797static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001798{
1799 int i, r;
1800
pding18847342017-11-06 10:21:26 +08001801 amdgpu_amdkfd_device_fini(adev);
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001802 /* need to disable SMC first */
1803 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04001804 if (!adev->ip_blocks[i].status.hw)
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001805 continue;
Rex Zhu57716322018-03-12 19:50:38 +08001806 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC &&
1807 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001808 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
Alex Deuchera1255102016-10-13 17:41:13 -04001809 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1810 AMD_CG_STATE_UNGATE);
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001811 if (r) {
1812 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
Alex Deuchera1255102016-10-13 17:41:13 -04001813 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001814 return r;
1815 }
Alex Deuchera1255102016-10-13 17:41:13 -04001816 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001817 /* XXX handle errors */
1818 if (r) {
1819 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
Alex Deuchera1255102016-10-13 17:41:13 -04001820 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001821 }
Alex Deuchera1255102016-10-13 17:41:13 -04001822 adev->ip_blocks[i].status.hw = false;
Alex Deucher3e96dbf2016-10-13 11:22:17 -04001823 break;
1824 }
1825 }
1826
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001827 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
Alex Deuchera1255102016-10-13 17:41:13 -04001828 if (!adev->ip_blocks[i].status.hw)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001829 continue;
Rex Zhu8201a672016-11-24 21:44:44 +08001830
1831 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
Rex Zhu81ce8be2018-03-20 16:28:56 +08001832 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
Rex Zhu34319b32018-05-16 20:06:53 +08001833 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
Rex Zhu81ce8be2018-03-20 16:28:56 +08001834 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
Rex Zhu8201a672016-11-24 21:44:44 +08001835 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1836 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1837 AMD_CG_STATE_UNGATE);
1838 if (r) {
1839 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1840 adev->ip_blocks[i].version->funcs->name, r);
1841 return r;
1842 }
Alex Deucher2c1a2782015-12-07 17:02:53 -05001843 }
Rex Zhu8201a672016-11-24 21:44:44 +08001844
Alex Deuchera1255102016-10-13 17:41:13 -04001845 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001846 /* XXX handle errors */
Alex Deucher2c1a2782015-12-07 17:02:53 -05001847 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001848 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1849 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001850 }
Rex Zhu8201a672016-11-24 21:44:44 +08001851
Alex Deuchera1255102016-10-13 17:41:13 -04001852 adev->ip_blocks[i].status.hw = false;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001853 }
1854
Alex Deucher9950cda2018-01-18 19:05:36 -05001855
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001856 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
Alex Deuchera1255102016-10-13 17:41:13 -04001857 if (!adev->ip_blocks[i].status.sw)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001858 continue;
Monk Liuc12aba32018-01-24 12:20:32 +08001859
1860 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1861 amdgpu_free_static_csa(adev);
1862 amdgpu_device_wb_fini(adev);
1863 amdgpu_device_vram_scratch_fini(adev);
1864 }
1865
Alex Deuchera1255102016-10-13 17:41:13 -04001866 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001867 /* XXX handle errors */
Alex Deucher2c1a2782015-12-07 17:02:53 -05001868 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001869 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1870 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001871 }
Alex Deuchera1255102016-10-13 17:41:13 -04001872 adev->ip_blocks[i].status.sw = false;
1873 adev->ip_blocks[i].status.valid = false;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001874 }
1875
Monk Liua6dcfd92016-05-19 14:36:34 +08001876 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
Alex Deuchera1255102016-10-13 17:41:13 -04001877 if (!adev->ip_blocks[i].status.late_initialized)
Grazvydas Ignotas8a2eef12016-10-03 00:06:44 +03001878 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04001879 if (adev->ip_blocks[i].version->funcs->late_fini)
1880 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1881 adev->ip_blocks[i].status.late_initialized = false;
Monk Liua6dcfd92016-05-19 14:36:34 +08001882 }
1883
Monk Liu030308f2017-09-15 15:34:52 +08001884 if (amdgpu_sriov_vf(adev))
Monk Liu24136132017-11-14 16:56:55 +08001885 if (amdgpu_virt_release_full_gpu(adev, false))
1886 DRM_ERROR("failed to release exclusive mode on fini\n");
Monk Liu24936642017-01-09 15:54:32 +08001887
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001888 return 0;
1889}
1890
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001891/**
1892 * amdgpu_device_ip_late_init_func_handler - work handler for clockgating
1893 *
1894 * @work: work_struct
1895 *
1896 * Work handler for amdgpu_device_ip_late_set_cg_state. We put the
1897 * clockgating setup into a worker thread to speed up driver init and
1898 * resume from suspend.
1899 */
Alex Deucher06ec9072017-12-14 15:02:39 -05001900static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
Shirish S2dc80b02017-05-25 10:05:25 +05301901{
1902 struct amdgpu_device *adev =
1903 container_of(work, struct amdgpu_device, late_init_work.work);
Alex Deucher06ec9072017-12-14 15:02:39 -05001904 amdgpu_device_ip_late_set_cg_state(adev);
Shirish S2dc80b02017-05-25 10:05:25 +05301905}
1906
Alex Deuchere3ecdff2018-03-15 17:39:45 -05001907/**
1908 * amdgpu_device_ip_suspend - run suspend for hardware IPs
1909 *
1910 * @adev: amdgpu_device pointer
1911 *
1912 * Main suspend function for hardware IPs. The list of all the hardware
1913 * IPs that make up the asic is walked, clockgating is disabled and the
1914 * suspend callbacks are run. suspend puts the hardware and software state
1915 * in each IP into a state suitable for suspend.
1916 * Returns 0 on success, negative error code on failure.
1917 */
Alex Deuchercdd61df2017-12-14 16:47:40 -05001918int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001919{
1920 int i, r;
1921
Xiangliang Yue941ea92017-01-18 12:47:55 +08001922 if (amdgpu_sriov_vf(adev))
1923 amdgpu_virt_request_full_gpu(adev, false);
1924
Huang Ruib0833692018-03-13 18:39:48 +08001925 /* ungate SMC block powergating */
1926 if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
1927 amdgpu_device_ip_set_powergating_state(adev,
1928 AMD_IP_BLOCK_TYPE_SMC,
1929 AMD_CG_STATE_UNGATE);
1930
Flora Cuic5a93a22016-02-26 10:45:25 +08001931 /* ungate SMC block first */
Alex Deucher2990a1f2017-12-15 16:18:00 -05001932 r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1933 AMD_CG_STATE_UNGATE);
Flora Cuic5a93a22016-02-26 10:45:25 +08001934 if (r) {
Alex Deucher2990a1f2017-12-15 16:18:00 -05001935 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
Flora Cuic5a93a22016-02-26 10:45:25 +08001936 }
1937
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001938 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
Alex Deuchera1255102016-10-13 17:41:13 -04001939 if (!adev->ip_blocks[i].status.valid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001940 continue;
1941 /* ungate blocks so that suspend can properly shut them down */
Rex Zhu5b2a3d22018-03-14 15:38:48 +08001942 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC &&
Rex Zhu57716322018-03-12 19:50:38 +08001943 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
Alex Deuchera1255102016-10-13 17:41:13 -04001944 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1945 AMD_CG_STATE_UNGATE);
Flora Cuic5a93a22016-02-26 10:45:25 +08001946 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001947 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1948 adev->ip_blocks[i].version->funcs->name, r);
Flora Cuic5a93a22016-02-26 10:45:25 +08001949 }
Alex Deucher2c1a2782015-12-07 17:02:53 -05001950 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001951 /* XXX handle errors */
Alex Deuchera1255102016-10-13 17:41:13 -04001952 r = adev->ip_blocks[i].version->funcs->suspend(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001953 /* XXX handle errors */
Alex Deucher2c1a2782015-12-07 17:02:53 -05001954 if (r) {
Alex Deuchera1255102016-10-13 17:41:13 -04001955 DRM_ERROR("suspend of IP block <%s> failed %d\n",
1956 adev->ip_blocks[i].version->funcs->name, r);
Alex Deucher2c1a2782015-12-07 17:02:53 -05001957 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001958 }
1959
Xiangliang Yue941ea92017-01-18 12:47:55 +08001960 if (amdgpu_sriov_vf(adev))
1961 amdgpu_virt_release_full_gpu(adev, false);
1962
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001963 return 0;
1964}
1965
Alex Deucher06ec9072017-12-14 15:02:39 -05001966static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
Monk Liua90ad3c2017-01-23 14:22:08 +08001967{
1968 int i, r;
1969
Monk Liu2cb681b2017-04-26 12:00:49 +08001970 static enum amd_ip_block_type ip_order[] = {
1971 AMD_IP_BLOCK_TYPE_GMC,
1972 AMD_IP_BLOCK_TYPE_COMMON,
Monk Liu2cb681b2017-04-26 12:00:49 +08001973 AMD_IP_BLOCK_TYPE_IH,
1974 };
Monk Liua90ad3c2017-01-23 14:22:08 +08001975
Monk Liu2cb681b2017-04-26 12:00:49 +08001976 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1977 int j;
1978 struct amdgpu_ip_block *block;
Monk Liua90ad3c2017-01-23 14:22:08 +08001979
Monk Liu2cb681b2017-04-26 12:00:49 +08001980 for (j = 0; j < adev->num_ip_blocks; j++) {
1981 block = &adev->ip_blocks[j];
1982
1983 if (block->version->type != ip_order[i] ||
1984 !block->status.valid)
1985 continue;
1986
1987 r = block->version->funcs->hw_init(adev);
1988 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
Monk Liuc41d1cf2017-12-25 11:59:27 +08001989 if (r)
1990 return r;
Monk Liua90ad3c2017-01-23 14:22:08 +08001991 }
1992 }
1993
1994 return 0;
1995}
1996
Alex Deucher06ec9072017-12-14 15:02:39 -05001997static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
Monk Liua90ad3c2017-01-23 14:22:08 +08001998{
1999 int i, r;
2000
Monk Liu2cb681b2017-04-26 12:00:49 +08002001 static enum amd_ip_block_type ip_order[] = {
2002 AMD_IP_BLOCK_TYPE_SMC,
Monk Liuef4c1662017-09-22 16:23:34 +08002003 AMD_IP_BLOCK_TYPE_PSP,
Monk Liu2cb681b2017-04-26 12:00:49 +08002004 AMD_IP_BLOCK_TYPE_DCE,
2005 AMD_IP_BLOCK_TYPE_GFX,
2006 AMD_IP_BLOCK_TYPE_SDMA,
Frank Min257deb82017-06-15 20:07:36 +08002007 AMD_IP_BLOCK_TYPE_UVD,
2008 AMD_IP_BLOCK_TYPE_VCE
Monk Liu2cb681b2017-04-26 12:00:49 +08002009 };
Monk Liua90ad3c2017-01-23 14:22:08 +08002010
Monk Liu2cb681b2017-04-26 12:00:49 +08002011 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2012 int j;
2013 struct amdgpu_ip_block *block;
Monk Liua90ad3c2017-01-23 14:22:08 +08002014
Monk Liu2cb681b2017-04-26 12:00:49 +08002015 for (j = 0; j < adev->num_ip_blocks; j++) {
2016 block = &adev->ip_blocks[j];
2017
2018 if (block->version->type != ip_order[i] ||
2019 !block->status.valid)
2020 continue;
2021
2022 r = block->version->funcs->hw_init(adev);
2023 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
Monk Liuc41d1cf2017-12-25 11:59:27 +08002024 if (r)
2025 return r;
Monk Liua90ad3c2017-01-23 14:22:08 +08002026 }
2027 }
2028
2029 return 0;
2030}
2031
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002032/**
2033 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2034 *
2035 * @adev: amdgpu_device pointer
2036 *
2037 * First resume function for hardware IPs. The list of all the hardware
2038 * IPs that make up the asic is walked and the resume callbacks are run for
2039 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2040 * after a suspend and updates the software state as necessary. This
2041 * function is also used for restoring the GPU after a GPU reset.
2042 * Returns 0 on success, negative error code on failure.
2043 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002044static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002045{
2046 int i, r;
2047
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002048 for (i = 0; i < adev->num_ip_blocks; i++) {
2049 if (!adev->ip_blocks[i].status.valid)
2050 continue;
Chunming Zhoufcf06492017-05-05 10:33:33 +08002051 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002052 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2053 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
Chunming Zhoufcf06492017-05-05 10:33:33 +08002054 r = adev->ip_blocks[i].version->funcs->resume(adev);
2055 if (r) {
2056 DRM_ERROR("resume of IP block <%s> failed %d\n",
2057 adev->ip_blocks[i].version->funcs->name, r);
2058 return r;
2059 }
2060 }
2061 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002062
Chunming Zhoufcf06492017-05-05 10:33:33 +08002063 return 0;
2064}
2065
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002066/**
2067 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2068 *
2069 * @adev: amdgpu_device pointer
2070 *
2071 * First resume function for hardware IPs. The list of all the hardware
2072 * IPs that make up the asic is walked and the resume callbacks are run for
2073 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2074 * functional state after a suspend and updates the software state as
2075 * necessary. This function is also used for restoring the GPU after a GPU
2076 * reset.
2077 * Returns 0 on success, negative error code on failure.
2078 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002079static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
Chunming Zhoufcf06492017-05-05 10:33:33 +08002080{
2081 int i, r;
2082
2083 for (i = 0; i < adev->num_ip_blocks; i++) {
2084 if (!adev->ip_blocks[i].status.valid)
2085 continue;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002086 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002087 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2088 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002089 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002090 r = adev->ip_blocks[i].version->funcs->resume(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002091 if (r) {
2092 DRM_ERROR("resume of IP block <%s> failed %d\n",
2093 adev->ip_blocks[i].version->funcs->name, r);
2094 return r;
2095 }
2096 }
2097
2098 return 0;
2099}
2100
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002101/**
2102 * amdgpu_device_ip_resume - run resume for hardware IPs
2103 *
2104 * @adev: amdgpu_device pointer
2105 *
2106 * Main resume function for hardware IPs. The hardware IPs
2107 * are split into two resume functions because they are
2108 * are also used in in recovering from a GPU reset and some additional
2109 * steps need to be take between them. In this case (S3/S4) they are
2110 * run sequentially.
2111 * Returns 0 on success, negative error code on failure.
2112 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002113static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002114{
Chunming Zhoufcf06492017-05-05 10:33:33 +08002115 int r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002116
Alex Deucher06ec9072017-12-14 15:02:39 -05002117 r = amdgpu_device_ip_resume_phase1(adev);
Chunming Zhoufcf06492017-05-05 10:33:33 +08002118 if (r)
2119 return r;
Alex Deucher06ec9072017-12-14 15:02:39 -05002120 r = amdgpu_device_ip_resume_phase2(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002121
Chunming Zhoufcf06492017-05-05 10:33:33 +08002122 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002123}
2124
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002125/**
2126 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2127 *
2128 * @adev: amdgpu_device pointer
2129 *
2130 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2131 */
Monk Liu4e99a442016-03-31 13:26:59 +08002132static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
Andres Rodriguez048765a2016-06-11 02:51:32 -04002133{
Monk Liu6867e1b2017-10-16 19:50:44 +08002134 if (amdgpu_sriov_vf(adev)) {
2135 if (adev->is_atom_fw) {
2136 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2137 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2138 } else {
2139 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2140 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2141 }
2142
2143 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2144 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
Alex Deuchera5bde2f2016-09-23 16:23:41 -04002145 }
Andres Rodriguez048765a2016-06-11 02:51:32 -04002146}
2147
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002148/**
2149 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2150 *
2151 * @asic_type: AMD asic type
2152 *
2153 * Check if there is DC (new modesetting infrastructre) support for an asic.
2154 * returns true if DC has support, false if not.
2155 */
Harry Wentland45622362017-09-12 15:58:20 -04002156bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2157{
2158 switch (asic_type) {
2159#if defined(CONFIG_DRM_AMD_DC)
2160 case CHIP_BONAIRE:
2161 case CHIP_HAWAII:
Alex Deucher0d6fbcc2017-08-10 14:39:48 -04002162 case CHIP_KAVERI:
Alex Deucher367e6682018-01-25 16:53:25 -05002163 case CHIP_KABINI:
2164 case CHIP_MULLINS:
Harry Wentland45622362017-09-12 15:58:20 -04002165 case CHIP_CARRIZO:
2166 case CHIP_STONEY:
Harry Wentland45622362017-09-12 15:58:20 -04002167 case CHIP_POLARIS10:
Leo Liu675fd322017-11-08 18:07:12 -05002168 case CHIP_POLARIS11:
Alex Deucher2c8ad2d2017-06-15 16:20:24 -04002169 case CHIP_POLARIS12:
Leo Liu675fd322017-11-08 18:07:12 -05002170 case CHIP_VEGAM:
Harry Wentland45622362017-09-12 15:58:20 -04002171 case CHIP_TONGA:
2172 case CHIP_FIJI:
Harry Wentland42f8ffa2017-09-15 14:07:30 -04002173 case CHIP_VEGA10:
Alex Deucherdca7b402017-09-02 02:05:29 -04002174 case CHIP_VEGA12:
Feifei Xuc6034aa2018-02-03 12:19:46 +08002175 case CHIP_VEGA20:
Harry Wentland42f8ffa2017-09-15 14:07:30 -04002176#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
Hawking Zhangfd187852017-03-06 14:01:11 +08002177 case CHIP_RAVEN:
Harry Wentland42f8ffa2017-09-15 14:07:30 -04002178#endif
Hawking Zhangfd187852017-03-06 14:01:11 +08002179 return amdgpu_dc != 0;
2180#endif
Harry Wentland45622362017-09-12 15:58:20 -04002181 default:
2182 return false;
2183 }
2184}
2185
2186/**
2187 * amdgpu_device_has_dc_support - check if dc is supported
2188 *
2189 * @adev: amdgpu_device_pointer
2190 *
2191 * Returns true for supported, false for not supported
2192 */
2193bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2194{
Xiangliang Yu2555039d2017-01-10 17:34:52 +08002195 if (amdgpu_sriov_vf(adev))
2196 return false;
2197
Harry Wentland45622362017-09-12 15:58:20 -04002198 return amdgpu_device_asic_has_dc_support(adev->asic_type);
2199}
2200
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002201/**
2202 * amdgpu_device_init - initialize the driver
2203 *
2204 * @adev: amdgpu_device pointer
2205 * @pdev: drm dev pointer
2206 * @pdev: pci dev pointer
2207 * @flags: driver flags
2208 *
2209 * Initializes the driver info and hw (all asics).
2210 * Returns 0 for success or an error on failure.
2211 * Called at driver startup.
2212 */
2213int amdgpu_device_init(struct amdgpu_device *adev,
2214 struct drm_device *ddev,
2215 struct pci_dev *pdev,
2216 uint32_t flags)
2217{
2218 int r, i;
2219 bool runtime = false;
Marek Olšák95844d22016-08-17 23:49:27 +02002220 u32 max_MBps;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002221
2222 adev->shutdown = false;
2223 adev->dev = &pdev->dev;
2224 adev->ddev = ddev;
2225 adev->pdev = pdev;
2226 adev->flags = flags;
Jammy Zhou2f7d10b2015-07-22 11:29:01 +08002227 adev->asic_type = flags & AMD_ASIC_MASK;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002228 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
Shaoyun Liu593aa2d2018-02-07 14:43:13 -05002229 if (amdgpu_emu_mode == 1)
2230 adev->usec_timeout *= 2;
Christian König770d13b2018-01-12 14:52:22 +01002231 adev->gmc.gart_size = 512 * 1024 * 1024;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002232 adev->accel_working = false;
2233 adev->num_rings = 0;
2234 adev->mman.buffer_funcs = NULL;
2235 adev->mman.buffer_funcs_ring = NULL;
2236 adev->vm_manager.vm_pte_funcs = NULL;
Christian König2d55e452016-02-08 17:37:38 +01002237 adev->vm_manager.vm_pte_num_rings = 0;
Christian König132f34e2018-01-12 15:26:08 +01002238 adev->gmc.gmc_funcs = NULL;
Chris Wilsonf54d1862016-10-25 13:00:45 +01002239 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
Andres Rodriguezb8866c22017-04-28 20:05:51 -04002240 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002241
2242 adev->smc_rreg = &amdgpu_invalid_rreg;
2243 adev->smc_wreg = &amdgpu_invalid_wreg;
2244 adev->pcie_rreg = &amdgpu_invalid_rreg;
2245 adev->pcie_wreg = &amdgpu_invalid_wreg;
Huang Rui36b9a952016-08-31 13:23:25 +08002246 adev->pciep_rreg = &amdgpu_invalid_rreg;
2247 adev->pciep_wreg = &amdgpu_invalid_wreg;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002248 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2249 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2250 adev->didt_rreg = &amdgpu_invalid_rreg;
2251 adev->didt_wreg = &amdgpu_invalid_wreg;
Rex Zhuccdbb202016-06-08 12:47:41 +08002252 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2253 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002254 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2255 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2256
Alex Deucher3e39ab92015-06-05 15:04:33 -04002257 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2258 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2259 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002260
2261 /* mutex initialization are all done here so we
2262 * can recall function without having locking issues */
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002263 atomic_set(&adev->irq.ih.lock, 0);
Huang Rui0e5ca0d2017-03-03 18:37:23 -05002264 mutex_init(&adev->firmware.mutex);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002265 mutex_init(&adev->pm.mutex);
2266 mutex_init(&adev->gfx.gpu_clock_mutex);
2267 mutex_init(&adev->srbm_mutex);
Andres Rodriguezb8866c22017-04-28 20:05:51 -04002268 mutex_init(&adev->gfx.pipe_reserve_mutex);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002269 mutex_init(&adev->grbm_idx_mutex);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002270 mutex_init(&adev->mn_lock);
Alex Deuchere23b74a2017-09-28 09:47:32 -04002271 mutex_init(&adev->virt.vf_errors.lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002272 hash_init(adev->mn_hash);
Monk Liu13a752e2017-10-17 15:11:12 +08002273 mutex_init(&adev->lock_reset);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002274
Alex Deucher06ec9072017-12-14 15:02:39 -05002275 amdgpu_device_check_arguments(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002276
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002277 spin_lock_init(&adev->mmio_idx_lock);
2278 spin_lock_init(&adev->smc_idx_lock);
2279 spin_lock_init(&adev->pcie_idx_lock);
2280 spin_lock_init(&adev->uvd_ctx_idx_lock);
2281 spin_lock_init(&adev->didt_idx_lock);
Rex Zhuccdbb202016-06-08 12:47:41 +08002282 spin_lock_init(&adev->gc_cac_idx_lock);
Evan Quan16abb5d2017-07-04 09:21:50 +08002283 spin_lock_init(&adev->se_cac_idx_lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002284 spin_lock_init(&adev->audio_endpt_idx_lock);
Marek Olšák95844d22016-08-17 23:49:27 +02002285 spin_lock_init(&adev->mm_stats.lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002286
Chunming Zhou0c4e7fa2016-08-17 11:41:30 +08002287 INIT_LIST_HEAD(&adev->shadow_list);
2288 mutex_init(&adev->shadow_list_lock);
2289
Andres Rodriguez795f2812017-03-06 16:27:55 -05002290 INIT_LIST_HEAD(&adev->ring_lru_list);
2291 spin_lock_init(&adev->ring_lru_list_lock);
2292
Alex Deucher06ec9072017-12-14 15:02:39 -05002293 INIT_DELAYED_WORK(&adev->late_init_work,
2294 amdgpu_device_ip_late_init_func_handler);
Shirish S2dc80b02017-05-25 10:05:25 +05302295
Alex Xie0fa49552017-06-08 14:58:05 -04002296 /* Registers mapping */
2297 /* TODO: block userspace mapping of io register */
Ken Wangda69c1612016-01-21 19:08:55 +08002298 if (adev->asic_type >= CHIP_BONAIRE) {
2299 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2300 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2301 } else {
2302 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2303 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2304 }
Chunming Zhou5c1354b2016-08-30 16:13:10 +08002305
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002306 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2307 if (adev->rmmio == NULL) {
2308 return -ENOMEM;
2309 }
2310 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2311 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2312
Christian König705e5192017-06-08 11:15:16 +02002313 /* doorbell bar mapping */
Alex Deucher06ec9072017-12-14 15:02:39 -05002314 amdgpu_device_doorbell_init(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002315
2316 /* io port mapping */
2317 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2318 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2319 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2320 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2321 break;
2322 }
2323 }
2324 if (adev->rio_mem == NULL)
Amber Linb64a18c2017-01-04 08:06:58 -05002325 DRM_INFO("PCI I/O BAR is not found.\n");
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002326
Alex Deucher5494d862018-03-09 15:14:11 -05002327 amdgpu_device_get_pcie_info(adev);
2328
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002329 /* early init functions */
Alex Deucher06ec9072017-12-14 15:02:39 -05002330 r = amdgpu_device_ip_early_init(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002331 if (r)
2332 return r;
2333
2334 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2335 /* this will fail for cards that aren't VGA class devices, just
2336 * ignore it */
Alex Deucher06ec9072017-12-14 15:02:39 -05002337 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002338
Alex Deuchere9bef452016-04-25 13:12:18 -04002339 if (amdgpu_device_is_px(ddev))
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002340 runtime = true;
Lukas Wunner84c8b222017-03-10 21:23:45 +01002341 if (!pci_is_thunderbolt_attached(adev->pdev))
2342 vga_switcheroo_register_client(adev->pdev,
2343 &amdgpu_switcheroo_ops, runtime);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002344 if (runtime)
2345 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2346
Shaoyun Liu9475a942018-02-01 18:13:23 -05002347 if (amdgpu_emu_mode == 1) {
2348 /* post the asic on emulation mode */
2349 emu_soc_asic_init(adev);
Shaoyun Liubfca0282018-02-01 17:37:50 -05002350 goto fence_driver_init;
Shaoyun Liu9475a942018-02-01 18:13:23 -05002351 }
Shaoyun Liubfca0282018-02-01 17:37:50 -05002352
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002353 /* Read BIOS */
Alex Deucher83ba1262016-06-03 18:21:41 -04002354 if (!amdgpu_get_bios(adev)) {
2355 r = -EINVAL;
2356 goto failed;
2357 }
Nils Wallméniusf7e9e9f2016-12-14 21:52:45 +01002358
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002359 r = amdgpu_atombios_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05002360 if (r) {
2361 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002362 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
Alex Deucher83ba1262016-06-03 18:21:41 -04002363 goto failed;
Alex Deucher2c1a2782015-12-07 17:02:53 -05002364 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002365
Monk Liu4e99a442016-03-31 13:26:59 +08002366 /* detect if we are with an SRIOV vbios */
2367 amdgpu_device_detect_sriov_bios(adev);
Andres Rodriguez048765a2016-06-11 02:51:32 -04002368
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002369 /* Post card if necessary */
Alex Deucher39c640c2017-12-15 16:22:11 -05002370 if (amdgpu_device_need_post(adev)) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002371 if (!adev->bios) {
Monk Liubec86372016-09-14 19:38:08 +08002372 dev_err(adev->dev, "no vBIOS found\n");
Alex Deucher83ba1262016-06-03 18:21:41 -04002373 r = -EINVAL;
2374 goto failed;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002375 }
Monk Liubec86372016-09-14 19:38:08 +08002376 DRM_INFO("GPU posting now...\n");
Monk Liu4e99a442016-03-31 13:26:59 +08002377 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2378 if (r) {
2379 dev_err(adev->dev, "gpu post error!\n");
2380 goto failed;
2381 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002382 }
2383
Alex Deucher88b64e92017-07-10 10:43:10 -04002384 if (adev->is_atom_fw) {
2385 /* Initialize clocks */
2386 r = amdgpu_atomfirmware_get_clock_info(adev);
2387 if (r) {
2388 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002389 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
Alex Deucher88b64e92017-07-10 10:43:10 -04002390 goto failed;
2391 }
2392 } else {
Alex Deuchera5bde2f2016-09-23 16:23:41 -04002393 /* Initialize clocks */
2394 r = amdgpu_atombios_get_clock_info(adev);
2395 if (r) {
2396 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002397 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
Gavin Wan89041942017-06-23 13:55:15 -04002398 goto failed;
Alex Deuchera5bde2f2016-09-23 16:23:41 -04002399 }
2400 /* init i2c buses */
Harry Wentland45622362017-09-12 15:58:20 -04002401 if (!amdgpu_device_has_dc_support(adev))
2402 amdgpu_atombios_i2c_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05002403 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002404
Shaoyun Liubfca0282018-02-01 17:37:50 -05002405fence_driver_init:
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002406 /* Fence driver */
2407 r = amdgpu_fence_driver_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05002408 if (r) {
2409 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002410 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
Alex Deucher83ba1262016-06-03 18:21:41 -04002411 goto failed;
Alex Deucher2c1a2782015-12-07 17:02:53 -05002412 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002413
2414 /* init the mode config */
2415 drm_mode_config_init(adev->ddev);
2416
Alex Deucher06ec9072017-12-14 15:02:39 -05002417 r = amdgpu_device_ip_init(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002418 if (r) {
pding8840a382017-10-23 17:22:09 +08002419 /* failed in exclusive mode due to timeout */
2420 if (amdgpu_sriov_vf(adev) &&
2421 !amdgpu_sriov_runtime(adev) &&
2422 amdgpu_virt_mmio_blocked(adev) &&
2423 !amdgpu_virt_wait_reset(adev)) {
2424 dev_err(adev->dev, "VF exclusive mode timeout\n");
Pixel Ding1daee8b2017-11-08 11:03:14 +08002425 /* Don't send request since VF is inactive. */
2426 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2427 adev->virt.ops = NULL;
pding8840a382017-10-23 17:22:09 +08002428 r = -EAGAIN;
2429 goto failed;
2430 }
Alex Deucher06ec9072017-12-14 15:02:39 -05002431 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002432 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
Alex Deucher83ba1262016-06-03 18:21:41 -04002433 goto failed;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002434 }
2435
2436 adev->accel_working = true;
2437
Alex Xiee59c0202017-06-01 09:42:59 -04002438 amdgpu_vm_check_compute_bug(adev);
2439
Marek Olšák95844d22016-08-17 23:49:27 +02002440 /* Initialize the buffer migration limit. */
2441 if (amdgpu_moverate >= 0)
2442 max_MBps = amdgpu_moverate;
2443 else
2444 max_MBps = 8; /* Allow 8 MB/s. */
2445 /* Get a log2 for easy divisions. */
2446 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2447
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002448 r = amdgpu_ib_pool_init(adev);
2449 if (r) {
2450 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
Alex Deuchere23b74a2017-09-28 09:47:32 -04002451 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
Alex Deucher83ba1262016-06-03 18:21:41 -04002452 goto failed;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002453 }
2454
Horace Chen2dc8f812017-10-09 16:17:16 +08002455 if (amdgpu_sriov_vf(adev))
2456 amdgpu_virt_init_data_exchange(adev);
2457
Monk Liu9bc92b92017-02-08 17:38:13 +08002458 amdgpu_fbdev_init(adev);
2459
Rex Zhud2f52ac2017-09-22 17:47:27 +08002460 r = amdgpu_pm_sysfs_init(adev);
2461 if (r)
2462 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2463
Alex Deucher75758252017-12-14 15:23:14 -05002464 r = amdgpu_debugfs_gem_init(adev);
Monk Liu3f14e622017-02-09 13:42:27 +08002465 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002466 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002467
2468 r = amdgpu_debugfs_regs_init(adev);
Monk Liu3f14e622017-02-09 13:42:27 +08002469 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002470 DRM_ERROR("registering register debugfs failed (%d).\n", r);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002471
Huang Rui50ab2532016-06-12 15:51:09 +08002472 r = amdgpu_debugfs_firmware_init(adev);
Monk Liu3f14e622017-02-09 13:42:27 +08002473 if (r)
Huang Rui50ab2532016-06-12 15:51:09 +08002474 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
Huang Rui50ab2532016-06-12 15:51:09 +08002475
Christian König763efb62017-12-06 15:44:51 +01002476 r = amdgpu_debugfs_init(adev);
Kent Russelldb95e212017-08-22 12:31:43 -04002477 if (r)
Christian König763efb62017-12-06 15:44:51 +01002478 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
Kent Russelldb95e212017-08-22 12:31:43 -04002479
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002480 if ((amdgpu_testing & 1)) {
2481 if (adev->accel_working)
2482 amdgpu_test_moves(adev);
2483 else
2484 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2485 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002486 if (amdgpu_benchmarking) {
2487 if (adev->accel_working)
2488 amdgpu_benchmark(adev, amdgpu_benchmarking);
2489 else
2490 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2491 }
2492
2493 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2494 * explicit gating rather than handling it automatically.
2495 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002496 r = amdgpu_device_ip_late_init(adev);
Alex Deucher2c1a2782015-12-07 17:02:53 -05002497 if (r) {
Alex Deucher06ec9072017-12-14 15:02:39 -05002498 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
Alex Deuchere23b74a2017-09-28 09:47:32 -04002499 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
Alex Deucher83ba1262016-06-03 18:21:41 -04002500 goto failed;
Alex Deucher2c1a2782015-12-07 17:02:53 -05002501 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002502
2503 return 0;
Alex Deucher83ba1262016-06-03 18:21:41 -04002504
2505failed:
Gavin Wan89041942017-06-23 13:55:15 -04002506 amdgpu_vf_error_trans_all(adev);
Alex Deucher83ba1262016-06-03 18:21:41 -04002507 if (runtime)
2508 vga_switcheroo_fini_domain_pm_ops(adev->dev);
pding8840a382017-10-23 17:22:09 +08002509
Alex Deucher83ba1262016-06-03 18:21:41 -04002510 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002511}
2512
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002513/**
2514 * amdgpu_device_fini - tear down the driver
2515 *
2516 * @adev: amdgpu_device pointer
2517 *
2518 * Tear down the driver info (all asics).
2519 * Called at driver shutdown.
2520 */
2521void amdgpu_device_fini(struct amdgpu_device *adev)
2522{
2523 int r;
2524
2525 DRM_INFO("amdgpu: finishing device.\n");
2526 adev->shutdown = true;
Mikita Lipskie5b03032018-03-15 16:53:08 -04002527 /* disable all interrupts */
2528 amdgpu_irq_disable_all(adev);
Mikita Lipskiff97cba2018-03-14 13:41:29 -04002529 if (adev->mode_info.mode_config_initialized){
2530 if (!amdgpu_device_has_dc_support(adev))
2531 drm_crtc_force_disable_all(adev->ddev);
2532 else
2533 drm_atomic_helper_shutdown(adev->ddev);
2534 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002535 amdgpu_ib_pool_fini(adev);
2536 amdgpu_fence_driver_fini(adev);
Emily Deng58e955d2018-03-08 09:35:19 +08002537 amdgpu_pm_sysfs_fini(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002538 amdgpu_fbdev_fini(adev);
Alex Deucher06ec9072017-12-14 15:02:39 -05002539 r = amdgpu_device_ip_fini(adev);
Huang Ruiab4fe3e2017-06-05 22:11:59 +08002540 if (adev->firmware.gpu_info_fw) {
2541 release_firmware(adev->firmware.gpu_info_fw);
2542 adev->firmware.gpu_info_fw = NULL;
2543 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002544 adev->accel_working = false;
Shirish S2dc80b02017-05-25 10:05:25 +05302545 cancel_delayed_work_sync(&adev->late_init_work);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002546 /* free i2c buses */
Harry Wentland45622362017-09-12 15:58:20 -04002547 if (!amdgpu_device_has_dc_support(adev))
2548 amdgpu_i2c_fini(adev);
Shaoyun Liubfca0282018-02-01 17:37:50 -05002549
2550 if (amdgpu_emu_mode != 1)
2551 amdgpu_atombios_fini(adev);
2552
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002553 kfree(adev->bios);
2554 adev->bios = NULL;
Lukas Wunner84c8b222017-03-10 21:23:45 +01002555 if (!pci_is_thunderbolt_attached(adev->pdev))
2556 vga_switcheroo_unregister_client(adev->pdev);
Alex Deucher83ba1262016-06-03 18:21:41 -04002557 if (adev->flags & AMD_IS_PX)
2558 vga_switcheroo_fini_domain_pm_ops(adev->dev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002559 vga_client_register(adev->pdev, NULL, NULL, NULL);
2560 if (adev->rio_mem)
2561 pci_iounmap(adev->pdev, adev->rio_mem);
2562 adev->rio_mem = NULL;
2563 iounmap(adev->rmmio);
2564 adev->rmmio = NULL;
Alex Deucher06ec9072017-12-14 15:02:39 -05002565 amdgpu_device_doorbell_fini(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002566 amdgpu_debugfs_regs_cleanup(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002567}
2568
2569
2570/*
2571 * Suspend & resume.
2572 */
2573/**
Alex Deucher810ddc32016-08-23 13:25:49 -04002574 * amdgpu_device_suspend - initiate device suspend
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002575 *
2576 * @pdev: drm dev pointer
2577 * @state: suspend state
2578 *
2579 * Puts the hw in the suspend state (all asics).
2580 * Returns 0 for success or an error on failure.
2581 * Called at driver suspend.
2582 */
Alex Deucher810ddc32016-08-23 13:25:49 -04002583int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002584{
2585 struct amdgpu_device *adev;
2586 struct drm_crtc *crtc;
2587 struct drm_connector *connector;
Alex Deucher5ceb54c2015-08-05 12:41:48 -04002588 int r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002589
2590 if (dev == NULL || dev->dev_private == NULL) {
2591 return -ENODEV;
2592 }
2593
2594 adev = dev->dev_private;
2595
2596 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2597 return 0;
2598
2599 drm_kms_helper_poll_disable(dev);
2600
Harry Wentland45622362017-09-12 15:58:20 -04002601 if (!amdgpu_device_has_dc_support(adev)) {
2602 /* turn off display hw */
2603 drm_modeset_lock_all(dev);
2604 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2605 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2606 }
2607 drm_modeset_unlock_all(dev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002608 }
2609
Yong Zhaoba997702015-11-09 17:21:45 -05002610 amdgpu_amdkfd_suspend(adev);
2611
Alex Deucher756e6882015-10-08 00:03:36 -04002612 /* unpin the front buffers and cursors */
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002613 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
Alex Deucher756e6882015-10-08 00:03:36 -04002614 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
Daniel Stonee68d14d2018-03-30 15:11:38 +01002615 struct drm_framebuffer *fb = crtc->primary->fb;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002616 struct amdgpu_bo *robj;
2617
Alex Deucher756e6882015-10-08 00:03:36 -04002618 if (amdgpu_crtc->cursor_bo) {
2619 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
Alex Xie7a6901d2017-04-24 13:52:41 -04002620 r = amdgpu_bo_reserve(aobj, true);
Alex Deucher756e6882015-10-08 00:03:36 -04002621 if (r == 0) {
2622 amdgpu_bo_unpin(aobj);
2623 amdgpu_bo_unreserve(aobj);
2624 }
2625 }
2626
Daniel Stonee68d14d2018-03-30 15:11:38 +01002627 if (fb == NULL || fb->obj[0] == NULL) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002628 continue;
2629 }
Daniel Stonee68d14d2018-03-30 15:11:38 +01002630 robj = gem_to_amdgpu_bo(fb->obj[0]);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002631 /* don't unpin kernel fb objects */
2632 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
Alex Xie7a6901d2017-04-24 13:52:41 -04002633 r = amdgpu_bo_reserve(robj, true);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002634 if (r == 0) {
2635 amdgpu_bo_unpin(robj);
2636 amdgpu_bo_unreserve(robj);
2637 }
2638 }
2639 }
2640 /* evict vram memory */
2641 amdgpu_bo_evict_vram(adev);
2642
Alex Deucher5ceb54c2015-08-05 12:41:48 -04002643 amdgpu_fence_driver_suspend(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002644
Alex Deuchercdd61df2017-12-14 16:47:40 -05002645 r = amdgpu_device_ip_suspend(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002646
Alex Deuchera0a71e42016-10-10 12:41:36 -04002647 /* evict remaining vram memory
2648 * This second call to evict vram is to evict the gart page table
2649 * using the CPU.
2650 */
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002651 amdgpu_bo_evict_vram(adev);
2652
2653 pci_save_state(dev->pdev);
2654 if (suspend) {
2655 /* Shut down the device */
2656 pci_disable_device(dev->pdev);
2657 pci_set_power_state(dev->pdev, PCI_D3hot);
jimqu74b0b152016-09-07 17:09:12 +08002658 } else {
2659 r = amdgpu_asic_reset(adev);
2660 if (r)
2661 DRM_ERROR("amdgpu asic reset failed\n");
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002662 }
2663
2664 if (fbcon) {
2665 console_lock();
2666 amdgpu_fbdev_set_suspend(adev, 1);
2667 console_unlock();
2668 }
2669 return 0;
2670}
2671
2672/**
Alex Deucher810ddc32016-08-23 13:25:49 -04002673 * amdgpu_device_resume - initiate device resume
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002674 *
2675 * @pdev: drm dev pointer
2676 *
2677 * Bring the hw back to operating state (all asics).
2678 * Returns 0 for success or an error on failure.
2679 * Called at driver resume.
2680 */
Alex Deucher810ddc32016-08-23 13:25:49 -04002681int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002682{
2683 struct drm_connector *connector;
2684 struct amdgpu_device *adev = dev->dev_private;
Alex Deucher756e6882015-10-08 00:03:36 -04002685 struct drm_crtc *crtc;
Huang Rui03161a62017-04-13 16:12:26 +08002686 int r = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002687
2688 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2689 return 0;
2690
jimqu74b0b152016-09-07 17:09:12 +08002691 if (fbcon)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002692 console_lock();
jimqu74b0b152016-09-07 17:09:12 +08002693
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002694 if (resume) {
2695 pci_set_power_state(dev->pdev, PCI_D0);
2696 pci_restore_state(dev->pdev);
jimqu74b0b152016-09-07 17:09:12 +08002697 r = pci_enable_device(dev->pdev);
Huang Rui03161a62017-04-13 16:12:26 +08002698 if (r)
2699 goto unlock;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002700 }
2701
2702 /* post card */
Alex Deucher39c640c2017-12-15 16:22:11 -05002703 if (amdgpu_device_need_post(adev)) {
jimqu74b0b152016-09-07 17:09:12 +08002704 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2705 if (r)
2706 DRM_ERROR("amdgpu asic init failed\n");
2707 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002708
Alex Deucher06ec9072017-12-14 15:02:39 -05002709 r = amdgpu_device_ip_resume(adev);
Rex Zhue6707212017-03-30 13:21:01 +08002710 if (r) {
Alex Deucher06ec9072017-12-14 15:02:39 -05002711 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
Huang Rui03161a62017-04-13 16:12:26 +08002712 goto unlock;
Rex Zhue6707212017-03-30 13:21:01 +08002713 }
Alex Deucher5ceb54c2015-08-05 12:41:48 -04002714 amdgpu_fence_driver_resume(adev);
2715
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002716
Alex Deucher06ec9072017-12-14 15:02:39 -05002717 r = amdgpu_device_ip_late_init(adev);
Huang Rui03161a62017-04-13 16:12:26 +08002718 if (r)
2719 goto unlock;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002720
Alex Deucher756e6882015-10-08 00:03:36 -04002721 /* pin cursors */
2722 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2723 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2724
2725 if (amdgpu_crtc->cursor_bo) {
2726 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
Alex Xie7a6901d2017-04-24 13:52:41 -04002727 r = amdgpu_bo_reserve(aobj, true);
Alex Deucher756e6882015-10-08 00:03:36 -04002728 if (r == 0) {
2729 r = amdgpu_bo_pin(aobj,
2730 AMDGPU_GEM_DOMAIN_VRAM,
2731 &amdgpu_crtc->cursor_addr);
2732 if (r != 0)
2733 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2734 amdgpu_bo_unreserve(aobj);
2735 }
2736 }
2737 }
Yong Zhaoba997702015-11-09 17:21:45 -05002738 r = amdgpu_amdkfd_resume(adev);
2739 if (r)
2740 return r;
Alex Deucher756e6882015-10-08 00:03:36 -04002741
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002742 /* blat the mode back in */
2743 if (fbcon) {
Harry Wentland45622362017-09-12 15:58:20 -04002744 if (!amdgpu_device_has_dc_support(adev)) {
2745 /* pre DCE11 */
2746 drm_helper_resume_force_mode(dev);
2747
2748 /* turn on display hw */
2749 drm_modeset_lock_all(dev);
2750 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2751 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2752 }
2753 drm_modeset_unlock_all(dev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002754 }
2755 }
2756
2757 drm_kms_helper_poll_enable(dev);
Lyude23a1a9e2016-07-18 11:41:37 -04002758
2759 /*
2760 * Most of the connector probing functions try to acquire runtime pm
2761 * refs to ensure that the GPU is powered on when connector polling is
2762 * performed. Since we're calling this from a runtime PM callback,
2763 * trying to acquire rpm refs will cause us to deadlock.
2764 *
2765 * Since we're guaranteed to be holding the rpm lock, it's safe to
2766 * temporarily disable the rpm helpers so this doesn't deadlock us.
2767 */
2768#ifdef CONFIG_PM
2769 dev->dev->power.disable_depth++;
2770#endif
Harry Wentland45622362017-09-12 15:58:20 -04002771 if (!amdgpu_device_has_dc_support(adev))
2772 drm_helper_hpd_irq_event(dev);
2773 else
2774 drm_kms_helper_hotplug_event(dev);
Lyude23a1a9e2016-07-18 11:41:37 -04002775#ifdef CONFIG_PM
2776 dev->dev->power.disable_depth--;
2777#endif
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002778
Huang Rui03161a62017-04-13 16:12:26 +08002779 if (fbcon)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002780 amdgpu_fbdev_set_suspend(adev, 0);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002781
Huang Rui03161a62017-04-13 16:12:26 +08002782unlock:
2783 if (fbcon)
2784 console_unlock();
2785
2786 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002787}
2788
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002789/**
2790 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
2791 *
2792 * @adev: amdgpu_device pointer
2793 *
2794 * The list of all the hardware IPs that make up the asic is walked and
2795 * the check_soft_reset callbacks are run. check_soft_reset determines
2796 * if the asic is still hung or not.
2797 * Returns true if any of the IPs are still in a hung state, false if not.
2798 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002799static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
Chunming Zhou63fbf422016-07-15 11:19:20 +08002800{
2801 int i;
2802 bool asic_hang = false;
2803
Monk Liuf993d622017-10-16 19:46:01 +08002804 if (amdgpu_sriov_vf(adev))
2805 return true;
2806
Alex Deucher8bc04c22018-03-29 14:48:37 -05002807 if (amdgpu_asic_need_full_reset(adev))
2808 return true;
2809
Chunming Zhou63fbf422016-07-15 11:19:20 +08002810 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04002811 if (!adev->ip_blocks[i].status.valid)
Chunming Zhou63fbf422016-07-15 11:19:20 +08002812 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002813 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2814 adev->ip_blocks[i].status.hang =
2815 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2816 if (adev->ip_blocks[i].status.hang) {
2817 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
Chunming Zhou63fbf422016-07-15 11:19:20 +08002818 asic_hang = true;
2819 }
2820 }
2821 return asic_hang;
2822}
2823
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002824/**
2825 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
2826 *
2827 * @adev: amdgpu_device pointer
2828 *
2829 * The list of all the hardware IPs that make up the asic is walked and the
2830 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
2831 * handles any IP specific hardware or software state changes that are
2832 * necessary for a soft reset to succeed.
2833 * Returns 0 on success, negative error code on failure.
2834 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002835static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
Chunming Zhoud31a5012016-07-18 10:04:34 +08002836{
2837 int i, r = 0;
2838
2839 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04002840 if (!adev->ip_blocks[i].status.valid)
Chunming Zhoud31a5012016-07-18 10:04:34 +08002841 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002842 if (adev->ip_blocks[i].status.hang &&
2843 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2844 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
Chunming Zhoud31a5012016-07-18 10:04:34 +08002845 if (r)
2846 return r;
2847 }
2848 }
2849
2850 return 0;
2851}
2852
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002853/**
2854 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
2855 *
2856 * @adev: amdgpu_device pointer
2857 *
2858 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
2859 * reset is necessary to recover.
2860 * Returns true if a full asic reset is required, false if not.
2861 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002862static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
Chunming Zhou35d782f2016-07-15 15:57:13 +08002863{
Alex Deucherda146d32016-10-13 16:07:03 -04002864 int i;
2865
Alex Deucher8bc04c22018-03-29 14:48:37 -05002866 if (amdgpu_asic_need_full_reset(adev))
2867 return true;
2868
Alex Deucherda146d32016-10-13 16:07:03 -04002869 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04002870 if (!adev->ip_blocks[i].status.valid)
Alex Deucherda146d32016-10-13 16:07:03 -04002871 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002872 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2873 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2874 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
Ken Wang98512bb2017-09-14 16:25:19 +08002875 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2876 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
Alex Deuchera1255102016-10-13 17:41:13 -04002877 if (adev->ip_blocks[i].status.hang) {
Alex Deucherda146d32016-10-13 16:07:03 -04002878 DRM_INFO("Some block need full reset!\n");
2879 return true;
2880 }
2881 }
Chunming Zhou35d782f2016-07-15 15:57:13 +08002882 }
2883 return false;
2884}
2885
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002886/**
2887 * amdgpu_device_ip_soft_reset - do a soft reset
2888 *
2889 * @adev: amdgpu_device pointer
2890 *
2891 * The list of all the hardware IPs that make up the asic is walked and the
2892 * soft_reset callbacks are run if the block is hung. soft_reset handles any
2893 * IP specific hardware or software state changes that are necessary to soft
2894 * reset the IP.
2895 * Returns 0 on success, negative error code on failure.
2896 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002897static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
Chunming Zhou35d782f2016-07-15 15:57:13 +08002898{
2899 int i, r = 0;
2900
2901 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04002902 if (!adev->ip_blocks[i].status.valid)
Chunming Zhou35d782f2016-07-15 15:57:13 +08002903 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002904 if (adev->ip_blocks[i].status.hang &&
2905 adev->ip_blocks[i].version->funcs->soft_reset) {
2906 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
Chunming Zhou35d782f2016-07-15 15:57:13 +08002907 if (r)
2908 return r;
2909 }
2910 }
2911
2912 return 0;
2913}
2914
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002915/**
2916 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
2917 *
2918 * @adev: amdgpu_device pointer
2919 *
2920 * The list of all the hardware IPs that make up the asic is walked and the
2921 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
2922 * handles any IP specific hardware or software state changes that are
2923 * necessary after the IP has been soft reset.
2924 * Returns 0 on success, negative error code on failure.
2925 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002926static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
Chunming Zhou35d782f2016-07-15 15:57:13 +08002927{
2928 int i, r = 0;
2929
2930 for (i = 0; i < adev->num_ip_blocks; i++) {
Alex Deuchera1255102016-10-13 17:41:13 -04002931 if (!adev->ip_blocks[i].status.valid)
Chunming Zhou35d782f2016-07-15 15:57:13 +08002932 continue;
Alex Deuchera1255102016-10-13 17:41:13 -04002933 if (adev->ip_blocks[i].status.hang &&
2934 adev->ip_blocks[i].version->funcs->post_soft_reset)
2935 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
Chunming Zhou35d782f2016-07-15 15:57:13 +08002936 if (r)
2937 return r;
2938 }
2939
2940 return 0;
2941}
2942
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002943/**
2944 * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers
2945 *
2946 * @adev: amdgpu_device pointer
2947 * @ring: amdgpu_ring for the engine handling the buffer operations
2948 * @bo: amdgpu_bo buffer whose shadow is being restored
2949 * @fence: dma_fence associated with the operation
2950 *
2951 * Restores the VRAM buffer contents from the shadow in GTT. Used to
2952 * restore things like GPUVM page tables after a GPU reset where
2953 * the contents of VRAM might be lost.
2954 * Returns 0 on success, negative error code on failure.
2955 */
Alex Deucher06ec9072017-12-14 15:02:39 -05002956static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
2957 struct amdgpu_ring *ring,
2958 struct amdgpu_bo *bo,
2959 struct dma_fence **fence)
Chunming Zhou53cdccd2016-07-21 17:20:52 +08002960{
2961 uint32_t domain;
2962 int r;
2963
Roger.He23d2e502017-04-21 14:24:26 +08002964 if (!bo->shadow)
2965 return 0;
Chunming Zhou53cdccd2016-07-21 17:20:52 +08002966
Alex Xie1d284792017-04-24 13:53:04 -04002967 r = amdgpu_bo_reserve(bo, true);
Roger.He23d2e502017-04-21 14:24:26 +08002968 if (r)
2969 return r;
2970 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2971 /* if bo has been evicted, then no need to recover */
2972 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
Roger.He82521312017-04-21 13:08:43 +08002973 r = amdgpu_bo_validate(bo->shadow);
2974 if (r) {
2975 DRM_ERROR("bo validate failed!\n");
2976 goto err;
2977 }
2978
Roger.He23d2e502017-04-21 14:24:26 +08002979 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
Chunming Zhou53cdccd2016-07-21 17:20:52 +08002980 NULL, fence, true);
Roger.He23d2e502017-04-21 14:24:26 +08002981 if (r) {
2982 DRM_ERROR("recover page table failed!\n");
2983 goto err;
2984 }
2985 }
Chunming Zhou53cdccd2016-07-21 17:20:52 +08002986err:
Roger.He23d2e502017-04-21 14:24:26 +08002987 amdgpu_bo_unreserve(bo);
2988 return r;
Chunming Zhou53cdccd2016-07-21 17:20:52 +08002989}
2990
Alex Deuchere3ecdff2018-03-15 17:39:45 -05002991/**
2992 * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents
2993 *
2994 * @adev: amdgpu_device pointer
2995 *
2996 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
2997 * restore things like GPUVM page tables after a GPU reset where
2998 * the contents of VRAM might be lost.
2999 * Returns 0 on success, 1 on failure.
3000 */
Monk Liuc41d1cf2017-12-25 11:59:27 +08003001static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
3002{
3003 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
3004 struct amdgpu_bo *bo, *tmp;
3005 struct dma_fence *fence = NULL, *next = NULL;
3006 long r = 1;
3007 int i = 0;
3008 long tmo;
3009
3010 if (amdgpu_sriov_runtime(adev))
3011 tmo = msecs_to_jiffies(amdgpu_lockup_timeout);
3012 else
3013 tmo = msecs_to_jiffies(100);
3014
3015 DRM_INFO("recover vram bo from shadow start\n");
3016 mutex_lock(&adev->shadow_list_lock);
3017 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
3018 next = NULL;
3019 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
3020 if (fence) {
3021 r = dma_fence_wait_timeout(fence, false, tmo);
3022 if (r == 0)
3023 pr_err("wait fence %p[%d] timeout\n", fence, i);
3024 else if (r < 0)
3025 pr_err("wait fence %p[%d] interrupted\n", fence, i);
3026 if (r < 1) {
3027 dma_fence_put(fence);
3028 fence = next;
3029 break;
3030 }
3031 i++;
3032 }
3033
3034 dma_fence_put(fence);
3035 fence = next;
3036 }
3037 mutex_unlock(&adev->shadow_list_lock);
3038
3039 if (fence) {
3040 r = dma_fence_wait_timeout(fence, false, tmo);
3041 if (r == 0)
3042 pr_err("wait fence %p[%d] timeout\n", fence, i);
3043 else if (r < 0)
3044 pr_err("wait fence %p[%d] interrupted\n", fence, i);
3045
3046 }
3047 dma_fence_put(fence);
3048
3049 if (r > 0)
3050 DRM_INFO("recover vram bo from shadow done\n");
3051 else
3052 DRM_ERROR("recover vram bo from shadow failed\n");
3053
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003054 return (r > 0) ? 0 : 1;
Monk Liuc41d1cf2017-12-25 11:59:27 +08003055}
3056
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003057/**
Alex Deucher06ec9072017-12-14 15:02:39 -05003058 * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
Monk Liua90ad3c2017-01-23 14:22:08 +08003059 *
3060 * @adev: amdgpu device pointer
Monk Liua90ad3c2017-01-23 14:22:08 +08003061 *
Monk Liu57406822017-10-25 16:37:02 +08003062 * attempt to do soft-reset or full-reset and reinitialize Asic
3063 * return 0 means successed otherwise failed
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003064 */
Monk Liuc41d1cf2017-12-25 11:59:27 +08003065static int amdgpu_device_reset(struct amdgpu_device *adev)
Monk Liua90ad3c2017-01-23 14:22:08 +08003066{
Monk Liu57406822017-10-25 16:37:02 +08003067 bool need_full_reset, vram_lost = 0;
3068 int r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003069
Alex Deucher06ec9072017-12-14 15:02:39 -05003070 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
Chunming Zhou35d782f2016-07-15 15:57:13 +08003071
3072 if (!need_full_reset) {
Alex Deucher06ec9072017-12-14 15:02:39 -05003073 amdgpu_device_ip_pre_soft_reset(adev);
3074 r = amdgpu_device_ip_soft_reset(adev);
3075 amdgpu_device_ip_post_soft_reset(adev);
3076 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
Chunming Zhou35d782f2016-07-15 15:57:13 +08003077 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3078 need_full_reset = true;
3079 }
3080 }
3081
3082 if (need_full_reset) {
Alex Deuchercdd61df2017-12-14 16:47:40 -05003083 r = amdgpu_device_ip_suspend(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003084
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003085retry:
Chunming Zhou35d782f2016-07-15 15:57:13 +08003086 r = amdgpu_asic_reset(adev);
3087 /* post card */
3088 amdgpu_atom_asic_init(adev->mode_info.atom_context);
Alex Deucherbfa99262016-01-15 11:59:48 -05003089
Chunming Zhou35d782f2016-07-15 15:57:13 +08003090 if (!r) {
3091 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
Alex Deucher06ec9072017-12-14 15:02:39 -05003092 r = amdgpu_device_ip_resume_phase1(adev);
Chunming Zhoufcf06492017-05-05 10:33:33 +08003093 if (r)
3094 goto out;
Monk Liu57406822017-10-25 16:37:02 +08003095
Alex Deucher06ec9072017-12-14 15:02:39 -05003096 vram_lost = amdgpu_device_check_vram_lost(adev);
Chunming Zhouf1892132017-05-15 16:48:27 +08003097 if (vram_lost) {
Chunming Zhou0c49e0b2017-05-15 14:20:00 +08003098 DRM_ERROR("VRAM is lost!\n");
Chunming Zhouf1892132017-05-15 16:48:27 +08003099 atomic_inc(&adev->vram_lost_counter);
3100 }
Monk Liu57406822017-10-25 16:37:02 +08003101
Christian Königc1c7ce82017-10-16 16:50:32 +02003102 r = amdgpu_gtt_mgr_recover(
3103 &adev->mman.bdev.man[TTM_PL_TT]);
Chunming Zhou2c0d7312016-08-30 16:36:25 +08003104 if (r)
Chunming Zhoufcf06492017-05-05 10:33:33 +08003105 goto out;
Monk Liu57406822017-10-25 16:37:02 +08003106
Alex Deucher06ec9072017-12-14 15:02:39 -05003107 r = amdgpu_device_ip_resume_phase2(adev);
Chunming Zhoufcf06492017-05-05 10:33:33 +08003108 if (r)
3109 goto out;
Monk Liu57406822017-10-25 16:37:02 +08003110
Chunming Zhou0c49e0b2017-05-15 14:20:00 +08003111 if (vram_lost)
Alex Deucher06ec9072017-12-14 15:02:39 -05003112 amdgpu_device_fill_reset_magic(adev);
Chunming Zhou2c0d7312016-08-30 16:36:25 +08003113 }
Chunming Zhoufcf06492017-05-05 10:33:33 +08003114 }
Monk Liu57406822017-10-25 16:37:02 +08003115
Chunming Zhoufcf06492017-05-05 10:33:33 +08003116out:
3117 if (!r) {
3118 amdgpu_irq_gpu_reset_resume_helper(adev);
Chunming Zhou1f465082016-06-30 15:02:26 +08003119 r = amdgpu_ib_ring_tests(adev);
3120 if (r) {
3121 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
Alex Deuchercdd61df2017-12-14 16:47:40 -05003122 r = amdgpu_device_ip_suspend(adev);
Chunming Zhou53cdccd2016-07-21 17:20:52 +08003123 need_full_reset = true;
Chunming Zhou40019dc2016-06-29 16:01:49 +08003124 goto retry;
Chunming Zhou1f465082016-06-30 15:02:26 +08003125 }
Monk Liu57406822017-10-25 16:37:02 +08003126 }
3127
Monk Liuc41d1cf2017-12-25 11:59:27 +08003128 if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost))
3129 r = amdgpu_device_handle_vram_lost(adev);
Monk Liu57406822017-10-25 16:37:02 +08003130
3131 return r;
3132}
3133
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003134/**
Alex Deucher06ec9072017-12-14 15:02:39 -05003135 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
Monk Liu57406822017-10-25 16:37:02 +08003136 *
3137 * @adev: amdgpu device pointer
Monk Liu57406822017-10-25 16:37:02 +08003138 *
3139 * do VF FLR and reinitialize Asic
3140 * return 0 means successed otherwise failed
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003141 */
3142static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3143 bool from_hypervisor)
Monk Liu57406822017-10-25 16:37:02 +08003144{
3145 int r;
3146
3147 if (from_hypervisor)
3148 r = amdgpu_virt_request_full_gpu(adev, true);
3149 else
3150 r = amdgpu_virt_reset_gpu(adev);
3151 if (r)
3152 return r;
3153
3154 /* Resume IP prior to SMC */
Alex Deucher06ec9072017-12-14 15:02:39 -05003155 r = amdgpu_device_ip_reinit_early_sriov(adev);
Monk Liu57406822017-10-25 16:37:02 +08003156 if (r)
3157 goto error;
3158
3159 /* we need recover gart prior to run SMC/CP/SDMA resume */
Christian Königc1c7ce82017-10-16 16:50:32 +02003160 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
Monk Liu57406822017-10-25 16:37:02 +08003161
3162 /* now we are okay to resume SMC/CP/SDMA */
Alex Deucher06ec9072017-12-14 15:02:39 -05003163 r = amdgpu_device_ip_reinit_late_sriov(adev);
Monk Liu57406822017-10-25 16:37:02 +08003164 if (r)
3165 goto error;
3166
3167 amdgpu_irq_gpu_reset_resume_helper(adev);
3168 r = amdgpu_ib_ring_tests(adev);
Monk Liuc41d1cf2017-12-25 11:59:27 +08003169
Emily Dengabc34252018-04-26 18:02:55 +08003170error:
3171 amdgpu_virt_release_full_gpu(adev, true);
Monk Liuc41d1cf2017-12-25 11:59:27 +08003172 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3173 atomic_inc(&adev->vram_lost_counter);
3174 r = amdgpu_device_handle_vram_lost(adev);
3175 }
Monk Liu57406822017-10-25 16:37:02 +08003176
Monk Liu57406822017-10-25 16:37:02 +08003177 return r;
3178}
3179
3180/**
Alex Deucher5f152b52017-12-15 16:40:49 -05003181 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
Monk Liu57406822017-10-25 16:37:02 +08003182 *
3183 * @adev: amdgpu device pointer
3184 * @job: which job trigger hang
Andrey Grodzovskydcebf022017-12-12 14:09:30 -05003185 * @force forces reset regardless of amdgpu_gpu_recovery
Monk Liu57406822017-10-25 16:37:02 +08003186 *
3187 * Attempt to reset the GPU if it has hung (all asics).
3188 * Returns 0 for success or an error on failure.
3189 */
Alex Deucher5f152b52017-12-15 16:40:49 -05003190int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3191 struct amdgpu_job *job, bool force)
Monk Liu57406822017-10-25 16:37:02 +08003192{
Monk Liu57406822017-10-25 16:37:02 +08003193 int i, r, resched;
3194
Andrey Grodzovsky54bc1392018-01-19 17:23:08 -05003195 if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
Monk Liu57406822017-10-25 16:37:02 +08003196 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
3197 return 0;
3198 }
3199
Andrey Grodzovskydcebf022017-12-12 14:09:30 -05003200 if (!force && (amdgpu_gpu_recovery == 0 ||
3201 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))) {
3202 DRM_INFO("GPU recovery disabled.\n");
3203 return 0;
3204 }
3205
Monk Liu57406822017-10-25 16:37:02 +08003206 dev_info(adev->dev, "GPU reset begin!\n");
3207
Monk Liu13a752e2017-10-17 15:11:12 +08003208 mutex_lock(&adev->lock_reset);
Monk Liu57406822017-10-25 16:37:02 +08003209 atomic_inc(&adev->gpu_reset_counter);
Monk Liu13a752e2017-10-17 15:11:12 +08003210 adev->in_gpu_reset = 1;
Monk Liu57406822017-10-25 16:37:02 +08003211
3212 /* block TTM */
3213 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
Monk Liu71182662017-12-25 15:14:58 +08003214
Monk Liu71182662017-12-25 15:14:58 +08003215 /* block all schedulers and reset given job's ring */
Monk Liu57406822017-10-25 16:37:02 +08003216 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3217 struct amdgpu_ring *ring = adev->rings[i];
3218
3219 if (!ring || !ring->sched.thread)
3220 continue;
3221
Monk Liu71182662017-12-25 15:14:58 +08003222 kthread_park(ring->sched.thread);
3223
Monk Liu57406822017-10-25 16:37:02 +08003224 if (job && job->ring->idx != i)
3225 continue;
3226
Lucas Stach1b1f42d2017-12-06 17:49:39 +01003227 drm_sched_hw_job_reset(&ring->sched, &job->base);
Monk Liu57406822017-10-25 16:37:02 +08003228
3229 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3230 amdgpu_fence_driver_force_completion(ring);
3231 }
3232
3233 if (amdgpu_sriov_vf(adev))
Monk Liuc41d1cf2017-12-25 11:59:27 +08003234 r = amdgpu_device_reset_sriov(adev, job ? false : true);
Monk Liu57406822017-10-25 16:37:02 +08003235 else
Monk Liuc41d1cf2017-12-25 11:59:27 +08003236 r = amdgpu_device_reset(adev);
Monk Liu57406822017-10-25 16:37:02 +08003237
Monk Liu71182662017-12-25 15:14:58 +08003238 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3239 struct amdgpu_ring *ring = adev->rings[i];
Chunming Zhou51687752017-04-24 17:09:15 +08003240
Monk Liu71182662017-12-25 15:14:58 +08003241 if (!ring || !ring->sched.thread)
3242 continue;
Chunming Zhou53cdccd2016-07-21 17:20:52 +08003243
Monk Liu71182662017-12-25 15:14:58 +08003244 /* only need recovery sched of the given job's ring
3245 * or all rings (in the case @job is NULL)
3246 * after above amdgpu_reset accomplished
3247 */
3248 if ((!job || job->ring->idx == i) && !r)
Lucas Stach1b1f42d2017-12-06 17:49:39 +01003249 drm_sched_job_recovery(&ring->sched);
Monk Liu57406822017-10-25 16:37:02 +08003250
Monk Liu71182662017-12-25 15:14:58 +08003251 kthread_unpark(ring->sched.thread);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003252 }
3253
Andrey Grodzovskybf830602018-05-17 11:18:34 -04003254 if (!amdgpu_device_has_dc_support(adev)) {
Harry Wentland45622362017-09-12 15:58:20 -04003255 drm_helper_resume_force_mode(adev->ddev);
Monk Liu57406822017-10-25 16:37:02 +08003256 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003257
3258 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
Monk Liu57406822017-10-25 16:37:02 +08003259
Gavin Wan89041942017-06-23 13:55:15 -04003260 if (r) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003261 /* bad news, how to tell it to userspace ? */
Monk Liu57406822017-10-25 16:37:02 +08003262 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3263 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3264 } else {
3265 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
Gavin Wan89041942017-06-23 13:55:15 -04003266 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003267
Gavin Wan89041942017-06-23 13:55:15 -04003268 amdgpu_vf_error_trans_all(adev);
Monk Liu13a752e2017-10-17 15:11:12 +08003269 adev->in_gpu_reset = 0;
3270 mutex_unlock(&adev->lock_reset);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003271 return r;
3272}
3273
Alex Deuchere3ecdff2018-03-15 17:39:45 -05003274/**
3275 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3276 *
3277 * @adev: amdgpu_device pointer
3278 *
3279 * Fetchs and stores in the driver the PCIE capabilities (gen speed
3280 * and lanes) of the slot the device is in. Handles APUs and
3281 * virtualized environments where PCIE config space may not be available.
3282 */
Alex Deucher5494d862018-03-09 15:14:11 -05003283static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
Alex Deucherd0dd7f02015-11-11 19:45:06 -05003284{
3285 u32 mask;
3286 int ret;
3287
Alex Deuchercd474ba2016-02-04 10:21:23 -05003288 if (amdgpu_pcie_gen_cap)
3289 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3290
3291 if (amdgpu_pcie_lane_cap)
3292 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3293
3294 /* covers APUs as well */
3295 if (pci_is_root_bus(adev->pdev->bus)) {
3296 if (adev->pm.pcie_gen_mask == 0)
3297 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3298 if (adev->pm.pcie_mlw_mask == 0)
3299 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
Alex Deucherd0dd7f02015-11-11 19:45:06 -05003300 return;
Alex Deucherd0dd7f02015-11-11 19:45:06 -05003301 }
Alex Deuchercd474ba2016-02-04 10:21:23 -05003302
3303 if (adev->pm.pcie_gen_mask == 0) {
3304 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
3305 if (!ret) {
3306 adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3307 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3308 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3309
3310 if (mask & DRM_PCIE_SPEED_25)
3311 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3312 if (mask & DRM_PCIE_SPEED_50)
3313 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
3314 if (mask & DRM_PCIE_SPEED_80)
3315 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
3316 } else {
3317 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3318 }
3319 }
3320 if (adev->pm.pcie_mlw_mask == 0) {
3321 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3322 if (!ret) {
3323 switch (mask) {
3324 case 32:
3325 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3326 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3327 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3328 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3329 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3330 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3331 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3332 break;
3333 case 16:
3334 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3335 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3336 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3337 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3338 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3339 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3340 break;
3341 case 12:
3342 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3343 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3344 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3345 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3346 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3347 break;
3348 case 8:
3349 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3350 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3351 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3352 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3353 break;
3354 case 4:
3355 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3356 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3357 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3358 break;
3359 case 2:
3360 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3361 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3362 break;
3363 case 1:
3364 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3365 break;
3366 default:
3367 break;
3368 }
3369 } else {
3370 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
Alex Deucherd0dd7f02015-11-11 19:45:06 -05003371 }
3372 }
3373}
Alex Deucherd38ceaf2015-04-20 16:55:21 -04003374