1 /*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28
29 #include "amdgpu.h"
30 #include "amdgpu_ucode.h"
31 #include "amdgpu_trace.h"
32
33 #include "gc/gc_10_3_0_offset.h"
34 #include "gc/gc_10_3_0_sh_mask.h"
35 #include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h"
36 #include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h"
37 #include "ivsrcid/sdma2/irqsrcs_sdma2_5_0.h"
38 #include "ivsrcid/sdma3/irqsrcs_sdma3_5_0.h"
39
40 #include "soc15_common.h"
41 #include "soc15.h"
42 #include "navi10_sdma_pkt_open.h"
43 #include "nbio_v2_3.h"
44 #include "sdma_common.h"
45 #include "sdma_v5_2.h"
46
47 MODULE_FIRMWARE("amdgpu/sienna_cichlid_sdma.bin");
48 MODULE_FIRMWARE("amdgpu/navy_flounder_sdma.bin");
49 MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_sdma.bin");
50 MODULE_FIRMWARE("amdgpu/beige_goby_sdma.bin");
51
52 MODULE_FIRMWARE("amdgpu/vangogh_sdma.bin");
53 MODULE_FIRMWARE("amdgpu/yellow_carp_sdma.bin");
54 MODULE_FIRMWARE("amdgpu/sdma_5_2_6.bin");
55 MODULE_FIRMWARE("amdgpu/sdma_5_2_7.bin");
56
57 #define SDMA1_REG_OFFSET 0x600
58 #define SDMA3_REG_OFFSET 0x400
59 #define SDMA0_HYP_DEC_REG_START 0x5880
60 #define SDMA0_HYP_DEC_REG_END 0x5893
61 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
62
63 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev);
64 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev);
65 static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev);
66 static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev);
67
sdma_v5_2_get_reg_offset(struct amdgpu_device * adev,u32 instance,u32 internal_offset)68 static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
69 {
70 u32 base;
71
72 if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
73 internal_offset <= SDMA0_HYP_DEC_REG_END) {
74 base = adev->reg_offset[GC_HWIP][0][1];
75 if (instance != 0)
76 internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
77 } else {
78 if (instance < 2) {
79 base = adev->reg_offset[GC_HWIP][0][0];
80 if (instance == 1)
81 internal_offset += SDMA1_REG_OFFSET;
82 } else {
83 base = adev->reg_offset[GC_HWIP][0][2];
84 if (instance == 3)
85 internal_offset += SDMA3_REG_OFFSET;
86 }
87 }
88
89 return base + internal_offset;
90 }
91
sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring * ring)92 static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring)
93 {
94 unsigned ret;
95
96 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE));
97 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
98 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
99 amdgpu_ring_write(ring, 1);
100 ret = ring->wptr & ring->buf_mask;/* this is the offset we need patch later */
101 amdgpu_ring_write(ring, 0x55aa55aa);/* insert dummy here and patch it later */
102
103 return ret;
104 }
105
sdma_v5_2_ring_patch_cond_exec(struct amdgpu_ring * ring,unsigned offset)106 static void sdma_v5_2_ring_patch_cond_exec(struct amdgpu_ring *ring,
107 unsigned offset)
108 {
109 unsigned cur;
110
111 BUG_ON(offset > ring->buf_mask);
112 BUG_ON(ring->ring[offset] != 0x55aa55aa);
113
114 cur = (ring->wptr - 1) & ring->buf_mask;
115 if (cur > offset)
116 ring->ring[offset] = cur - offset;
117 else
118 ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
119 }
120
121 /**
122 * sdma_v5_2_ring_get_rptr - get the current read pointer
123 *
124 * @ring: amdgpu ring pointer
125 *
126 * Get the current rptr from the hardware (NAVI10+).
127 */
sdma_v5_2_ring_get_rptr(struct amdgpu_ring * ring)128 static uint64_t sdma_v5_2_ring_get_rptr(struct amdgpu_ring *ring)
129 {
130 u64 *rptr;
131
132 /* XXX check if swapping is necessary on BE */
133 rptr = (u64 *)ring->rptr_cpu_addr;
134
135 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
136 return ((*rptr) >> 2);
137 }
138
139 /**
140 * sdma_v5_2_ring_get_wptr - get the current write pointer
141 *
142 * @ring: amdgpu ring pointer
143 *
144 * Get the current wptr from the hardware (NAVI10+).
145 */
sdma_v5_2_ring_get_wptr(struct amdgpu_ring * ring)146 static uint64_t sdma_v5_2_ring_get_wptr(struct amdgpu_ring *ring)
147 {
148 struct amdgpu_device *adev = ring->adev;
149 u64 wptr;
150
151 if (ring->use_doorbell) {
152 /* XXX check if swapping is necessary on BE */
153 wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
154 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
155 } else {
156 wptr = RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI));
157 wptr = wptr << 32;
158 wptr |= RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR));
159 DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr);
160 }
161
162 return wptr >> 2;
163 }
164
165 /**
166 * sdma_v5_2_ring_set_wptr - commit the write pointer
167 *
168 * @ring: amdgpu ring pointer
169 *
170 * Write the wptr back to the hardware (NAVI10+).
171 */
sdma_v5_2_ring_set_wptr(struct amdgpu_ring * ring)172 static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring)
173 {
174 struct amdgpu_device *adev = ring->adev;
175
176 DRM_DEBUG("Setting write pointer\n");
177 if (ring->use_doorbell) {
178 DRM_DEBUG("Using doorbell -- "
179 "wptr_offs == 0x%08x "
180 "lower_32_bits(ring->wptr << 2) == 0x%08x "
181 "upper_32_bits(ring->wptr << 2) == 0x%08x\n",
182 ring->wptr_offs,
183 lower_32_bits(ring->wptr << 2),
184 upper_32_bits(ring->wptr << 2));
185 /* XXX check if swapping is necessary on BE */
186 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
187 ring->wptr << 2);
188 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
189 ring->doorbell_index, ring->wptr << 2);
190 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
191 } else {
192 DRM_DEBUG("Not using doorbell -- "
193 "mmSDMA%i_GFX_RB_WPTR == 0x%08x "
194 "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
195 ring->me,
196 lower_32_bits(ring->wptr << 2),
197 ring->me,
198 upper_32_bits(ring->wptr << 2));
199 WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR),
200 lower_32_bits(ring->wptr << 2));
201 WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI),
202 upper_32_bits(ring->wptr << 2));
203 }
204 }
205
sdma_v5_2_ring_insert_nop(struct amdgpu_ring * ring,uint32_t count)206 static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
207 {
208 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
209 int i;
210
211 for (i = 0; i < count; i++)
212 if (sdma && sdma->burst_nop && (i == 0))
213 amdgpu_ring_write(ring, ring->funcs->nop |
214 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
215 else
216 amdgpu_ring_write(ring, ring->funcs->nop);
217 }
218
219 /**
220 * sdma_v5_2_ring_emit_ib - Schedule an IB on the DMA engine
221 *
222 * @ring: amdgpu ring pointer
223 * @job: job to retrieve vmid from
224 * @ib: IB object to schedule
225 * @flags: unused
226 *
227 * Schedule an IB in the DMA ring.
228 */
sdma_v5_2_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)229 static void sdma_v5_2_ring_emit_ib(struct amdgpu_ring *ring,
230 struct amdgpu_job *job,
231 struct amdgpu_ib *ib,
232 uint32_t flags)
233 {
234 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
235 uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
236
237 /* An IB packet must end on a 8 DW boundary--the next dword
238 * must be on a 8-dword boundary. Our IB packet below is 6
239 * dwords long, thus add x number of NOPs, such that, in
240 * modular arithmetic,
241 * wptr + 6 + x = 8k, k >= 0, which in C is,
242 * (wptr + 6 + x) % 8 = 0.
243 * The expression below, is a solution of x.
244 */
245 sdma_v5_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
246
247 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
248 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
249 /* base must be 32 byte aligned */
250 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
251 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
252 amdgpu_ring_write(ring, ib->length_dw);
253 amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
254 amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
255 }
256
257 /**
258 * sdma_v5_2_ring_emit_mem_sync - flush the IB by graphics cache rinse
259 *
260 * @ring: amdgpu ring pointer
261 *
262 * flush the IB by graphics cache rinse.
263 */
sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring * ring)264 static void sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring *ring)
265 {
266 uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB |
267 SDMA_GCR_GLM_INV | SDMA_GCR_GL1_INV |
268 SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
269 SDMA_GCR_GLI_INV(1);
270
271 /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
272 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
273 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
274 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
275 SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
276 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
277 SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
278 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
279 SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
280 }
281
282 /**
283 * sdma_v5_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
284 *
285 * @ring: amdgpu ring pointer
286 *
287 * Emit an hdp flush packet on the requested DMA ring.
288 */
sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring * ring)289 static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
290 {
291 struct amdgpu_device *adev = ring->adev;
292 u32 ref_and_mask = 0;
293 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
294
295 ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
296
297 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
298 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
299 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
300 amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
301 amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
302 amdgpu_ring_write(ring, ref_and_mask); /* reference */
303 amdgpu_ring_write(ring, ref_and_mask); /* mask */
304 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
305 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
306 }
307
308 /**
309 * sdma_v5_2_ring_emit_fence - emit a fence on the DMA ring
310 *
311 * @ring: amdgpu ring pointer
312 * @addr: address
313 * @seq: sequence number
314 * @flags: fence related flags
315 *
316 * Add a DMA fence packet to the ring to write
317 * the fence seq number and DMA trap packet to generate
318 * an interrupt if needed.
319 */
sdma_v5_2_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)320 static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
321 unsigned flags)
322 {
323 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
324 /* write the fence */
325 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
326 SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
327 /* zero in first two bits */
328 BUG_ON(addr & 0x3);
329 amdgpu_ring_write(ring, lower_32_bits(addr));
330 amdgpu_ring_write(ring, upper_32_bits(addr));
331 amdgpu_ring_write(ring, lower_32_bits(seq));
332
333 /* optionally write high bits as well */
334 if (write64bit) {
335 addr += 4;
336 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
337 SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
338 /* zero in first two bits */
339 BUG_ON(addr & 0x3);
340 amdgpu_ring_write(ring, lower_32_bits(addr));
341 amdgpu_ring_write(ring, upper_32_bits(addr));
342 amdgpu_ring_write(ring, upper_32_bits(seq));
343 }
344
345 if ((flags & AMDGPU_FENCE_FLAG_INT)) {
346 uint32_t ctx = ring->is_mes_queue ?
347 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0;
348 /* generate an interrupt */
349 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
350 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx));
351 }
352 }
353
354
355 /**
356 * sdma_v5_2_gfx_stop - stop the gfx async dma engines
357 *
358 * @adev: amdgpu_device pointer
359 *
360 * Stop the gfx async dma ring buffers.
361 */
sdma_v5_2_gfx_stop(struct amdgpu_device * adev)362 static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev)
363 {
364 u32 rb_cntl, ib_cntl;
365 int i;
366
367 amdgpu_sdma_unset_buffer_funcs_helper(adev);
368
369 for (i = 0; i < adev->sdma.num_instances; i++) {
370 rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
371 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
372 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
373 ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
374 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);
375 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
376 }
377 }
378
379 /**
380 * sdma_v5_2_rlc_stop - stop the compute async dma engines
381 *
382 * @adev: amdgpu_device pointer
383 *
384 * Stop the compute async dma queues.
385 */
sdma_v5_2_rlc_stop(struct amdgpu_device * adev)386 static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev)
387 {
388 /* XXX todo */
389 }
390
391 /**
392 * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch
393 *
394 * @adev: amdgpu_device pointer
395 * @enable: enable/disable the DMA MEs context switch.
396 *
397 * Halt or unhalt the async dma engines context switch.
398 */
sdma_v5_2_ctx_switch_enable(struct amdgpu_device * adev,bool enable)399 static void sdma_v5_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
400 {
401 u32 f32_cntl, phase_quantum = 0;
402 int i;
403
404 if (amdgpu_sdma_phase_quantum) {
405 unsigned value = amdgpu_sdma_phase_quantum;
406 unsigned unit = 0;
407
408 while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
409 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
410 value = (value + 1) >> 1;
411 unit++;
412 }
413 if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
414 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
415 value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
416 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
417 unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
418 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
419 WARN_ONCE(1,
420 "clamping sdma_phase_quantum to %uK clock cycles\n",
421 value << unit);
422 }
423 phase_quantum =
424 value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
425 unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
426 }
427
428 for (i = 0; i < adev->sdma.num_instances; i++) {
429 if (enable && amdgpu_sdma_phase_quantum) {
430 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM),
431 phase_quantum);
432 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM),
433 phase_quantum);
434 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM),
435 phase_quantum);
436 }
437
438 if (!amdgpu_sriov_vf(adev)) {
439 f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
440 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
441 AUTO_CTXSW_ENABLE, enable ? 1 : 0);
442 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl);
443 }
444 }
445
446 }
447
448 /**
449 * sdma_v5_2_enable - stop the async dma engines
450 *
451 * @adev: amdgpu_device pointer
452 * @enable: enable/disable the DMA MEs.
453 *
454 * Halt or unhalt the async dma engines.
455 */
sdma_v5_2_enable(struct amdgpu_device * adev,bool enable)456 static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable)
457 {
458 u32 f32_cntl;
459 int i;
460
461 if (!enable) {
462 sdma_v5_2_gfx_stop(adev);
463 sdma_v5_2_rlc_stop(adev);
464 }
465
466 if (!amdgpu_sriov_vf(adev)) {
467 for (i = 0; i < adev->sdma.num_instances; i++) {
468 f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
469 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
470 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
471 }
472 }
473 }
474
475 /**
476 * sdma_v5_2_gfx_resume - setup and start the async dma engines
477 *
478 * @adev: amdgpu_device pointer
479 *
480 * Set up the gfx DMA ring buffers and enable them.
481 * Returns 0 for success, error for failure.
482 */
sdma_v5_2_gfx_resume(struct amdgpu_device * adev)483 static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev)
484 {
485 struct amdgpu_ring *ring;
486 u32 rb_cntl, ib_cntl;
487 u32 rb_bufsz;
488 u32 doorbell;
489 u32 doorbell_offset;
490 u32 temp;
491 u32 wptr_poll_cntl;
492 u64 wptr_gpu_addr;
493 int i, r;
494
495 for (i = 0; i < adev->sdma.num_instances; i++) {
496 ring = &adev->sdma.instance[i].ring;
497
498 if (!amdgpu_sriov_vf(adev))
499 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
500
501 /* Set ring buffer size in dwords */
502 rb_bufsz = order_base_2(ring->ring_size / 4);
503 rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
504 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
505 #ifdef __BIG_ENDIAN
506 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
507 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
508 RPTR_WRITEBACK_SWAP_ENABLE, 1);
509 #endif
510 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
511
512 /* Initialize the ring buffer's read and write pointers */
513 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0);
514 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0);
515 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0);
516 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0);
517
518 /* setup the wptr shadow polling */
519 wptr_gpu_addr = ring->wptr_gpu_addr;
520 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
521 lower_32_bits(wptr_gpu_addr));
522 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
523 upper_32_bits(wptr_gpu_addr));
524 wptr_poll_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i,
525 mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
526 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
527 SDMA0_GFX_RB_WPTR_POLL_CNTL,
528 F32_POLL_ENABLE, 1);
529 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL),
530 wptr_poll_cntl);
531
532 /* set the wb address whether it's enabled or not */
533 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
534 upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
535 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
536 lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
537
538 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
539
540 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
541 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
542
543 ring->wptr = 0;
544
545 /* before programing wptr to a less value, need set minor_ptr_update first */
546 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
547
548 if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
549 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
550 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
551 }
552
553 doorbell = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL));
554 doorbell_offset = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET));
555
556 if (ring->use_doorbell) {
557 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
558 doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET,
559 OFFSET, ring->doorbell_index);
560 } else {
561 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0);
562 }
563 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell);
564 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
565
566 adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
567 ring->doorbell_index,
568 adev->doorbell_index.sdma_doorbell_range);
569
570 if (amdgpu_sriov_vf(adev))
571 sdma_v5_2_ring_set_wptr(ring);
572
573 /* set minor_ptr_update to 0 after wptr programed */
574
575 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
576
577 /* SRIOV VF has no control of any of registers below */
578 if (!amdgpu_sriov_vf(adev)) {
579 /* set utc l1 enable flag always to 1 */
580 temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
581 temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
582
583 /* enable MCBP */
584 temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1);
585 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), temp);
586
587 /* Set up RESP_MODE to non-copy addresses */
588 temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL));
589 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
590 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
591 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp);
592
593 /* program default cache read and write policy */
594 temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE));
595 /* clean read policy and write policy bits */
596 temp &= 0xFF0FFF;
597 temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
598 (CACHE_WRITE_POLICY_L2__DEFAULT << 14) |
599 SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK);
600 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp);
601
602 /* unhalt engine */
603 temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
604 temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
605 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp);
606 }
607
608 /* enable DMA RB */
609 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
610 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
611
612 ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
613 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
614 #ifdef __BIG_ENDIAN
615 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
616 #endif
617 /* enable DMA IBs */
618 WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
619
620 if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
621 sdma_v5_2_ctx_switch_enable(adev, true);
622 sdma_v5_2_enable(adev, true);
623 }
624
625 r = amdgpu_ring_test_helper(ring);
626 if (r)
627 return r;
628
629 if (adev->mman.buffer_funcs_ring == ring)
630 amdgpu_ttm_set_buffer_funcs_status(adev, true);
631 }
632
633 return 0;
634 }
635
636 /**
637 * sdma_v5_2_rlc_resume - setup and start the async dma engines
638 *
639 * @adev: amdgpu_device pointer
640 *
641 * Set up the compute DMA queues and enable them.
642 * Returns 0 for success, error for failure.
643 */
sdma_v5_2_rlc_resume(struct amdgpu_device * adev)644 static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev)
645 {
646 return 0;
647 }
648
649 /**
650 * sdma_v5_2_load_microcode - load the sDMA ME ucode
651 *
652 * @adev: amdgpu_device pointer
653 *
654 * Loads the sDMA0/1/2/3 ucode.
655 * Returns 0 for success, -EINVAL if the ucode is not available.
656 */
sdma_v5_2_load_microcode(struct amdgpu_device * adev)657 static int sdma_v5_2_load_microcode(struct amdgpu_device *adev)
658 {
659 const struct sdma_firmware_header_v1_0 *hdr;
660 const __le32 *fw_data;
661 u32 fw_size;
662 int i, j;
663
664 /* halt the MEs */
665 sdma_v5_2_enable(adev, false);
666
667 for (i = 0; i < adev->sdma.num_instances; i++) {
668 if (!adev->sdma.instance[i].fw)
669 return -EINVAL;
670
671 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
672 amdgpu_ucode_print_sdma_hdr(&hdr->header);
673 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
674
675 fw_data = (const __le32 *)
676 (adev->sdma.instance[i].fw->data +
677 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
678
679 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0);
680
681 for (j = 0; j < fw_size; j++) {
682 if (amdgpu_emu_mode == 1 && j % 500 == 0)
683 msleep(1);
684 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
685 }
686
687 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
688 }
689
690 return 0;
691 }
692
sdma_v5_2_soft_reset(void * handle)693 static int sdma_v5_2_soft_reset(void *handle)
694 {
695 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
696 u32 grbm_soft_reset;
697 u32 tmp;
698 int i;
699
700 for (i = 0; i < adev->sdma.num_instances; i++) {
701 grbm_soft_reset = REG_SET_FIELD(0,
702 GRBM_SOFT_RESET, SOFT_RESET_SDMA0,
703 1);
704 grbm_soft_reset <<= i;
705
706 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
707 tmp |= grbm_soft_reset;
708 DRM_DEBUG("GRBM_SOFT_RESET=0x%08X\n", tmp);
709 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
710 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
711
712 udelay(50);
713
714 tmp &= ~grbm_soft_reset;
715 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
716 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
717
718 udelay(50);
719 }
720
721 return 0;
722 }
723
724 /**
725 * sdma_v5_2_start - setup and start the async dma engines
726 *
727 * @adev: amdgpu_device pointer
728 *
729 * Set up the DMA engines and enable them.
730 * Returns 0 for success, error for failure.
731 */
sdma_v5_2_start(struct amdgpu_device * adev)732 static int sdma_v5_2_start(struct amdgpu_device *adev)
733 {
734 int r = 0;
735
736 if (amdgpu_sriov_vf(adev)) {
737 sdma_v5_2_ctx_switch_enable(adev, false);
738 sdma_v5_2_enable(adev, false);
739
740 /* set RB registers */
741 r = sdma_v5_2_gfx_resume(adev);
742 return r;
743 }
744
745 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
746 r = sdma_v5_2_load_microcode(adev);
747 if (r)
748 return r;
749
750 /* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */
751 if (amdgpu_emu_mode == 1)
752 msleep(1000);
753 }
754
755 sdma_v5_2_soft_reset(adev);
756 /* unhalt the MEs */
757 sdma_v5_2_enable(adev, true);
758 /* enable sdma ring preemption */
759 sdma_v5_2_ctx_switch_enable(adev, true);
760
761 /* start the gfx rings and rlc compute queues */
762 r = sdma_v5_2_gfx_resume(adev);
763 if (r)
764 return r;
765 r = sdma_v5_2_rlc_resume(adev);
766
767 return r;
768 }
769
sdma_v5_2_mqd_init(struct amdgpu_device * adev,void * mqd,struct amdgpu_mqd_prop * prop)770 static int sdma_v5_2_mqd_init(struct amdgpu_device *adev, void *mqd,
771 struct amdgpu_mqd_prop *prop)
772 {
773 struct v10_sdma_mqd *m = mqd;
774 uint64_t wb_gpu_addr;
775
776 m->sdmax_rlcx_rb_cntl =
777 order_base_2(prop->queue_size / 4) << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
778 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
779 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
780 1 << SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT;
781
782 m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
783 m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
784
785 m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
786 mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
787
788 wb_gpu_addr = prop->wptr_gpu_addr;
789 m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
790 m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
791
792 wb_gpu_addr = prop->rptr_gpu_addr;
793 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
794 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
795
796 m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
797 mmSDMA0_GFX_IB_CNTL));
798
799 m->sdmax_rlcx_doorbell_offset =
800 prop->doorbell_index << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
801
802 m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1);
803
804 return 0;
805 }
806
sdma_v5_2_set_mqd_funcs(struct amdgpu_device * adev)807 static void sdma_v5_2_set_mqd_funcs(struct amdgpu_device *adev)
808 {
809 adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v10_sdma_mqd);
810 adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v5_2_mqd_init;
811 }
812
813 /**
814 * sdma_v5_2_ring_test_ring - simple async dma engine test
815 *
816 * @ring: amdgpu_ring structure holding ring information
817 *
818 * Test the DMA engine by writing using it to write an
819 * value to memory.
820 * Returns 0 for success, error for failure.
821 */
sdma_v5_2_ring_test_ring(struct amdgpu_ring * ring)822 static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
823 {
824 struct amdgpu_device *adev = ring->adev;
825 unsigned i;
826 unsigned index;
827 int r;
828 u32 tmp;
829 u64 gpu_addr;
830 volatile uint32_t *cpu_ptr = NULL;
831
832 tmp = 0xCAFEDEAD;
833
834 if (ring->is_mes_queue) {
835 uint32_t offset = 0;
836 offset = amdgpu_mes_ctx_get_offs(ring,
837 AMDGPU_MES_CTX_PADDING_OFFS);
838 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
839 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
840 *cpu_ptr = tmp;
841 } else {
842 r = amdgpu_device_wb_get(adev, &index);
843 if (r) {
844 dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
845 return r;
846 }
847
848 gpu_addr = adev->wb.gpu_addr + (index * 4);
849 adev->wb.wb[index] = cpu_to_le32(tmp);
850 }
851
852 r = amdgpu_ring_alloc(ring, 20);
853 if (r) {
854 DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
855 amdgpu_device_wb_free(adev, index);
856 return r;
857 }
858
859 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
860 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
861 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
862 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
863 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
864 amdgpu_ring_write(ring, 0xDEADBEEF);
865 amdgpu_ring_commit(ring);
866
867 for (i = 0; i < adev->usec_timeout; i++) {
868 if (ring->is_mes_queue)
869 tmp = le32_to_cpu(*cpu_ptr);
870 else
871 tmp = le32_to_cpu(adev->wb.wb[index]);
872 if (tmp == 0xDEADBEEF)
873 break;
874 if (amdgpu_emu_mode == 1)
875 msleep(1);
876 else
877 udelay(1);
878 }
879
880 if (i >= adev->usec_timeout)
881 r = -ETIMEDOUT;
882
883 if (!ring->is_mes_queue)
884 amdgpu_device_wb_free(adev, index);
885
886 return r;
887 }
888
889 /**
890 * sdma_v5_2_ring_test_ib - test an IB on the DMA engine
891 *
892 * @ring: amdgpu_ring structure holding ring information
893 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
894 *
895 * Test a simple IB in the DMA ring.
896 * Returns 0 on success, error on failure.
897 */
sdma_v5_2_ring_test_ib(struct amdgpu_ring * ring,long timeout)898 static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
899 {
900 struct amdgpu_device *adev = ring->adev;
901 struct amdgpu_ib ib;
902 struct dma_fence *f = NULL;
903 unsigned index;
904 long r;
905 u32 tmp = 0;
906 u64 gpu_addr;
907 volatile uint32_t *cpu_ptr = NULL;
908
909 tmp = 0xCAFEDEAD;
910 memset(&ib, 0, sizeof(ib));
911
912 if (ring->is_mes_queue) {
913 uint32_t offset = 0;
914 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
915 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
916 ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
917
918 offset = amdgpu_mes_ctx_get_offs(ring,
919 AMDGPU_MES_CTX_PADDING_OFFS);
920 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
921 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
922 *cpu_ptr = tmp;
923 } else {
924 r = amdgpu_device_wb_get(adev, &index);
925 if (r) {
926 dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
927 return r;
928 }
929
930 gpu_addr = adev->wb.gpu_addr + (index * 4);
931 adev->wb.wb[index] = cpu_to_le32(tmp);
932
933 r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
934 if (r) {
935 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
936 goto err0;
937 }
938 }
939
940 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
941 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
942 ib.ptr[1] = lower_32_bits(gpu_addr);
943 ib.ptr[2] = upper_32_bits(gpu_addr);
944 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
945 ib.ptr[4] = 0xDEADBEEF;
946 ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
947 ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
948 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
949 ib.length_dw = 8;
950
951 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
952 if (r)
953 goto err1;
954
955 r = dma_fence_wait_timeout(f, false, timeout);
956 if (r == 0) {
957 DRM_ERROR("amdgpu: IB test timed out\n");
958 r = -ETIMEDOUT;
959 goto err1;
960 } else if (r < 0) {
961 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
962 goto err1;
963 }
964
965 if (ring->is_mes_queue)
966 tmp = le32_to_cpu(*cpu_ptr);
967 else
968 tmp = le32_to_cpu(adev->wb.wb[index]);
969
970 if (tmp == 0xDEADBEEF)
971 r = 0;
972 else
973 r = -EINVAL;
974
975 err1:
976 amdgpu_ib_free(adev, &ib, NULL);
977 dma_fence_put(f);
978 err0:
979 if (!ring->is_mes_queue)
980 amdgpu_device_wb_free(adev, index);
981 return r;
982 }
983
984
985 /**
986 * sdma_v5_2_vm_copy_pte - update PTEs by copying them from the GART
987 *
988 * @ib: indirect buffer to fill with commands
989 * @pe: addr of the page entry
990 * @src: src addr to copy from
991 * @count: number of page entries to update
992 *
993 * Update PTEs by copying them from the GART using sDMA.
994 */
sdma_v5_2_vm_copy_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t src,unsigned count)995 static void sdma_v5_2_vm_copy_pte(struct amdgpu_ib *ib,
996 uint64_t pe, uint64_t src,
997 unsigned count)
998 {
999 unsigned bytes = count * 8;
1000
1001 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1002 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1003 ib->ptr[ib->length_dw++] = bytes - 1;
1004 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1005 ib->ptr[ib->length_dw++] = lower_32_bits(src);
1006 ib->ptr[ib->length_dw++] = upper_32_bits(src);
1007 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1008 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1009
1010 }
1011
1012 /**
1013 * sdma_v5_2_vm_write_pte - update PTEs by writing them manually
1014 *
1015 * @ib: indirect buffer to fill with commands
1016 * @pe: addr of the page entry
1017 * @value: dst addr to write into pe
1018 * @count: number of page entries to update
1019 * @incr: increase next addr by incr bytes
1020 *
1021 * Update PTEs by writing them manually using sDMA.
1022 */
sdma_v5_2_vm_write_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t value,unsigned count,uint32_t incr)1023 static void sdma_v5_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1024 uint64_t value, unsigned count,
1025 uint32_t incr)
1026 {
1027 unsigned ndw = count * 2;
1028
1029 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1030 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1031 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1032 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1033 ib->ptr[ib->length_dw++] = ndw - 1;
1034 for (; ndw > 0; ndw -= 2) {
1035 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1036 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1037 value += incr;
1038 }
1039 }
1040
1041 /**
1042 * sdma_v5_2_vm_set_pte_pde - update the page tables using sDMA
1043 *
1044 * @ib: indirect buffer to fill with commands
1045 * @pe: addr of the page entry
1046 * @addr: dst addr to write into pe
1047 * @count: number of page entries to update
1048 * @incr: increase next addr by incr bytes
1049 * @flags: access flags
1050 *
1051 * Update the page tables using sDMA.
1052 */
sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib * ib,uint64_t pe,uint64_t addr,unsigned count,uint32_t incr,uint64_t flags)1053 static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1054 uint64_t pe,
1055 uint64_t addr, unsigned count,
1056 uint32_t incr, uint64_t flags)
1057 {
1058 /* for physically contiguous pages (vram) */
1059 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1060 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1061 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1062 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1063 ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1064 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1065 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1066 ib->ptr[ib->length_dw++] = incr; /* increment size */
1067 ib->ptr[ib->length_dw++] = 0;
1068 ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1069 }
1070
1071 /**
1072 * sdma_v5_2_ring_pad_ib - pad the IB
1073 *
1074 * @ib: indirect buffer to fill with padding
1075 * @ring: amdgpu_ring structure holding ring information
1076 *
1077 * Pad the IB with NOPs to a boundary multiple of 8.
1078 */
sdma_v5_2_ring_pad_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib)1079 static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1080 {
1081 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1082 u32 pad_count;
1083 int i;
1084
1085 pad_count = (-ib->length_dw) & 0x7;
1086 for (i = 0; i < pad_count; i++)
1087 if (sdma && sdma->burst_nop && (i == 0))
1088 ib->ptr[ib->length_dw++] =
1089 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1090 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1091 else
1092 ib->ptr[ib->length_dw++] =
1093 SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1094 }
1095
1096
1097 /**
1098 * sdma_v5_2_ring_emit_pipeline_sync - sync the pipeline
1099 *
1100 * @ring: amdgpu_ring pointer
1101 *
1102 * Make sure all previous operations are completed (CIK).
1103 */
sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring * ring)1104 static void sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1105 {
1106 uint32_t seq = ring->fence_drv.sync_seq;
1107 uint64_t addr = ring->fence_drv.gpu_addr;
1108
1109 /* wait for idle */
1110 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1111 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1112 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1113 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1114 amdgpu_ring_write(ring, addr & 0xfffffffc);
1115 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1116 amdgpu_ring_write(ring, seq); /* reference */
1117 amdgpu_ring_write(ring, 0xffffffff); /* mask */
1118 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1119 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1120 }
1121
1122
1123 /**
1124 * sdma_v5_2_ring_emit_vm_flush - vm flush using sDMA
1125 *
1126 * @ring: amdgpu_ring pointer
1127 * @vmid: vmid number to use
1128 * @pd_addr: address
1129 *
1130 * Update the page table base and flush the VM TLB
1131 * using sDMA.
1132 */
sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)1133 static void sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1134 unsigned vmid, uint64_t pd_addr)
1135 {
1136 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1137 }
1138
sdma_v5_2_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)1139 static void sdma_v5_2_ring_emit_wreg(struct amdgpu_ring *ring,
1140 uint32_t reg, uint32_t val)
1141 {
1142 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1143 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1144 amdgpu_ring_write(ring, reg);
1145 amdgpu_ring_write(ring, val);
1146 }
1147
sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)1148 static void sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1149 uint32_t val, uint32_t mask)
1150 {
1151 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1152 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1153 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1154 amdgpu_ring_write(ring, reg << 2);
1155 amdgpu_ring_write(ring, 0);
1156 amdgpu_ring_write(ring, val); /* reference */
1157 amdgpu_ring_write(ring, mask); /* mask */
1158 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1159 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1160 }
1161
sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)1162 static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1163 uint32_t reg0, uint32_t reg1,
1164 uint32_t ref, uint32_t mask)
1165 {
1166 amdgpu_ring_emit_wreg(ring, reg0, ref);
1167 /* wait for a cycle to reset vm_inv_eng*_ack */
1168 amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1169 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1170 }
1171
sdma_v5_2_early_init(void * handle)1172 static int sdma_v5_2_early_init(void *handle)
1173 {
1174 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1175
1176 sdma_v5_2_set_ring_funcs(adev);
1177 sdma_v5_2_set_buffer_funcs(adev);
1178 sdma_v5_2_set_vm_pte_funcs(adev);
1179 sdma_v5_2_set_irq_funcs(adev);
1180 sdma_v5_2_set_mqd_funcs(adev);
1181
1182 return 0;
1183 }
1184
sdma_v5_2_seq_to_irq_id(int seq_num)1185 static unsigned sdma_v5_2_seq_to_irq_id(int seq_num)
1186 {
1187 switch (seq_num) {
1188 case 0:
1189 return SOC15_IH_CLIENTID_SDMA0;
1190 case 1:
1191 return SOC15_IH_CLIENTID_SDMA1;
1192 case 2:
1193 return SOC15_IH_CLIENTID_SDMA2;
1194 case 3:
1195 return SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid;
1196 default:
1197 break;
1198 }
1199 return -EINVAL;
1200 }
1201
sdma_v5_2_seq_to_trap_id(int seq_num)1202 static unsigned sdma_v5_2_seq_to_trap_id(int seq_num)
1203 {
1204 switch (seq_num) {
1205 case 0:
1206 return SDMA0_5_0__SRCID__SDMA_TRAP;
1207 case 1:
1208 return SDMA1_5_0__SRCID__SDMA_TRAP;
1209 case 2:
1210 return SDMA2_5_0__SRCID__SDMA_TRAP;
1211 case 3:
1212 return SDMA3_5_0__SRCID__SDMA_TRAP;
1213 default:
1214 break;
1215 }
1216 return -EINVAL;
1217 }
1218
sdma_v5_2_sw_init(void * handle)1219 static int sdma_v5_2_sw_init(void *handle)
1220 {
1221 struct amdgpu_ring *ring;
1222 int r, i;
1223 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1224
1225 /* SDMA trap event */
1226 for (i = 0; i < adev->sdma.num_instances; i++) {
1227 r = amdgpu_irq_add_id(adev, sdma_v5_2_seq_to_irq_id(i),
1228 sdma_v5_2_seq_to_trap_id(i),
1229 &adev->sdma.trap_irq);
1230 if (r)
1231 return r;
1232 }
1233
1234 r = amdgpu_sdma_init_microcode(adev, 0, true);
1235 if (r) {
1236 DRM_ERROR("Failed to load sdma firmware!\n");
1237 return r;
1238 }
1239
1240 for (i = 0; i < adev->sdma.num_instances; i++) {
1241 ring = &adev->sdma.instance[i].ring;
1242 ring->ring_obj = NULL;
1243 ring->use_doorbell = true;
1244 ring->me = i;
1245
1246 DRM_INFO("use_doorbell being set to: [%s]\n",
1247 ring->use_doorbell?"true":"false");
1248
1249 ring->doorbell_index =
1250 (adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset
1251
1252 ring->vm_hub = AMDGPU_GFXHUB(0);
1253 sprintf(ring->name, "sdma%d", i);
1254 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1255 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1256 AMDGPU_RING_PRIO_DEFAULT, NULL);
1257 if (r)
1258 return r;
1259 }
1260
1261 return r;
1262 }
1263
sdma_v5_2_sw_fini(void * handle)1264 static int sdma_v5_2_sw_fini(void *handle)
1265 {
1266 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1267 int i;
1268
1269 for (i = 0; i < adev->sdma.num_instances; i++)
1270 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1271
1272 amdgpu_sdma_destroy_inst_ctx(adev, true);
1273
1274 return 0;
1275 }
1276
sdma_v5_2_hw_init(void * handle)1277 static int sdma_v5_2_hw_init(void *handle)
1278 {
1279 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1280
1281 return sdma_v5_2_start(adev);
1282 }
1283
sdma_v5_2_hw_fini(void * handle)1284 static int sdma_v5_2_hw_fini(void *handle)
1285 {
1286 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1287
1288 if (amdgpu_sriov_vf(adev)) {
1289 /* disable the scheduler for SDMA */
1290 amdgpu_sdma_unset_buffer_funcs_helper(adev);
1291 return 0;
1292 }
1293
1294 sdma_v5_2_ctx_switch_enable(adev, false);
1295 sdma_v5_2_enable(adev, false);
1296
1297 return 0;
1298 }
1299
sdma_v5_2_suspend(void * handle)1300 static int sdma_v5_2_suspend(void *handle)
1301 {
1302 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1303
1304 return sdma_v5_2_hw_fini(adev);
1305 }
1306
sdma_v5_2_resume(void * handle)1307 static int sdma_v5_2_resume(void *handle)
1308 {
1309 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1310
1311 return sdma_v5_2_hw_init(adev);
1312 }
1313
sdma_v5_2_is_idle(void * handle)1314 static bool sdma_v5_2_is_idle(void *handle)
1315 {
1316 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1317 u32 i;
1318
1319 for (i = 0; i < adev->sdma.num_instances; i++) {
1320 u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
1321
1322 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1323 return false;
1324 }
1325
1326 return true;
1327 }
1328
sdma_v5_2_wait_for_idle(void * handle)1329 static int sdma_v5_2_wait_for_idle(void *handle)
1330 {
1331 unsigned i;
1332 u32 sdma0, sdma1, sdma2, sdma3;
1333 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1334
1335 for (i = 0; i < adev->usec_timeout; i++) {
1336 sdma0 = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG));
1337 sdma1 = RREG32(sdma_v5_2_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG));
1338 sdma2 = RREG32(sdma_v5_2_get_reg_offset(adev, 2, mmSDMA0_STATUS_REG));
1339 sdma3 = RREG32(sdma_v5_2_get_reg_offset(adev, 3, mmSDMA0_STATUS_REG));
1340
1341 if (sdma0 & sdma1 & sdma2 & sdma3 & SDMA0_STATUS_REG__IDLE_MASK)
1342 return 0;
1343 udelay(1);
1344 }
1345 return -ETIMEDOUT;
1346 }
1347
sdma_v5_2_ring_preempt_ib(struct amdgpu_ring * ring)1348 static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring)
1349 {
1350 int i, r = 0;
1351 struct amdgpu_device *adev = ring->adev;
1352 u32 index = 0;
1353 u64 sdma_gfx_preempt;
1354
1355 amdgpu_sdma_get_index_from_ring(ring, &index);
1356 sdma_gfx_preempt =
1357 sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT);
1358
1359 /* assert preemption condition */
1360 amdgpu_ring_set_preempt_cond_exec(ring, false);
1361
1362 /* emit the trailing fence */
1363 ring->trail_seq += 1;
1364 amdgpu_ring_alloc(ring, 10);
1365 sdma_v5_2_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1366 ring->trail_seq, 0);
1367 amdgpu_ring_commit(ring);
1368
1369 /* assert IB preemption */
1370 WREG32(sdma_gfx_preempt, 1);
1371
1372 /* poll the trailing fence */
1373 for (i = 0; i < adev->usec_timeout; i++) {
1374 if (ring->trail_seq ==
1375 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1376 break;
1377 udelay(1);
1378 }
1379
1380 if (i >= adev->usec_timeout) {
1381 r = -EINVAL;
1382 DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1383 }
1384
1385 /* deassert IB preemption */
1386 WREG32(sdma_gfx_preempt, 0);
1387
1388 /* deassert the preemption condition */
1389 amdgpu_ring_set_preempt_cond_exec(ring, true);
1390 return r;
1391 }
1392
sdma_v5_2_set_trap_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1393 static int sdma_v5_2_set_trap_irq_state(struct amdgpu_device *adev,
1394 struct amdgpu_irq_src *source,
1395 unsigned type,
1396 enum amdgpu_interrupt_state state)
1397 {
1398 u32 sdma_cntl;
1399 u32 reg_offset = sdma_v5_2_get_reg_offset(adev, type, mmSDMA0_CNTL);
1400
1401 if (!amdgpu_sriov_vf(adev)) {
1402 sdma_cntl = RREG32(reg_offset);
1403 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1404 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1405 WREG32(reg_offset, sdma_cntl);
1406 }
1407
1408 return 0;
1409 }
1410
sdma_v5_2_process_trap_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1411 static int sdma_v5_2_process_trap_irq(struct amdgpu_device *adev,
1412 struct amdgpu_irq_src *source,
1413 struct amdgpu_iv_entry *entry)
1414 {
1415 uint32_t mes_queue_id = entry->src_data[0];
1416
1417 DRM_DEBUG("IH: SDMA trap\n");
1418
1419 if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
1420 struct amdgpu_mes_queue *queue;
1421
1422 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
1423
1424 spin_lock(&adev->mes.queue_id_lock);
1425 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
1426 if (queue) {
1427 DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
1428 amdgpu_fence_process(queue->ring);
1429 }
1430 spin_unlock(&adev->mes.queue_id_lock);
1431 return 0;
1432 }
1433
1434 switch (entry->client_id) {
1435 case SOC15_IH_CLIENTID_SDMA0:
1436 switch (entry->ring_id) {
1437 case 0:
1438 amdgpu_fence_process(&adev->sdma.instance[0].ring);
1439 break;
1440 case 1:
1441 /* XXX compute */
1442 break;
1443 case 2:
1444 /* XXX compute */
1445 break;
1446 case 3:
1447 /* XXX page queue*/
1448 break;
1449 }
1450 break;
1451 case SOC15_IH_CLIENTID_SDMA1:
1452 switch (entry->ring_id) {
1453 case 0:
1454 amdgpu_fence_process(&adev->sdma.instance[1].ring);
1455 break;
1456 case 1:
1457 /* XXX compute */
1458 break;
1459 case 2:
1460 /* XXX compute */
1461 break;
1462 case 3:
1463 /* XXX page queue*/
1464 break;
1465 }
1466 break;
1467 case SOC15_IH_CLIENTID_SDMA2:
1468 switch (entry->ring_id) {
1469 case 0:
1470 amdgpu_fence_process(&adev->sdma.instance[2].ring);
1471 break;
1472 case 1:
1473 /* XXX compute */
1474 break;
1475 case 2:
1476 /* XXX compute */
1477 break;
1478 case 3:
1479 /* XXX page queue*/
1480 break;
1481 }
1482 break;
1483 case SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid:
1484 switch (entry->ring_id) {
1485 case 0:
1486 amdgpu_fence_process(&adev->sdma.instance[3].ring);
1487 break;
1488 case 1:
1489 /* XXX compute */
1490 break;
1491 case 2:
1492 /* XXX compute */
1493 break;
1494 case 3:
1495 /* XXX page queue*/
1496 break;
1497 }
1498 break;
1499 }
1500 return 0;
1501 }
1502
sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1503 static int sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1504 struct amdgpu_irq_src *source,
1505 struct amdgpu_iv_entry *entry)
1506 {
1507 return 0;
1508 }
1509
sdma_v5_2_firmware_mgcg_support(struct amdgpu_device * adev,int i)1510 static bool sdma_v5_2_firmware_mgcg_support(struct amdgpu_device *adev,
1511 int i)
1512 {
1513 switch (adev->ip_versions[SDMA0_HWIP][0]) {
1514 case IP_VERSION(5, 2, 1):
1515 if (adev->sdma.instance[i].fw_version < 70)
1516 return false;
1517 break;
1518 case IP_VERSION(5, 2, 3):
1519 if (adev->sdma.instance[i].fw_version < 47)
1520 return false;
1521 break;
1522 case IP_VERSION(5, 2, 7):
1523 if (adev->sdma.instance[i].fw_version < 9)
1524 return false;
1525 break;
1526 default:
1527 return true;
1528 }
1529
1530 return true;
1531
1532 }
1533
sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)1534 static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *adev,
1535 bool enable)
1536 {
1537 uint32_t data, def;
1538 int i;
1539
1540 for (i = 0; i < adev->sdma.num_instances; i++) {
1541
1542 if (!sdma_v5_2_firmware_mgcg_support(adev, i))
1543 adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG;
1544
1545 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
1546 /* Enable sdma clock gating */
1547 def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1548 data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1549 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1550 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1551 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1552 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1553 SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1554 if (def != data)
1555 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1556 } else {
1557 /* Disable sdma clock gating */
1558 def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1559 data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1560 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1561 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1562 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1563 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1564 SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1565 if (def != data)
1566 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1567 }
1568 }
1569 }
1570
sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device * adev,bool enable)1571 static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev,
1572 bool enable)
1573 {
1574 uint32_t data, def;
1575 int i;
1576
1577 for (i = 0; i < adev->sdma.num_instances; i++) {
1578
1579 if (adev->sdma.instance[i].fw_version < 70 && adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(5, 2, 1))
1580 adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS;
1581
1582 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
1583 /* Enable sdma mem light sleep */
1584 def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1585 data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1586 if (def != data)
1587 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1588
1589 } else {
1590 /* Disable sdma mem light sleep */
1591 def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1592 data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1593 if (def != data)
1594 WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1595
1596 }
1597 }
1598 }
1599
sdma_v5_2_set_clockgating_state(void * handle,enum amd_clockgating_state state)1600 static int sdma_v5_2_set_clockgating_state(void *handle,
1601 enum amd_clockgating_state state)
1602 {
1603 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1604
1605 if (amdgpu_sriov_vf(adev))
1606 return 0;
1607
1608 switch (adev->ip_versions[SDMA0_HWIP][0]) {
1609 case IP_VERSION(5, 2, 0):
1610 case IP_VERSION(5, 2, 2):
1611 case IP_VERSION(5, 2, 1):
1612 case IP_VERSION(5, 2, 4):
1613 case IP_VERSION(5, 2, 5):
1614 case IP_VERSION(5, 2, 6):
1615 case IP_VERSION(5, 2, 3):
1616 case IP_VERSION(5, 2, 7):
1617 sdma_v5_2_update_medium_grain_clock_gating(adev,
1618 state == AMD_CG_STATE_GATE);
1619 sdma_v5_2_update_medium_grain_light_sleep(adev,
1620 state == AMD_CG_STATE_GATE);
1621 break;
1622 default:
1623 break;
1624 }
1625
1626 return 0;
1627 }
1628
sdma_v5_2_set_powergating_state(void * handle,enum amd_powergating_state state)1629 static int sdma_v5_2_set_powergating_state(void *handle,
1630 enum amd_powergating_state state)
1631 {
1632 return 0;
1633 }
1634
sdma_v5_2_get_clockgating_state(void * handle,u64 * flags)1635 static void sdma_v5_2_get_clockgating_state(void *handle, u64 *flags)
1636 {
1637 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1638 int data;
1639
1640 if (amdgpu_sriov_vf(adev))
1641 *flags = 0;
1642
1643 /* AMD_CG_SUPPORT_SDMA_MGCG */
1644 data = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL));
1645 if (!(data & SDMA0_CLK_CTRL__CGCG_EN_OVERRIDE_MASK))
1646 *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1647
1648 /* AMD_CG_SUPPORT_SDMA_LS */
1649 data = RREG32_KIQ(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL));
1650 if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1651 *flags |= AMD_CG_SUPPORT_SDMA_LS;
1652 }
1653
1654 const struct amd_ip_funcs sdma_v5_2_ip_funcs = {
1655 .name = "sdma_v5_2",
1656 .early_init = sdma_v5_2_early_init,
1657 .late_init = NULL,
1658 .sw_init = sdma_v5_2_sw_init,
1659 .sw_fini = sdma_v5_2_sw_fini,
1660 .hw_init = sdma_v5_2_hw_init,
1661 .hw_fini = sdma_v5_2_hw_fini,
1662 .suspend = sdma_v5_2_suspend,
1663 .resume = sdma_v5_2_resume,
1664 .is_idle = sdma_v5_2_is_idle,
1665 .wait_for_idle = sdma_v5_2_wait_for_idle,
1666 .soft_reset = sdma_v5_2_soft_reset,
1667 .set_clockgating_state = sdma_v5_2_set_clockgating_state,
1668 .set_powergating_state = sdma_v5_2_set_powergating_state,
1669 .get_clockgating_state = sdma_v5_2_get_clockgating_state,
1670 };
1671
1672 static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = {
1673 .type = AMDGPU_RING_TYPE_SDMA,
1674 .align_mask = 0xf,
1675 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1676 .support_64bit_ptrs = true,
1677 .secure_submission_supported = true,
1678 .get_rptr = sdma_v5_2_ring_get_rptr,
1679 .get_wptr = sdma_v5_2_ring_get_wptr,
1680 .set_wptr = sdma_v5_2_ring_set_wptr,
1681 .emit_frame_size =
1682 5 + /* sdma_v5_2_ring_init_cond_exec */
1683 6 + /* sdma_v5_2_ring_emit_hdp_flush */
1684 3 + /* hdp_invalidate */
1685 6 + /* sdma_v5_2_ring_emit_pipeline_sync */
1686 /* sdma_v5_2_ring_emit_vm_flush */
1687 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1688 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1689 10 + 10 + 10, /* sdma_v5_2_ring_emit_fence x3 for user fence, vm fence */
1690 .emit_ib_size = 7 + 6, /* sdma_v5_2_ring_emit_ib */
1691 .emit_ib = sdma_v5_2_ring_emit_ib,
1692 .emit_mem_sync = sdma_v5_2_ring_emit_mem_sync,
1693 .emit_fence = sdma_v5_2_ring_emit_fence,
1694 .emit_pipeline_sync = sdma_v5_2_ring_emit_pipeline_sync,
1695 .emit_vm_flush = sdma_v5_2_ring_emit_vm_flush,
1696 .emit_hdp_flush = sdma_v5_2_ring_emit_hdp_flush,
1697 .test_ring = sdma_v5_2_ring_test_ring,
1698 .test_ib = sdma_v5_2_ring_test_ib,
1699 .insert_nop = sdma_v5_2_ring_insert_nop,
1700 .pad_ib = sdma_v5_2_ring_pad_ib,
1701 .emit_wreg = sdma_v5_2_ring_emit_wreg,
1702 .emit_reg_wait = sdma_v5_2_ring_emit_reg_wait,
1703 .emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait,
1704 .init_cond_exec = sdma_v5_2_ring_init_cond_exec,
1705 .patch_cond_exec = sdma_v5_2_ring_patch_cond_exec,
1706 .preempt_ib = sdma_v5_2_ring_preempt_ib,
1707 };
1708
sdma_v5_2_set_ring_funcs(struct amdgpu_device * adev)1709 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev)
1710 {
1711 int i;
1712
1713 for (i = 0; i < adev->sdma.num_instances; i++) {
1714 adev->sdma.instance[i].ring.funcs = &sdma_v5_2_ring_funcs;
1715 adev->sdma.instance[i].ring.me = i;
1716 }
1717 }
1718
1719 static const struct amdgpu_irq_src_funcs sdma_v5_2_trap_irq_funcs = {
1720 .set = sdma_v5_2_set_trap_irq_state,
1721 .process = sdma_v5_2_process_trap_irq,
1722 };
1723
1724 static const struct amdgpu_irq_src_funcs sdma_v5_2_illegal_inst_irq_funcs = {
1725 .process = sdma_v5_2_process_illegal_inst_irq,
1726 };
1727
sdma_v5_2_set_irq_funcs(struct amdgpu_device * adev)1728 static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev)
1729 {
1730 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1731 adev->sdma.num_instances;
1732 adev->sdma.trap_irq.funcs = &sdma_v5_2_trap_irq_funcs;
1733 adev->sdma.illegal_inst_irq.funcs = &sdma_v5_2_illegal_inst_irq_funcs;
1734 }
1735
1736 /**
1737 * sdma_v5_2_emit_copy_buffer - copy buffer using the sDMA engine
1738 *
1739 * @ib: indirect buffer to copy to
1740 * @src_offset: src GPU address
1741 * @dst_offset: dst GPU address
1742 * @byte_count: number of bytes to xfer
1743 * @tmz: if a secure copy should be used
1744 *
1745 * Copy GPU buffers using the DMA engine.
1746 * Used by the amdgpu ttm implementation to move pages if
1747 * registered as the asic copy callback.
1748 */
sdma_v5_2_emit_copy_buffer(struct amdgpu_ib * ib,uint64_t src_offset,uint64_t dst_offset,uint32_t byte_count,bool tmz)1749 static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib,
1750 uint64_t src_offset,
1751 uint64_t dst_offset,
1752 uint32_t byte_count,
1753 bool tmz)
1754 {
1755 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1756 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1757 SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
1758 ib->ptr[ib->length_dw++] = byte_count - 1;
1759 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1760 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1761 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1762 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1763 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1764 }
1765
1766 /**
1767 * sdma_v5_2_emit_fill_buffer - fill buffer using the sDMA engine
1768 *
1769 * @ib: indirect buffer to fill
1770 * @src_data: value to write to buffer
1771 * @dst_offset: dst GPU address
1772 * @byte_count: number of bytes to xfer
1773 *
1774 * Fill GPU buffers using the DMA engine.
1775 */
sdma_v5_2_emit_fill_buffer(struct amdgpu_ib * ib,uint32_t src_data,uint64_t dst_offset,uint32_t byte_count)1776 static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib,
1777 uint32_t src_data,
1778 uint64_t dst_offset,
1779 uint32_t byte_count)
1780 {
1781 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
1782 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1783 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1784 ib->ptr[ib->length_dw++] = src_data;
1785 ib->ptr[ib->length_dw++] = byte_count - 1;
1786 }
1787
1788 static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
1789 .copy_max_bytes = 0x400000,
1790 .copy_num_dw = 7,
1791 .emit_copy_buffer = sdma_v5_2_emit_copy_buffer,
1792
1793 .fill_max_bytes = 0x400000,
1794 .fill_num_dw = 5,
1795 .emit_fill_buffer = sdma_v5_2_emit_fill_buffer,
1796 };
1797
sdma_v5_2_set_buffer_funcs(struct amdgpu_device * adev)1798 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
1799 {
1800 if (adev->mman.buffer_funcs == NULL) {
1801 adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
1802 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1803 }
1804 }
1805
1806 static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = {
1807 .copy_pte_num_dw = 7,
1808 .copy_pte = sdma_v5_2_vm_copy_pte,
1809 .write_pte = sdma_v5_2_vm_write_pte,
1810 .set_pte_pde = sdma_v5_2_vm_set_pte_pde,
1811 };
1812
sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device * adev)1813 static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev)
1814 {
1815 unsigned i;
1816
1817 if (adev->vm_manager.vm_pte_funcs == NULL) {
1818 adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs;
1819 for (i = 0; i < adev->sdma.num_instances; i++) {
1820 adev->vm_manager.vm_pte_scheds[i] =
1821 &adev->sdma.instance[i].ring.sched;
1822 }
1823 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1824 }
1825 }
1826
1827 const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {
1828 .type = AMD_IP_BLOCK_TYPE_SDMA,
1829 .major = 5,
1830 .minor = 2,
1831 .rev = 0,
1832 .funcs = &sdma_v5_2_ip_funcs,
1833 };
1834