1 /*
2 * Copyright 2014-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #define pr_fmt(fmt) "kfd2kgd: " fmt
24
25 #include <linux/module.h>
26 #include <linux/fdtable.h>
27 #include <linux/uaccess.h>
28 #include <linux/firmware.h>
29 #include <drm/drmP.h>
30 #include "amdgpu.h"
31 #include "amdgpu_amdkfd.h"
32 #include "amdgpu_ucode.h"
33 #include "soc15_hw_ip.h"
34 #include "gc/gc_9_0_offset.h"
35 #include "gc/gc_9_0_sh_mask.h"
36 #include "vega10_enum.h"
37 #include "sdma0/sdma0_4_0_offset.h"
38 #include "sdma0/sdma0_4_0_sh_mask.h"
39 #include "sdma1/sdma1_4_0_offset.h"
40 #include "sdma1/sdma1_4_0_sh_mask.h"
41 #include "athub/athub_1_0_offset.h"
42 #include "athub/athub_1_0_sh_mask.h"
43 #include "oss/osssys_4_0_offset.h"
44 #include "oss/osssys_4_0_sh_mask.h"
45 #include "soc15_common.h"
46 #include "v9_structs.h"
47 #include "soc15.h"
48 #include "soc15d.h"
49
50 /* HACK: MMHUB and GC both have VM-related register with the same
51 * names but different offsets. Define the MMHUB register we need here
52 * with a prefix. A proper solution would be to move the functions
53 * programming these registers into gfx_v9_0.c and mmhub_v1_0.c
54 * respectively.
55 */
56 #define mmMMHUB_VM_INVALIDATE_ENG16_REQ 0x06f3
57 #define mmMMHUB_VM_INVALIDATE_ENG16_REQ_BASE_IDX 0
58
59 #define mmMMHUB_VM_INVALIDATE_ENG16_ACK 0x0705
60 #define mmMMHUB_VM_INVALIDATE_ENG16_ACK_BASE_IDX 0
61
62 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32 0x072b
63 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 0
64 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32 0x072c
65 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 0
66
67 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32 0x074b
68 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 0
69 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32 0x074c
70 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 0
71
72 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32 0x076b
73 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 0
74 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32 0x076c
75 #define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 0
76
77 #define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32 0x0727
78 #define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX 0
79 #define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32 0x0728
80 #define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX 0
81
82 #define V9_PIPE_PER_MEC (4)
83 #define V9_QUEUES_PER_PIPE_MEC (8)
84
85 enum hqd_dequeue_request_type {
86 NO_ACTION = 0,
87 DRAIN_PIPE,
88 RESET_WAVES
89 };
90
91 /*
92 * Register access functions
93 */
94
95 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
96 uint32_t sh_mem_config,
97 uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
98 uint32_t sh_mem_bases);
99 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
100 unsigned int vmid);
101 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
102 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
103 uint32_t queue_id, uint32_t __user *wptr,
104 uint32_t wptr_shift, uint32_t wptr_mask,
105 struct mm_struct *mm);
106 static int kgd_hqd_dump(struct kgd_dev *kgd,
107 uint32_t pipe_id, uint32_t queue_id,
108 uint32_t (**dump)[2], uint32_t *n_regs);
109 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
110 uint32_t __user *wptr, struct mm_struct *mm);
111 static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
112 uint32_t engine_id, uint32_t queue_id,
113 uint32_t (**dump)[2], uint32_t *n_regs);
114 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
115 uint32_t pipe_id, uint32_t queue_id);
116 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
117 static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
118 enum kfd_preempt_type reset_type,
119 unsigned int utimeout, uint32_t pipe_id,
120 uint32_t queue_id);
121 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
122 unsigned int utimeout);
123 static int kgd_address_watch_disable(struct kgd_dev *kgd);
124 static int kgd_address_watch_execute(struct kgd_dev *kgd,
125 unsigned int watch_point_id,
126 uint32_t cntl_val,
127 uint32_t addr_hi,
128 uint32_t addr_lo);
129 static int kgd_wave_control_execute(struct kgd_dev *kgd,
130 uint32_t gfx_index_val,
131 uint32_t sq_cmd);
132 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
133 unsigned int watch_point_id,
134 unsigned int reg_offset);
135
136 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
137 uint8_t vmid);
138 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
139 uint8_t vmid);
140 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
141 uint32_t page_table_base);
142 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
143 static void set_scratch_backing_va(struct kgd_dev *kgd,
144 uint64_t va, uint32_t vmid);
145 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
146 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
147
148 /* Because of REG_GET_FIELD() being used, we put this function in the
149 * asic specific file.
150 */
amdgpu_amdkfd_get_tile_config(struct kgd_dev * kgd,struct tile_config * config)151 static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
152 struct tile_config *config)
153 {
154 struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
155
156 config->gb_addr_config = adev->gfx.config.gb_addr_config;
157
158 config->tile_config_ptr = adev->gfx.config.tile_mode_array;
159 config->num_tile_configs =
160 ARRAY_SIZE(adev->gfx.config.tile_mode_array);
161 config->macro_tile_config_ptr =
162 adev->gfx.config.macrotile_mode_array;
163 config->num_macro_tile_configs =
164 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
165
166 return 0;
167 }
168
169 static const struct kfd2kgd_calls kfd2kgd = {
170 .init_gtt_mem_allocation = alloc_gtt_mem,
171 .free_gtt_mem = free_gtt_mem,
172 .get_local_mem_info = get_local_mem_info,
173 .get_gpu_clock_counter = get_gpu_clock_counter,
174 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
175 .alloc_pasid = amdgpu_pasid_alloc,
176 .free_pasid = amdgpu_pasid_free,
177 .program_sh_mem_settings = kgd_program_sh_mem_settings,
178 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
179 .init_interrupts = kgd_init_interrupts,
180 .hqd_load = kgd_hqd_load,
181 .hqd_sdma_load = kgd_hqd_sdma_load,
182 .hqd_dump = kgd_hqd_dump,
183 .hqd_sdma_dump = kgd_hqd_sdma_dump,
184 .hqd_is_occupied = kgd_hqd_is_occupied,
185 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
186 .hqd_destroy = kgd_hqd_destroy,
187 .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
188 .address_watch_disable = kgd_address_watch_disable,
189 .address_watch_execute = kgd_address_watch_execute,
190 .wave_control_execute = kgd_wave_control_execute,
191 .address_watch_get_offset = kgd_address_watch_get_offset,
192 .get_atc_vmid_pasid_mapping_pasid =
193 get_atc_vmid_pasid_mapping_pasid,
194 .get_atc_vmid_pasid_mapping_valid =
195 get_atc_vmid_pasid_mapping_valid,
196 .get_fw_version = get_fw_version,
197 .set_scratch_backing_va = set_scratch_backing_va,
198 .get_tile_config = amdgpu_amdkfd_get_tile_config,
199 .get_cu_info = get_cu_info,
200 .get_vram_usage = amdgpu_amdkfd_get_vram_usage,
201 .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,
202 .acquire_process_vm = amdgpu_amdkfd_gpuvm_acquire_process_vm,
203 .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,
204 .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,
205 .set_vm_context_page_table_base = set_vm_context_page_table_base,
206 .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,
207 .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,
208 .map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,
209 .unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,
210 .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,
211 .map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,
212 .restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,
213 .invalidate_tlbs = invalidate_tlbs,
214 .invalidate_tlbs_vmid = invalidate_tlbs_vmid,
215 .submit_ib = amdgpu_amdkfd_submit_ib,
216 .gpu_recover = amdgpu_amdkfd_gpu_reset,
217 .set_compute_idle = amdgpu_amdkfd_set_compute_idle
218 };
219
amdgpu_amdkfd_gfx_9_0_get_functions(void)220 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
221 {
222 return (struct kfd2kgd_calls *)&kfd2kgd;
223 }
224
get_amdgpu_device(struct kgd_dev * kgd)225 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
226 {
227 return (struct amdgpu_device *)kgd;
228 }
229
lock_srbm(struct kgd_dev * kgd,uint32_t mec,uint32_t pipe,uint32_t queue,uint32_t vmid)230 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
231 uint32_t queue, uint32_t vmid)
232 {
233 struct amdgpu_device *adev = get_amdgpu_device(kgd);
234
235 mutex_lock(&adev->srbm_mutex);
236 soc15_grbm_select(adev, mec, pipe, queue, vmid);
237 }
238
unlock_srbm(struct kgd_dev * kgd)239 static void unlock_srbm(struct kgd_dev *kgd)
240 {
241 struct amdgpu_device *adev = get_amdgpu_device(kgd);
242
243 soc15_grbm_select(adev, 0, 0, 0, 0);
244 mutex_unlock(&adev->srbm_mutex);
245 }
246
acquire_queue(struct kgd_dev * kgd,uint32_t pipe_id,uint32_t queue_id)247 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
248 uint32_t queue_id)
249 {
250 struct amdgpu_device *adev = get_amdgpu_device(kgd);
251
252 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
253 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
254
255 lock_srbm(kgd, mec, pipe, queue_id, 0);
256 }
257
get_queue_mask(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id)258 static uint32_t get_queue_mask(struct amdgpu_device *adev,
259 uint32_t pipe_id, uint32_t queue_id)
260 {
261 unsigned int bit = (pipe_id * adev->gfx.mec.num_queue_per_pipe +
262 queue_id) & 31;
263
264 return ((uint32_t)1) << bit;
265 }
266
release_queue(struct kgd_dev * kgd)267 static void release_queue(struct kgd_dev *kgd)
268 {
269 unlock_srbm(kgd);
270 }
271
kgd_program_sh_mem_settings(struct kgd_dev * kgd,uint32_t vmid,uint32_t sh_mem_config,uint32_t sh_mem_ape1_base,uint32_t sh_mem_ape1_limit,uint32_t sh_mem_bases)272 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
273 uint32_t sh_mem_config,
274 uint32_t sh_mem_ape1_base,
275 uint32_t sh_mem_ape1_limit,
276 uint32_t sh_mem_bases)
277 {
278 struct amdgpu_device *adev = get_amdgpu_device(kgd);
279
280 lock_srbm(kgd, 0, 0, 0, vmid);
281
282 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
283 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
284 /* APE1 no longer exists on GFX9 */
285
286 unlock_srbm(kgd);
287 }
288
kgd_set_pasid_vmid_mapping(struct kgd_dev * kgd,unsigned int pasid,unsigned int vmid)289 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
290 unsigned int vmid)
291 {
292 struct amdgpu_device *adev = get_amdgpu_device(kgd);
293
294 /*
295 * We have to assume that there is no outstanding mapping.
296 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
297 * a mapping is in progress or because a mapping finished
298 * and the SW cleared it.
299 * So the protocol is to always wait & clear.
300 */
301 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
302 ATC_VMID0_PASID_MAPPING__VALID_MASK;
303
304 /*
305 * need to do this twice, once for gfx and once for mmhub
306 * for ATC add 16 to VMID for mmhub, for IH different registers.
307 * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
308 */
309
310 WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
311 pasid_mapping);
312
313 while (!(RREG32(SOC15_REG_OFFSET(
314 ATHUB, 0,
315 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
316 (1U << vmid)))
317 cpu_relax();
318
319 WREG32(SOC15_REG_OFFSET(ATHUB, 0,
320 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
321 1U << vmid);
322
323 /* Mapping vmid to pasid also for IH block */
324 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid,
325 pasid_mapping);
326
327 WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID16_PASID_MAPPING) + vmid,
328 pasid_mapping);
329
330 while (!(RREG32(SOC15_REG_OFFSET(
331 ATHUB, 0,
332 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
333 (1U << (vmid + 16))))
334 cpu_relax();
335
336 WREG32(SOC15_REG_OFFSET(ATHUB, 0,
337 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
338 1U << (vmid + 16));
339
340 /* Mapping vmid to pasid also for IH block */
341 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid,
342 pasid_mapping);
343 return 0;
344 }
345
346 /* TODO - RING0 form of field is obsolete, seems to date back to SI
347 * but still works
348 */
349
kgd_init_interrupts(struct kgd_dev * kgd,uint32_t pipe_id)350 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
351 {
352 struct amdgpu_device *adev = get_amdgpu_device(kgd);
353 uint32_t mec;
354 uint32_t pipe;
355
356 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
357 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
358
359 lock_srbm(kgd, mec, pipe, 0, 0);
360
361 WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL),
362 CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
363 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
364
365 unlock_srbm(kgd);
366
367 return 0;
368 }
369
get_sdma_base_addr(struct amdgpu_device * adev,unsigned int engine_id,unsigned int queue_id)370 static uint32_t get_sdma_base_addr(struct amdgpu_device *adev,
371 unsigned int engine_id,
372 unsigned int queue_id)
373 {
374 uint32_t base[2] = {
375 SOC15_REG_OFFSET(SDMA0, 0,
376 mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL,
377 SOC15_REG_OFFSET(SDMA1, 0,
378 mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL
379 };
380 uint32_t retval;
381
382 retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL -
383 mmSDMA0_RLC0_RB_CNTL);
384
385 pr_debug("sdma base address: 0x%x\n", retval);
386
387 return retval;
388 }
389
get_mqd(void * mqd)390 static inline struct v9_mqd *get_mqd(void *mqd)
391 {
392 return (struct v9_mqd *)mqd;
393 }
394
get_sdma_mqd(void * mqd)395 static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
396 {
397 return (struct v9_sdma_mqd *)mqd;
398 }
399
kgd_hqd_load(struct kgd_dev * kgd,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t __user * wptr,uint32_t wptr_shift,uint32_t wptr_mask,struct mm_struct * mm)400 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
401 uint32_t queue_id, uint32_t __user *wptr,
402 uint32_t wptr_shift, uint32_t wptr_mask,
403 struct mm_struct *mm)
404 {
405 struct amdgpu_device *adev = get_amdgpu_device(kgd);
406 struct v9_mqd *m;
407 uint32_t *mqd_hqd;
408 uint32_t reg, hqd_base, data;
409
410 m = get_mqd(mqd);
411
412 acquire_queue(kgd, pipe_id, queue_id);
413
414 /* HIQ is set during driver init period with vmid set to 0*/
415 if (m->cp_hqd_vmid == 0) {
416 uint32_t value, mec, pipe;
417
418 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
419 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
420
421 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
422 mec, pipe, queue_id);
423 value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
424 value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
425 ((mec << 5) | (pipe << 3) | queue_id | 0x80));
426 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
427 }
428
429 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
430 mqd_hqd = &m->cp_mqd_base_addr_lo;
431 hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
432
433 for (reg = hqd_base;
434 reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
435 WREG32(reg, mqd_hqd[reg - hqd_base]);
436
437
438 /* Activate doorbell logic before triggering WPTR poll. */
439 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
440 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
441 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
442
443 if (wptr) {
444 /* Don't read wptr with get_user because the user
445 * context may not be accessible (if this function
446 * runs in a work queue). Instead trigger a one-shot
447 * polling read from memory in the CP. This assumes
448 * that wptr is GPU-accessible in the queue's VMID via
449 * ATC or SVM. WPTR==RPTR before starting the poll so
450 * the CP starts fetching new commands from the right
451 * place.
452 *
453 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
454 * tricky. Assume that the queue didn't overflow. The
455 * number of valid bits in the 32-bit RPTR depends on
456 * the queue size. The remaining bits are taken from
457 * the saved 64-bit WPTR. If the WPTR wrapped, add the
458 * queue size.
459 */
460 uint32_t queue_size =
461 2 << REG_GET_FIELD(m->cp_hqd_pq_control,
462 CP_HQD_PQ_CONTROL, QUEUE_SIZE);
463 uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
464
465 if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
466 guessed_wptr += queue_size;
467 guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
468 guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
469
470 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
471 lower_32_bits(guessed_wptr));
472 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
473 upper_32_bits(guessed_wptr));
474 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
475 lower_32_bits((uintptr_t)wptr));
476 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
477 upper_32_bits((uintptr_t)wptr));
478 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1),
479 get_queue_mask(adev, pipe_id, queue_id));
480 }
481
482 /* Start the EOP fetcher */
483 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
484 REG_SET_FIELD(m->cp_hqd_eop_rptr,
485 CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
486
487 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
488 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
489
490 release_queue(kgd);
491
492 return 0;
493 }
494
kgd_hqd_dump(struct kgd_dev * kgd,uint32_t pipe_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)495 static int kgd_hqd_dump(struct kgd_dev *kgd,
496 uint32_t pipe_id, uint32_t queue_id,
497 uint32_t (**dump)[2], uint32_t *n_regs)
498 {
499 struct amdgpu_device *adev = get_amdgpu_device(kgd);
500 uint32_t i = 0, reg;
501 #define HQD_N_REGS 56
502 #define DUMP_REG(addr) do { \
503 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
504 break; \
505 (*dump)[i][0] = (addr) << 2; \
506 (*dump)[i++][1] = RREG32(addr); \
507 } while (0)
508
509 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
510 if (*dump == NULL)
511 return -ENOMEM;
512
513 acquire_queue(kgd, pipe_id, queue_id);
514
515 for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
516 reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
517 DUMP_REG(reg);
518
519 release_queue(kgd);
520
521 WARN_ON_ONCE(i != HQD_N_REGS);
522 *n_regs = i;
523
524 return 0;
525 }
526
kgd_hqd_sdma_load(struct kgd_dev * kgd,void * mqd,uint32_t __user * wptr,struct mm_struct * mm)527 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
528 uint32_t __user *wptr, struct mm_struct *mm)
529 {
530 struct amdgpu_device *adev = get_amdgpu_device(kgd);
531 struct v9_sdma_mqd *m;
532 uint32_t sdma_base_addr, sdmax_gfx_context_cntl;
533 unsigned long end_jiffies;
534 uint32_t data;
535 uint64_t data64;
536 uint64_t __user *wptr64 = (uint64_t __user *)wptr;
537
538 m = get_sdma_mqd(mqd);
539 sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
540 m->sdma_queue_id);
541 sdmax_gfx_context_cntl = m->sdma_engine_id ?
542 SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) :
543 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL);
544
545 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
546 m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
547
548 end_jiffies = msecs_to_jiffies(2000) + jiffies;
549 while (true) {
550 data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
551 if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
552 break;
553 if (time_after(jiffies, end_jiffies))
554 return -ETIME;
555 usleep_range(500, 1000);
556 }
557 data = RREG32(sdmax_gfx_context_cntl);
558 data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
559 RESUME_CTX, 0);
560 WREG32(sdmax_gfx_context_cntl, data);
561
562 WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET,
563 m->sdmax_rlcx_doorbell_offset);
564
565 data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
566 ENABLE, 1);
567 WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
568 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
569 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI,
570 m->sdmax_rlcx_rb_rptr_hi);
571
572 WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);
573 if (read_user_wptr(mm, wptr64, data64)) {
574 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
575 lower_32_bits(data64));
576 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
577 upper_32_bits(data64));
578 } else {
579 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
580 m->sdmax_rlcx_rb_rptr);
581 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
582 m->sdmax_rlcx_rb_rptr_hi);
583 }
584 WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);
585
586 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
587 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
588 m->sdmax_rlcx_rb_base_hi);
589 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
590 m->sdmax_rlcx_rb_rptr_addr_lo);
591 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
592 m->sdmax_rlcx_rb_rptr_addr_hi);
593
594 data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
595 RB_ENABLE, 1);
596 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
597
598 return 0;
599 }
600
kgd_hqd_sdma_dump(struct kgd_dev * kgd,uint32_t engine_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)601 static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
602 uint32_t engine_id, uint32_t queue_id,
603 uint32_t (**dump)[2], uint32_t *n_regs)
604 {
605 struct amdgpu_device *adev = get_amdgpu_device(kgd);
606 uint32_t sdma_base_addr = get_sdma_base_addr(adev, engine_id, queue_id);
607 uint32_t i = 0, reg;
608 #undef HQD_N_REGS
609 #define HQD_N_REGS (19+6+7+10)
610
611 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
612 if (*dump == NULL)
613 return -ENOMEM;
614
615 for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
616 DUMP_REG(sdma_base_addr + reg);
617 for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)
618 DUMP_REG(sdma_base_addr + reg);
619 for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;
620 reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)
621 DUMP_REG(sdma_base_addr + reg);
622 for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;
623 reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)
624 DUMP_REG(sdma_base_addr + reg);
625
626 WARN_ON_ONCE(i != HQD_N_REGS);
627 *n_regs = i;
628
629 return 0;
630 }
631
kgd_hqd_is_occupied(struct kgd_dev * kgd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)632 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
633 uint32_t pipe_id, uint32_t queue_id)
634 {
635 struct amdgpu_device *adev = get_amdgpu_device(kgd);
636 uint32_t act;
637 bool retval = false;
638 uint32_t low, high;
639
640 acquire_queue(kgd, pipe_id, queue_id);
641 act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
642 if (act) {
643 low = lower_32_bits(queue_address >> 8);
644 high = upper_32_bits(queue_address >> 8);
645
646 if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) &&
647 high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI)))
648 retval = true;
649 }
650 release_queue(kgd);
651 return retval;
652 }
653
kgd_hqd_sdma_is_occupied(struct kgd_dev * kgd,void * mqd)654 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
655 {
656 struct amdgpu_device *adev = get_amdgpu_device(kgd);
657 struct v9_sdma_mqd *m;
658 uint32_t sdma_base_addr;
659 uint32_t sdma_rlc_rb_cntl;
660
661 m = get_sdma_mqd(mqd);
662 sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
663 m->sdma_queue_id);
664
665 sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
666
667 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
668 return true;
669
670 return false;
671 }
672
kgd_hqd_destroy(struct kgd_dev * kgd,void * mqd,enum kfd_preempt_type reset_type,unsigned int utimeout,uint32_t pipe_id,uint32_t queue_id)673 static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
674 enum kfd_preempt_type reset_type,
675 unsigned int utimeout, uint32_t pipe_id,
676 uint32_t queue_id)
677 {
678 struct amdgpu_device *adev = get_amdgpu_device(kgd);
679 enum hqd_dequeue_request_type type;
680 unsigned long end_jiffies;
681 uint32_t temp;
682 struct v9_mqd *m = get_mqd(mqd);
683
684 if (adev->in_gpu_reset)
685 return -EIO;
686
687 acquire_queue(kgd, pipe_id, queue_id);
688
689 if (m->cp_hqd_vmid == 0)
690 WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
691
692 switch (reset_type) {
693 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
694 type = DRAIN_PIPE;
695 break;
696 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
697 type = RESET_WAVES;
698 break;
699 default:
700 type = DRAIN_PIPE;
701 break;
702 }
703
704 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
705
706 end_jiffies = (utimeout * HZ / 1000) + jiffies;
707 while (true) {
708 temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
709 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
710 break;
711 if (time_after(jiffies, end_jiffies)) {
712 pr_err("cp queue preemption time out.\n");
713 release_queue(kgd);
714 return -ETIME;
715 }
716 usleep_range(500, 1000);
717 }
718
719 release_queue(kgd);
720 return 0;
721 }
722
kgd_hqd_sdma_destroy(struct kgd_dev * kgd,void * mqd,unsigned int utimeout)723 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
724 unsigned int utimeout)
725 {
726 struct amdgpu_device *adev = get_amdgpu_device(kgd);
727 struct v9_sdma_mqd *m;
728 uint32_t sdma_base_addr;
729 uint32_t temp;
730 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
731
732 m = get_sdma_mqd(mqd);
733 sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
734 m->sdma_queue_id);
735
736 temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
737 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
738 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
739
740 while (true) {
741 temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
742 if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
743 break;
744 if (time_after(jiffies, end_jiffies))
745 return -ETIME;
746 usleep_range(500, 1000);
747 }
748
749 WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
750 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
751 RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
752 SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
753
754 m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
755 m->sdmax_rlcx_rb_rptr_hi =
756 RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI);
757
758 return 0;
759 }
760
get_atc_vmid_pasid_mapping_valid(struct kgd_dev * kgd,uint8_t vmid)761 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
762 uint8_t vmid)
763 {
764 uint32_t reg;
765 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
766
767 reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
768 + vmid);
769 return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
770 }
771
get_atc_vmid_pasid_mapping_pasid(struct kgd_dev * kgd,uint8_t vmid)772 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
773 uint8_t vmid)
774 {
775 uint32_t reg;
776 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
777
778 reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
779 + vmid);
780 return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
781 }
782
write_vmid_invalidate_request(struct kgd_dev * kgd,uint8_t vmid)783 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
784 {
785 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
786 uint32_t req = (1 << vmid) |
787 (0 << VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT) | /* legacy */
788 VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK |
789 VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK |
790 VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK |
791 VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK |
792 VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK;
793
794 mutex_lock(&adev->srbm_mutex);
795
796 /* Use legacy mode tlb invalidation.
797 *
798 * Currently on Raven the code below is broken for anything but
799 * legacy mode due to a MMHUB power gating problem. A workaround
800 * is for MMHUB to wait until the condition PER_VMID_INVALIDATE_REQ
801 * == PER_VMID_INVALIDATE_ACK instead of simply waiting for the ack
802 * bit.
803 *
804 * TODO 1: agree on the right set of invalidation registers for
805 * KFD use. Use the last one for now. Invalidate both GC and
806 * MMHUB.
807 *
808 * TODO 2: support range-based invalidation, requires kfg2kgd
809 * interface change
810 */
811 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
812 0xffffffff);
813 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
814 0x0000001f);
815
816 WREG32(SOC15_REG_OFFSET(MMHUB, 0,
817 mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
818 0xffffffff);
819 WREG32(SOC15_REG_OFFSET(MMHUB, 0,
820 mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
821 0x0000001f);
822
823 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_REQ), req);
824
825 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_INVALIDATE_ENG16_REQ),
826 req);
827
828 while (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ACK)) &
829 (1 << vmid)))
830 cpu_relax();
831
832 while (!(RREG32(SOC15_REG_OFFSET(MMHUB, 0,
833 mmMMHUB_VM_INVALIDATE_ENG16_ACK)) &
834 (1 << vmid)))
835 cpu_relax();
836
837 mutex_unlock(&adev->srbm_mutex);
838
839 }
840
invalidate_tlbs_with_kiq(struct amdgpu_device * adev,uint16_t pasid)841 static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
842 {
843 signed long r;
844 uint32_t seq;
845 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
846
847 spin_lock(&adev->gfx.kiq.ring_lock);
848 amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/
849 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
850 amdgpu_ring_write(ring,
851 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
852 PACKET3_INVALIDATE_TLBS_ALL_HUB(1) |
853 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
854 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(0)); /* legacy */
855 amdgpu_fence_emit_polling(ring, &seq);
856 amdgpu_ring_commit(ring);
857 spin_unlock(&adev->gfx.kiq.ring_lock);
858
859 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
860 if (r < 1) {
861 DRM_ERROR("wait for kiq fence error: %ld.\n", r);
862 return -ETIME;
863 }
864
865 return 0;
866 }
867
invalidate_tlbs(struct kgd_dev * kgd,uint16_t pasid)868 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
869 {
870 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
871 int vmid;
872 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
873
874 if (adev->in_gpu_reset)
875 return -EIO;
876
877 if (ring->ready)
878 return invalidate_tlbs_with_kiq(adev, pasid);
879
880 for (vmid = 0; vmid < 16; vmid++) {
881 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
882 continue;
883 if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) {
884 if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid)
885 == pasid) {
886 write_vmid_invalidate_request(kgd, vmid);
887 break;
888 }
889 }
890 }
891
892 return 0;
893 }
894
invalidate_tlbs_vmid(struct kgd_dev * kgd,uint16_t vmid)895 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)
896 {
897 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
898
899 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
900 pr_err("non kfd vmid %d\n", vmid);
901 return 0;
902 }
903
904 write_vmid_invalidate_request(kgd, vmid);
905 return 0;
906 }
907
kgd_address_watch_disable(struct kgd_dev * kgd)908 static int kgd_address_watch_disable(struct kgd_dev *kgd)
909 {
910 return 0;
911 }
912
kgd_address_watch_execute(struct kgd_dev * kgd,unsigned int watch_point_id,uint32_t cntl_val,uint32_t addr_hi,uint32_t addr_lo)913 static int kgd_address_watch_execute(struct kgd_dev *kgd,
914 unsigned int watch_point_id,
915 uint32_t cntl_val,
916 uint32_t addr_hi,
917 uint32_t addr_lo)
918 {
919 return 0;
920 }
921
kgd_wave_control_execute(struct kgd_dev * kgd,uint32_t gfx_index_val,uint32_t sq_cmd)922 static int kgd_wave_control_execute(struct kgd_dev *kgd,
923 uint32_t gfx_index_val,
924 uint32_t sq_cmd)
925 {
926 struct amdgpu_device *adev = get_amdgpu_device(kgd);
927 uint32_t data = 0;
928
929 mutex_lock(&adev->grbm_idx_mutex);
930
931 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val);
932 WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd);
933
934 data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
935 INSTANCE_BROADCAST_WRITES, 1);
936 data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
937 SH_BROADCAST_WRITES, 1);
938 data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
939 SE_BROADCAST_WRITES, 1);
940
941 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
942 mutex_unlock(&adev->grbm_idx_mutex);
943
944 return 0;
945 }
946
kgd_address_watch_get_offset(struct kgd_dev * kgd,unsigned int watch_point_id,unsigned int reg_offset)947 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
948 unsigned int watch_point_id,
949 unsigned int reg_offset)
950 {
951 return 0;
952 }
953
set_scratch_backing_va(struct kgd_dev * kgd,uint64_t va,uint32_t vmid)954 static void set_scratch_backing_va(struct kgd_dev *kgd,
955 uint64_t va, uint32_t vmid)
956 {
957 /* No longer needed on GFXv9. The scratch base address is
958 * passed to the shader by the CP. It's the user mode driver's
959 * responsibility.
960 */
961 }
962
963 /* FIXME: Does this need to be ASIC-specific code? */
get_fw_version(struct kgd_dev * kgd,enum kgd_engine_type type)964 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
965 {
966 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
967 const union amdgpu_firmware_header *hdr;
968
969 switch (type) {
970 case KGD_ENGINE_PFP:
971 hdr = (const union amdgpu_firmware_header *)adev->gfx.pfp_fw->data;
972 break;
973
974 case KGD_ENGINE_ME:
975 hdr = (const union amdgpu_firmware_header *)adev->gfx.me_fw->data;
976 break;
977
978 case KGD_ENGINE_CE:
979 hdr = (const union amdgpu_firmware_header *)adev->gfx.ce_fw->data;
980 break;
981
982 case KGD_ENGINE_MEC1:
983 hdr = (const union amdgpu_firmware_header *)adev->gfx.mec_fw->data;
984 break;
985
986 case KGD_ENGINE_MEC2:
987 hdr = (const union amdgpu_firmware_header *)adev->gfx.mec2_fw->data;
988 break;
989
990 case KGD_ENGINE_RLC:
991 hdr = (const union amdgpu_firmware_header *)adev->gfx.rlc_fw->data;
992 break;
993
994 case KGD_ENGINE_SDMA1:
995 hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[0].fw->data;
996 break;
997
998 case KGD_ENGINE_SDMA2:
999 hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[1].fw->data;
1000 break;
1001
1002 default:
1003 return 0;
1004 }
1005
1006 if (hdr == NULL)
1007 return 0;
1008
1009 /* Only 12 bit in use*/
1010 return hdr->common.ucode_version;
1011 }
1012
set_vm_context_page_table_base(struct kgd_dev * kgd,uint32_t vmid,uint32_t page_table_base)1013 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
1014 uint32_t page_table_base)
1015 {
1016 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1017 uint64_t base = (uint64_t)page_table_base << PAGE_SHIFT |
1018 AMDGPU_PTE_VALID;
1019
1020 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
1021 pr_err("trying to set page table base for wrong VMID %u\n",
1022 vmid);
1023 return;
1024 }
1025
1026 /* TODO: take advantage of per-process address space size. For
1027 * now, all processes share the same address space size, like
1028 * on GFX8 and older.
1029 */
1030 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
1031 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
1032
1033 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
1034 lower_32_bits(adev->vm_manager.max_pfn - 1));
1035 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
1036 upper_32_bits(adev->vm_manager.max_pfn - 1));
1037
1038 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
1039 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
1040
1041 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
1042 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
1043
1044 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
1045 lower_32_bits(adev->vm_manager.max_pfn - 1));
1046 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
1047 upper_32_bits(adev->vm_manager.max_pfn - 1));
1048
1049 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
1050 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
1051 }
1052
