60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
63 #define mmGCEA_PROBE_MAP                        0x070c
64 #define mmGCEA_PROBE_MAP_BASE_IDX               0
130 #define mmTCP_CHAN_STEER_0_ARCT								0x0b03
131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX							0
132 #define mmTCP_CHAN_STEER_1_ARCT								0x0b04
133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX							0
134 #define mmTCP_CHAN_STEER_2_ARCT								0x0b09
135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX							0
136 #define mmTCP_CHAN_STEER_3_ARCT								0x0b0a
137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX							0
138 #define mmTCP_CHAN_STEER_4_ARCT								0x0b0b
139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX							0
140 #define mmTCP_CHAN_STEER_5_ARCT								0x0b0c
141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX							0
145 	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
186 	/* SQC range 0*/
242 	/* TCC range 0*/
305 	/* EA range 0*/
361 	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
362 	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
363 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
364 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
365 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
366 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
367 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
368 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
369 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
370 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
371 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
372 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
373 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
374 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
375 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
376 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
377 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
378 			     0),
379 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
380 			     0),
381 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
382 	AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
383 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
384 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
385 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
386 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
387 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
388 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
389 			     0, 0),
390 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
391 			     0),
392 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
393 			     0, 0),
394 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
395 			     0),
396 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
397 			     0, 0),
398 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
399 			     0),
400 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
402 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
403 			     0, 0, 0),
404 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
405 			     0),
406 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
407 			     0),
408 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
409 			     0),
410 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
411 			     0),
412 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
413 			     0),
414 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
415 			     0, 0),
416 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
417 			     0),
418 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
419 			     0),
420 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
421 			     0, 0, 0),
422 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
423 			     0),
424 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
425 			     0),
426 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
427 			     0),
428 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
429 			     0),
430 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
431 			     0),
432 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
433 			     0, 0),
434 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
435 			     0),
436 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
437 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
438 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
439 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
440 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
441 	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
442 	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
443 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
444 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
446 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
448 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
450 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
451 			     0),
452 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
453 			     0),
454 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
455 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
456 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
457 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
458 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
459 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
460 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
461 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
462 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
463 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
464 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
465 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
466 			     0),
467 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
468 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
469 			     0),
470 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
471 			     0, 0),
472 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
473 			     0),
474 	AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
475 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
476 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
477 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
478 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
479 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
480 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
481 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
482 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
483 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
484 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
485 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
486 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
487 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
488 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
489 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
490 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
491 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
492 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
493 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
494 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
495 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
496 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
497 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
498 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
499 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
500 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
501 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
502 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
503 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
504 	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
505 	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
506 	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
507 	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
512 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
513 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
514 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
515 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
516 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
517 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
518 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
519 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
520 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
521 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
522 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
523 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
524 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
525 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
526 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
527 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
528 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
529 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
530 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
531 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
536 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
537 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
538 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
539 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
540 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
541 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
542 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
543 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
544 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
545 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
546 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
547 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
548 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
549 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
550 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
551 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
552 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
553 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
558 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
559 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
560 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
561 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
562 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
563 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
564 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
565 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
566 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
567 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
568 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
573 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
574 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
575 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
576 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
577 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
578 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
579 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
580 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
581 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
582 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
583 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
584 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
585 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
586 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
587 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
588 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
589 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
590 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
591 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
592 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
593 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
594 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
595 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
596 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
601 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
602 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
603 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
604 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
605 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
606 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
607 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
612 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
613 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
614 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
615 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
616 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
617 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
618 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
619 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
620 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
621 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
622 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
623 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
624 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
625 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
626 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
627 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
628 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
629 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
630 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
635 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
636 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
637 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
638 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
639 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
640 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
641 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
642 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
643 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
644 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
645 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
646 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
651 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
652 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
653 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
658 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
659 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
660 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
661 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
662 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
663 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
664 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
665 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
666 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
667 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
668 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
669 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
670 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
671 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
672 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
673 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
678 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
679 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
680 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
681 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
682 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
683 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
684 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
685 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
686 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
687 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
688 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
689 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
690 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
695 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
696 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
697 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
698 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
699 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
700 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
701 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
702 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
703 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
704 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
705 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
709 	{SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
710 	{SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
747 …scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_RE…  in gfx_v9_0_rlcg_w()
748 …scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_RE…  in gfx_v9_0_rlcg_w()
749 …scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_RE…  in gfx_v9_0_rlcg_w()
750 …scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_RE…  in gfx_v9_0_rlcg_w()
751 …spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_IN…  in gfx_v9_0_rlcg_w()
753 	grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;  in gfx_v9_0_rlcg_w()
754 	grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;  in gfx_v9_0_rlcg_w()
769 		uint32_t i = 0;  in gfx_v9_0_rlcg_w()
773 		writel(offset | 0x80000000, scratch_reg1);  in gfx_v9_0_rlcg_w()
775 		for (i = 0; i < retries; i++) {  in gfx_v9_0_rlcg_w()
779 			if (!(tmp & 0x80000000))  in gfx_v9_0_rlcg_w()
785 			pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);  in gfx_v9_0_rlcg_w()
806 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
807 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
808 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
809 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
831 		PACKET3_SET_RESOURCES_VMID_MASK(0) |  in gfx_v9_0_kiq_set_resources()
832 		/* vmid_mask:0* queue_type:0 (KIQ) */  in gfx_v9_0_kiq_set_resources()
833 		PACKET3_SET_RESOURCES_QUEUE_TYPE(0));  in gfx_v9_0_kiq_set_resources()
838 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */  in gfx_v9_0_kiq_set_resources()
839 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */  in gfx_v9_0_kiq_set_resources()
840 	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */  in gfx_v9_0_kiq_set_resources()
841 	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */  in gfx_v9_0_kiq_set_resources()
850 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_map_queues()
853 	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/  in gfx_v9_0_kiq_map_queues()
854 	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */  in gfx_v9_0_kiq_map_queues()
855 			 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */  in gfx_v9_0_kiq_map_queues()
856 			 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */  in gfx_v9_0_kiq_map_queues()
859 			 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |  in gfx_v9_0_kiq_map_queues()
861 			 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |  in gfx_v9_0_kiq_map_queues()
863 			 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |  in gfx_v9_0_kiq_map_queues()
880 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_unmap_queues()
883 	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */  in gfx_v9_0_kiq_unmap_queues()
885 			  PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |  in gfx_v9_0_kiq_unmap_queues()
896 		amdgpu_ring_write(kiq_ring, 0);  in gfx_v9_0_kiq_unmap_queues()
897 		amdgpu_ring_write(kiq_ring, 0);  in gfx_v9_0_kiq_unmap_queues()
898 		amdgpu_ring_write(kiq_ring, 0);  in gfx_v9_0_kiq_unmap_queues()
907 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_query_status()
911 			  PACKET3_QUERY_STATUS_CONTEXT_ID(0) |  in gfx_v9_0_kiq_query_status()
912 			  PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |  in gfx_v9_0_kiq_query_status()
914 	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */  in gfx_v9_0_kiq_query_status()
928 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));  in gfx_v9_0_kiq_invalidate_tlbs()
1020 	adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);  in gfx_v9_0_scratch_init()
1029 				WRITE_DATA_DST_SEL(0) |  in gfx_v9_0_write_data_to_reg()
1030 				(wc ? WR_CONFIRM : 0));  in gfx_v9_0_write_data_to_reg()
1032 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_write_data_to_reg()
1043 				 /* memory (1) or register (0) */  in gfx_v9_0_wait_reg_mem()
1050 		BUG_ON(addr0 & 0x3); /* Dword align */  in gfx_v9_0_wait_reg_mem()
1062 	uint32_t tmp = 0;  in gfx_v9_0_ring_test_ring()
1070 	WREG32(scratch, 0xCAFEDEAD);  in gfx_v9_0_ring_test_ring()
1077 	amdgpu_ring_write(ring, 0xDEADBEEF);  in gfx_v9_0_ring_test_ring()
1080 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_ring_test_ring()
1082 		if (tmp == 0xDEADBEEF)  in gfx_v9_0_ring_test_ring()
1111 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);  in gfx_v9_0_ring_test_ib()
1112 	memset(&ib, 0, sizeof(ib));  in gfx_v9_0_ring_test_ib()
1118 	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);  in gfx_v9_0_ring_test_ib()
1122 	ib.ptr[4] = 0xDEADBEEF;  in gfx_v9_0_ring_test_ib()
1130 	if (r == 0) {  in gfx_v9_0_ring_test_ib()
1133 	} else if (r < 0) {  in gfx_v9_0_ring_test_ib()
1138 	if (tmp == 0xDEADBEEF)  in gfx_v9_0_ring_test_ib()
1139 		r = 0;  in gfx_v9_0_ring_test_ib()
1197 	    ((adev->gfx.mec_fw_version < 0x000001a5) ||  in gfx_v9_0_check_fw_write_wait()
1199 	    (adev->gfx.pfp_fw_version < 0x000000b7) ||  in gfx_v9_0_check_fw_write_wait()
1205 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1207 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
1211 		if ((adev->gfx.mec_fw_version >=  0x00000193) &&  in gfx_v9_0_check_fw_write_wait()
1216 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1218 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
1222 		if ((adev->gfx.mec_fw_version >=  0x00000196) &&  in gfx_v9_0_check_fw_write_wait()
1227 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1229 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
1233 		if ((adev->gfx.mec_fw_version >=  0x00000197) &&  in gfx_v9_0_check_fw_write_wait()
1238 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1240 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
1244 		if ((adev->gfx.mec_fw_version >=  0x00000192) &&  in gfx_v9_0_check_fw_write_wait()
1265 	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1267 	{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1269 	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1270 	{ 0, 0, 0, 0, 0 },
1277 	while (p && p->chip_device != 0) {  in gfx_v9_0_should_disable_gfxoff()
1292 	if (adev->pm.fw_version >= 0x41e2b)  in is_raven_kicker()
1301 	    (adev->gfx.me_fw_version >= 0x000000a5) &&  in check_if_enlarge_doorbell_range()
1432 	unsigned int i = 0;  in gfx_v9_0_init_rlc_microcode()
1441 	 * PCO AM4: revision >= 0xC8 && revision <= 0xCF  in gfx_v9_0_init_rlc_microcode()
1442 	 *          or revision >= 0xD8 && revision <= 0xDF  in gfx_v9_0_init_rlc_microcode()
1446 		(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||  in gfx_v9_0_init_rlc_microcode()
1447 		((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))  in gfx_v9_0_init_rlc_microcode()
1449 	else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&  in gfx_v9_0_init_rlc_microcode()
1450 		(smu_version >= 0x41e2b))  in gfx_v9_0_init_rlc_microcode()
1498 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)  in gfx_v9_0_init_rlc_microcode()
1505 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)  in gfx_v9_0_init_rlc_microcode()
1599 			err = 0;  in gfx_v9_0_init_cp_compute_microcode()
1720 	u32 count = 0;  in gfx_v9_0_get_csb_size()
1734 				return 0;  in gfx_v9_0_get_csb_size()
1749 	u32 count = 0, i;  in gfx_v9_0_get_csb_buffer()
1758 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_get_csb_buffer()
1762 	buffer[count++] = cpu_to_le32(0x80000000);  in gfx_v9_0_get_csb_buffer()
1763 	buffer[count++] = cpu_to_le32(0x80000000);  in gfx_v9_0_get_csb_buffer()
1772 				for (i = 0; i < ext->reg_count; i++)  in gfx_v9_0_get_csb_buffer()
1780 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_get_csb_buffer()
1783 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));  in gfx_v9_0_get_csb_buffer()
1784 	buffer[count++] = cpu_to_le32(0);  in gfx_v9_0_get_csb_buffer()
1803 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_init_always_on_cu_mask()
1804 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_init_always_on_cu_mask()
1806 			cu_bitmap = 0;  in gfx_v9_0_init_always_on_cu_mask()
1807 			counter = 0;  in gfx_v9_0_init_always_on_cu_mask()
1808 			gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);  in gfx_v9_0_init_always_on_cu_mask()
1810 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_init_always_on_cu_mask()
1813 						WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);  in gfx_v9_0_init_always_on_cu_mask()
1823 			WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);  in gfx_v9_0_init_always_on_cu_mask()
1827 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_init_always_on_cu_mask()
1836 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);  in gfx_v9_0_init_lbpw()
1837 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);  in gfx_v9_0_init_lbpw()
1838 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);  in gfx_v9_0_init_lbpw()
1839 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));  in gfx_v9_0_init_lbpw()
1841 	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */  in gfx_v9_0_init_lbpw()
1842 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);  in gfx_v9_0_init_lbpw()
1844 	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */  in gfx_v9_0_init_lbpw()
1845 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);  in gfx_v9_0_init_lbpw()
1849 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_init_lbpw()
1850 	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);  in gfx_v9_0_init_lbpw()
1852 	/* set mmRLC_LB_PARAMS = 0x003F_1006 */  in gfx_v9_0_init_lbpw()
1853 	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);  in gfx_v9_0_init_lbpw()
1854 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);  in gfx_v9_0_init_lbpw()
1855 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);  in gfx_v9_0_init_lbpw()
1856 	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);  in gfx_v9_0_init_lbpw()
1858 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */  in gfx_v9_0_init_lbpw()
1859 	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);  in gfx_v9_0_init_lbpw()
1860 	data &= 0x0000FFFF;  in gfx_v9_0_init_lbpw()
1861 	data |= 0x00C00000;  in gfx_v9_0_init_lbpw()
1862 	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);  in gfx_v9_0_init_lbpw()
1865 	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),  in gfx_v9_0_init_lbpw()
1869 	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,  in gfx_v9_0_init_lbpw()
1872 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);  in gfx_v9_0_init_lbpw()
1873 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);  in gfx_v9_0_init_lbpw()
1874 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);  in gfx_v9_0_init_lbpw()
1885 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);  in gfx_v9_4_init_lbpw()
1886 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);  in gfx_v9_4_init_lbpw()
1887 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);  in gfx_v9_4_init_lbpw()
1888 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));  in gfx_v9_4_init_lbpw()
1890 	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */  in gfx_v9_4_init_lbpw()
1891 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);  in gfx_v9_4_init_lbpw()
1893 	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */  in gfx_v9_4_init_lbpw()
1894 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);  in gfx_v9_4_init_lbpw()
1898 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_4_init_lbpw()
1899 	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);  in gfx_v9_4_init_lbpw()
1901 	/* set mmRLC_LB_PARAMS = 0x003F_1006 */  in gfx_v9_4_init_lbpw()
1902 	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);  in gfx_v9_4_init_lbpw()
1903 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);  in gfx_v9_4_init_lbpw()
1904 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);  in gfx_v9_4_init_lbpw()
1905 	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);  in gfx_v9_4_init_lbpw()
1907 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */  in gfx_v9_4_init_lbpw()
1908 	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);  in gfx_v9_4_init_lbpw()
1909 	data &= 0x0000FFFF;  in gfx_v9_4_init_lbpw()
1910 	data |= 0x00C00000;  in gfx_v9_4_init_lbpw()
1911 	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);  in gfx_v9_4_init_lbpw()
1914 	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),  in gfx_v9_4_init_lbpw()
1918 	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,  in gfx_v9_4_init_lbpw()
1921 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);  in gfx_v9_4_init_lbpw()
1922 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);  in gfx_v9_4_init_lbpw()
1923 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);  in gfx_v9_4_init_lbpw()
1931 	WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);  in gfx_v9_0_enable_lbpw()
1977 	/* init spm vmid with 0xf */  in gfx_v9_0_rlc_init()
1979 		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);  in gfx_v9_0_rlc_init()
1981 	return 0;  in gfx_v9_0_rlc_init()
2018 		memset(hpd, 0, mec_hpd_size);  in gfx_v9_0_mec_init()
2047 	return 0;  in gfx_v9_0_mec_init()
2052 	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,  in wave_read_ind()
2057 	return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);  in wave_read_ind()
2064 	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,  in wave_read_regs()
2072 		*(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);  in wave_read_regs()
2101 		adev, simd, wave, 0,  in gfx_v9_0_read_wave_sgprs()
2148 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2149 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2150 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2151 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2156 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2157 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2158 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2159 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2166 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2167 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2168 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2169 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2170 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);  in gfx_v9_0_gpu_early_init()
2171 		gb_addr_config &= ~0xf3e777ff;  in gfx_v9_0_gpu_early_init()
2172 		gb_addr_config |= 0x22014042;  in gfx_v9_0_gpu_early_init()
2180 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2181 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2182 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2183 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2192 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2193 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2194 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2195 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2196 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);  in gfx_v9_0_gpu_early_init()
2197 		gb_addr_config &= ~0xf3e777ff;  in gfx_v9_0_gpu_early_init()
2198 		gb_addr_config |= 0x22014042;  in gfx_v9_0_gpu_early_init()
2202 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2203 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2204 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;  in gfx_v9_0_gpu_early_init()
2205 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2206 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);  in gfx_v9_0_gpu_early_init()
2207 		gb_addr_config &= ~0xf3e777ff;  in gfx_v9_0_gpu_early_init()
2208 		gb_addr_config |= 0x22010042;  in gfx_v9_0_gpu_early_init()
2213 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2214 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2215 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2216 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2217 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);  in gfx_v9_0_gpu_early_init()
2218 		gb_addr_config &= ~0xf3e777ff;  in gfx_v9_0_gpu_early_init()
2219 		gb_addr_config |= 0x22014042;  in gfx_v9_0_gpu_early_init()
2267 	return 0;  in gfx_v9_0_gpu_early_init()
2378 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in gfx_v9_0_sw_init()
2395 	ring_id = 0;  in gfx_v9_0_sw_init()
2396 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {  in gfx_v9_0_sw_init()
2397 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {  in gfx_v9_0_sw_init()
2398 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {  in gfx_v9_0_sw_init()
2429 	adev->gfx.ce_ram_size = 0x8000;  in gfx_v9_0_sw_init()
2435 	return 0;  in gfx_v9_0_sw_init()
2448 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_sw_fini()
2450 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_sw_fini()
2466 	return 0;  in gfx_v9_0_sw_fini()
2480 	if (instance == 0xffffffff)  in gfx_v9_0_select_se_sh()
2481 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);  in gfx_v9_0_select_se_sh()
2483 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);  in gfx_v9_0_select_se_sh()
2485 	if (se_num == 0xffffffff)  in gfx_v9_0_select_se_sh()
2490 	if (sh_num == 0xffffffff)  in gfx_v9_0_select_se_sh()
2495 	WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);  in gfx_v9_0_select_se_sh()
2502 	data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);  in gfx_v9_0_get_rb_active_bitmap()
2503 	data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);  in gfx_v9_0_get_rb_active_bitmap()
2518 	u32 active_rbs = 0;  in gfx_v9_0_setup_rb()
2523 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_setup_rb()
2524 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_setup_rb()
2525 			gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);  in gfx_v9_0_setup_rb()
2531 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_setup_rb()
2538 #define DEFAULT_SH_MEM_BASES	(0x6000)
2547 	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)  in gfx_v9_0_init_compute_vmid()
2548 	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)  in gfx_v9_0_init_compute_vmid()
2549 	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)  in gfx_v9_0_init_compute_vmid()
2559 		soc15_grbm_select(adev, 0, 0, 0, i);  in gfx_v9_0_init_compute_vmid()
2561 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);  in gfx_v9_0_init_compute_vmid()
2562 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);  in gfx_v9_0_init_compute_vmid()
2564 	soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_init_compute_vmid()
2570 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);  in gfx_v9_0_init_compute_vmid()
2571 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);  in gfx_v9_0_init_compute_vmid()
2572 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);  in gfx_v9_0_init_compute_vmid()
2573 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);  in gfx_v9_0_init_compute_vmid()
2588 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);  in gfx_v9_0_init_gds_vmid()
2589 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);  in gfx_v9_0_init_gds_vmid()
2590 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);  in gfx_v9_0_init_gds_vmid()
2591 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);  in gfx_v9_0_init_gds_vmid()
2601 		tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);  in gfx_v9_0_init_sq_config()
2604 		WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);  in gfx_v9_0_init_sq_config()
2616 	WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);  in gfx_v9_0_constants_init()
2622 	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);  in gfx_v9_0_constants_init()
2627 	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {  in gfx_v9_0_constants_init()
2628 		soc15_grbm_select(adev, 0, 0, 0, i);  in gfx_v9_0_constants_init()
2630 		if (i == 0) {  in gfx_v9_0_constants_init()
2631 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,  in gfx_v9_0_constants_init()
2635 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);  in gfx_v9_0_constants_init()
2636 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);  in gfx_v9_0_constants_init()
2638 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,  in gfx_v9_0_constants_init()
2642 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);  in gfx_v9_0_constants_init()
2643 			tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,  in gfx_v9_0_constants_init()
2647 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);  in gfx_v9_0_constants_init()
2650 	soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_constants_init()
2665 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_wait_for_rlc_serdes()
2666 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_wait_for_rlc_serdes()
2667 			gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);  in gfx_v9_0_wait_for_rlc_serdes()
2668 			for (k = 0; k < adev->usec_timeout; k++) {  in gfx_v9_0_wait_for_rlc_serdes()
2669 				if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)  in gfx_v9_0_wait_for_rlc_serdes()
2674 				gfx_v9_0_select_se_sh(adev, 0xffffffff,  in gfx_v9_0_wait_for_rlc_serdes()
2675 						      0xffffffff, 0xffffffff);  in gfx_v9_0_wait_for_rlc_serdes()
2683 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_wait_for_rlc_serdes()
2690 	for (k = 0; k < adev->usec_timeout; k++) {  in gfx_v9_0_wait_for_rlc_serdes()
2691 		if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)  in gfx_v9_0_wait_for_rlc_serdes()
2704 	tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);  in gfx_v9_0_enable_gui_idle_interrupt()
2706 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);  in gfx_v9_0_enable_gui_idle_interrupt()
2707 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);  in gfx_v9_0_enable_gui_idle_interrupt()
2708 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);  in gfx_v9_0_enable_gui_idle_interrupt()
2710 		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);  in gfx_v9_0_enable_gui_idle_interrupt()
2712 	WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);  in gfx_v9_0_enable_gui_idle_interrupt()
2719 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),  in gfx_v9_0_init_csb()
2721 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),  in gfx_v9_0_init_csb()
2722 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);  in gfx_v9_0_init_csb()
2723 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),  in gfx_v9_0_init_csb()
2743 		while (register_list_format[indirect_offset] != 0xFFFFFFFF) {  in gfx_v9_1_parse_ind_reg_list()
2747 			for (idx = 0; idx < unique_indirect_reg_count; idx++) {  in gfx_v9_1_parse_ind_reg_list()
2766 	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};  in gfx_v9_1_init_rlc_save_restore_list()
2767 	int unique_indirect_reg_count = 0;  in gfx_v9_1_init_rlc_save_restore_list()
2769 	int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};  in gfx_v9_1_init_rlc_save_restore_list()
2770 	int indirect_start_offsets_count = 0;  in gfx_v9_1_init_rlc_save_restore_list()
2772 	int list_size = 0;  in gfx_v9_1_init_rlc_save_restore_list()
2773 	int i = 0, j = 0;  in gfx_v9_1_init_rlc_save_restore_list()
2774 	u32 tmp = 0;  in gfx_v9_1_init_rlc_save_restore_list()
2794 	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));  in gfx_v9_1_init_rlc_save_restore_list()
2796 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);  in gfx_v9_1_init_rlc_save_restore_list()
2798 	/* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */  in gfx_v9_1_init_rlc_save_restore_list()
2799 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),  in gfx_v9_1_init_rlc_save_restore_list()
2801 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2802 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),  in gfx_v9_1_init_rlc_save_restore_list()
2806 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),  in gfx_v9_1_init_rlc_save_restore_list()
2810 	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2811 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),  in gfx_v9_1_init_rlc_save_restore_list()
2816 		if (register_list_format[i] == 0xFFFFFFFF) {  in gfx_v9_1_init_rlc_save_restore_list()
2817 			WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);  in gfx_v9_1_init_rlc_save_restore_list()
2821 		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);  in gfx_v9_1_init_rlc_save_restore_list()
2822 		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);  in gfx_v9_1_init_rlc_save_restore_list()
2824 		for (j = 0; j < unique_indirect_reg_count; j++) {  in gfx_v9_1_init_rlc_save_restore_list()
2826 				WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);  in gfx_v9_1_init_rlc_save_restore_list()
2839 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),  in gfx_v9_1_init_rlc_save_restore_list()
2841 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);  in gfx_v9_1_init_rlc_save_restore_list()
2844 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),  in gfx_v9_1_init_rlc_save_restore_list()
2846 	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)  in gfx_v9_1_init_rlc_save_restore_list()
2847 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),  in gfx_v9_1_init_rlc_save_restore_list()
2851 	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {  in gfx_v9_1_init_rlc_save_restore_list()
2852 		if (unique_indirect_regs[i] != 0) {  in gfx_v9_1_init_rlc_save_restore_list()
2853 			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)  in gfx_v9_1_init_rlc_save_restore_list()
2855 			       unique_indirect_regs[i] & 0x3FFFF);  in gfx_v9_1_init_rlc_save_restore_list()
2857 			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)  in gfx_v9_1_init_rlc_save_restore_list()
2864 	return 0;  in gfx_v9_1_init_rlc_save_restore_list()
2869 	WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);  in gfx_v9_0_enable_save_restore_machine()
2875 	uint32_t data = 0;  in pwr_10_0_gfxip_control_over_cgpg()
2876 	uint32_t default_data = 0;  in pwr_10_0_gfxip_control_over_cgpg()
2878 	default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));  in pwr_10_0_gfxip_control_over_cgpg()
2883 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);  in pwr_10_0_gfxip_control_over_cgpg()
2889 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);  in pwr_10_0_gfxip_control_over_cgpg()
2894 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);  in pwr_10_0_gfxip_control_over_cgpg()
2900 	uint32_t data = 0;  in gfx_v9_0_init_gfx_power_gating()
2906 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));  in gfx_v9_0_init_gfx_power_gating()
2908 		data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2909 		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);  in gfx_v9_0_init_gfx_power_gating()
2912 		data = 0;  in gfx_v9_0_init_gfx_power_gating()
2913 		data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2914 		data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2915 		data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2916 		data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2917 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);  in gfx_v9_0_init_gfx_power_gating()
2919 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));  in gfx_v9_0_init_gfx_power_gating()
2921 		data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2922 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);  in gfx_v9_0_init_gfx_power_gating()
2924 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));  in gfx_v9_0_init_gfx_power_gating()
2926 		data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2927 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);  in gfx_v9_0_init_gfx_power_gating()
2929 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));  in gfx_v9_0_init_gfx_power_gating()
2932 		/* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */  in gfx_v9_0_init_gfx_power_gating()
2933 		data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);  in gfx_v9_0_init_gfx_power_gating()
2934 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);  in gfx_v9_0_init_gfx_power_gating()
2943 	uint32_t data = 0;  in gfx_v9_0_enable_sck_slow_down_on_power_up()
2944 	uint32_t default_data = 0;  in gfx_v9_0_enable_sck_slow_down_on_power_up()
2946 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_sck_slow_down_on_power_up()
2949 			     enable ? 1 : 0);  in gfx_v9_0_enable_sck_slow_down_on_power_up()
2951 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_sck_slow_down_on_power_up()
2957 	uint32_t data = 0;  in gfx_v9_0_enable_sck_slow_down_on_power_down()
2958 	uint32_t default_data = 0;  in gfx_v9_0_enable_sck_slow_down_on_power_down()
2960 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_sck_slow_down_on_power_down()
2963 			     enable ? 1 : 0);  in gfx_v9_0_enable_sck_slow_down_on_power_down()
2965 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_sck_slow_down_on_power_down()
2971 	uint32_t data = 0;  in gfx_v9_0_enable_cp_power_gating()
2972 	uint32_t default_data = 0;  in gfx_v9_0_enable_cp_power_gating()
2974 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_cp_power_gating()
2977 			     enable ? 0 : 1);  in gfx_v9_0_enable_cp_power_gating()
2979 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_cp_power_gating()
2987 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_gfx_cg_power_gating()
2990 			     enable ? 1 : 0);  in gfx_v9_0_enable_gfx_cg_power_gating()
2992 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_gfx_cg_power_gating()
3000 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_gfx_pipeline_powergating()
3003 			     enable ? 1 : 0);  in gfx_v9_0_enable_gfx_pipeline_powergating()
3005 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_gfx_pipeline_powergating()
3009 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));  in gfx_v9_0_enable_gfx_pipeline_powergating()
3017 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_gfx_static_mg_power_gating()
3020 			     enable ? 1 : 0);  in gfx_v9_0_enable_gfx_static_mg_power_gating()
3022 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_gfx_static_mg_power_gating()
3030 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));  in gfx_v9_0_enable_gfx_dynamic_mg_power_gating()
3033 			     enable ? 1 : 0);  in gfx_v9_0_enable_gfx_dynamic_mg_power_gating()
3035 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);  in gfx_v9_0_enable_gfx_dynamic_mg_power_gating()
3067 	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);  in gfx_v9_0_rlc_stop()
3074 	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);  in gfx_v9_0_rlc_reset()
3076 	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);  in gfx_v9_0_rlc_reset()
3086 	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);  in gfx_v9_0_rlc_start()
3097 	rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);  in gfx_v9_0_rlc_start()
3098 	if(rlc_ucode_ver == 0x108) {  in gfx_v9_0_rlc_start()
3099 		DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",  in gfx_v9_0_rlc_start()
3102 		 * default is 0x9C4 to create a 100us interval */  in gfx_v9_0_rlc_start()
3103 		WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);  in gfx_v9_0_rlc_start()
3106 		 * 0x100 (256) */  in gfx_v9_0_rlc_start()
3107 		WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);  in gfx_v9_0_rlc_start()
3128 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,  in gfx_v9_0_rlc_load_microcode()
3130 	for (i = 0; i < fw_size; i++)  in gfx_v9_0_rlc_load_microcode()
3131 		WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));  in gfx_v9_0_rlc_load_microcode()
3132 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);  in gfx_v9_0_rlc_load_microcode()
3134 	return 0;  in gfx_v9_0_rlc_load_microcode()
3143 		return 0;  in gfx_v9_0_rlc_resume()
3149 	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);  in gfx_v9_0_rlc_resume()
3162 		if (amdgpu_lbpw == 0)  in gfx_v9_0_rlc_resume()
3168 		if (amdgpu_lbpw > 0)  in gfx_v9_0_rlc_resume()
3179 	return 0;  in gfx_v9_0_rlc_resume()
3184 	u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);  in gfx_v9_0_cp_gfx_enable()
3186 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);  in gfx_v9_0_cp_gfx_enable()
3187 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);  in gfx_v9_0_cp_gfx_enable()
3188 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);  in gfx_v9_0_cp_gfx_enable()
3189 	WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);  in gfx_v9_0_cp_gfx_enable()
3222 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);  in gfx_v9_0_cp_gfx_load_microcode()
3223 	for (i = 0; i < fw_size; i++)  in gfx_v9_0_cp_gfx_load_microcode()
3224 		WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));  in gfx_v9_0_cp_gfx_load_microcode()
3225 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3232 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);  in gfx_v9_0_cp_gfx_load_microcode()
3233 	for (i = 0; i < fw_size; i++)  in gfx_v9_0_cp_gfx_load_microcode()
3234 		WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));  in gfx_v9_0_cp_gfx_load_microcode()
3235 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3242 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);  in gfx_v9_0_cp_gfx_load_microcode()
3243 	for (i = 0; i < fw_size; i++)  in gfx_v9_0_cp_gfx_load_microcode()
3244 		WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));  in gfx_v9_0_cp_gfx_load_microcode()
3245 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3247 	return 0;  in gfx_v9_0_cp_gfx_load_microcode()
3252 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_start()
3258 	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);  in gfx_v9_0_cp_gfx_start()
3259 	WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);  in gfx_v9_0_cp_gfx_start()
3269 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_cp_gfx_start()
3273 	amdgpu_ring_write(ring, 0x80000000);  in gfx_v9_0_cp_gfx_start()
3274 	amdgpu_ring_write(ring, 0x80000000);  in gfx_v9_0_cp_gfx_start()
3284 				for (i = 0; i < ext->reg_count; i++)  in gfx_v9_0_cp_gfx_start()
3290 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_cp_gfx_start()
3293 	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));  in gfx_v9_0_cp_gfx_start()
3294 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_cp_gfx_start()
3298 	amdgpu_ring_write(ring, 0x8000);  in gfx_v9_0_cp_gfx_start()
3299 	amdgpu_ring_write(ring, 0x8000);  in gfx_v9_0_cp_gfx_start()
3303 		(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));  in gfx_v9_0_cp_gfx_start()
3305 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_cp_gfx_start()
3309 	return 0;  in gfx_v9_0_cp_gfx_start()
3320 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);  in gfx_v9_0_cp_gfx_resume()
3322 	/* set the RB to use vmid 0 */  in gfx_v9_0_cp_gfx_resume()
3323 	WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);  in gfx_v9_0_cp_gfx_resume()
3326 	ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_resume()
3328 	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);  in gfx_v9_0_cp_gfx_resume()
3333 	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);  in gfx_v9_0_cp_gfx_resume()
3336 	ring->wptr = 0;  in gfx_v9_0_cp_gfx_resume()
3337 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));  in gfx_v9_0_cp_gfx_resume()
3338 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));  in gfx_v9_0_cp_gfx_resume()
3342 	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));  in gfx_v9_0_cp_gfx_resume()
3343 …WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_…  in gfx_v9_0_cp_gfx_resume()
3346 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));  in gfx_v9_0_cp_gfx_resume()
3347 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));  in gfx_v9_0_cp_gfx_resume()
3350 	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);  in gfx_v9_0_cp_gfx_resume()
3353 	WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);  in gfx_v9_0_cp_gfx_resume()
3354 	WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));  in gfx_v9_0_cp_gfx_resume()
3356 	tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);  in gfx_v9_0_cp_gfx_resume()
3363 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);  in gfx_v9_0_cp_gfx_resume()
3365 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);  in gfx_v9_0_cp_gfx_resume()
3367 	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,  in gfx_v9_0_cp_gfx_resume()
3369 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);  in gfx_v9_0_cp_gfx_resume()
3371 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,  in gfx_v9_0_cp_gfx_resume()
3379 	return 0;  in gfx_v9_0_cp_gfx_resume()
3385 		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);  in gfx_v9_0_cp_compute_enable()
3387 		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,  in gfx_v9_0_cp_compute_enable()
3412 	tmp = 0;  in gfx_v9_0_cp_compute_load_microcode()
3413 	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);  in gfx_v9_0_cp_compute_load_microcode()
3414 	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);  in gfx_v9_0_cp_compute_load_microcode()
3415 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);  in gfx_v9_0_cp_compute_load_microcode()
3417 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,  in gfx_v9_0_cp_compute_load_microcode()
3418 		adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);  in gfx_v9_0_cp_compute_load_microcode()
3419 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,  in gfx_v9_0_cp_compute_load_microcode()
3423 	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,  in gfx_v9_0_cp_compute_load_microcode()
3425 	for (i = 0; i < mec_hdr->jt_size; i++)  in gfx_v9_0_cp_compute_load_microcode()
3426 		WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,  in gfx_v9_0_cp_compute_load_microcode()
3429 	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,  in gfx_v9_0_cp_compute_load_microcode()
3433 	return 0;  in gfx_v9_0_cp_compute_load_microcode()
3443 	tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);  in gfx_v9_0_kiq_setting()
3444 	tmp &= 0xffffff00;  in gfx_v9_0_kiq_setting()
3446 	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);  in gfx_v9_0_kiq_setting()
3447 	tmp |= 0x80;  in gfx_v9_0_kiq_setting()
3448 	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);  in gfx_v9_0_kiq_setting()
3471 	mqd->header = 0xC0310800;  in gfx_v9_0_mqd_init()
3472 	mqd->compute_pipelinestat_enable = 0x00000001;  in gfx_v9_0_mqd_init()
3473 	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;  in gfx_v9_0_mqd_init()
3474 	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;  in gfx_v9_0_mqd_init()
3475 	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;  in gfx_v9_0_mqd_init()
3476 	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;  in gfx_v9_0_mqd_init()
3477 	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;  in gfx_v9_0_mqd_init()
3478 	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;  in gfx_v9_0_mqd_init()
3479 	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;  in gfx_v9_0_mqd_init()
3480 	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;  in gfx_v9_0_mqd_init()
3481 	mqd->compute_misc_reserved = 0x00000003;  in gfx_v9_0_mqd_init()
3495 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);  in gfx_v9_0_mqd_init()
3502 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);  in gfx_v9_0_mqd_init()
3510 				    DOORBELL_SOURCE, 0);  in gfx_v9_0_mqd_init()
3512 				    DOORBELL_HIT, 0);  in gfx_v9_0_mqd_init()
3515 					 DOORBELL_EN, 0);  in gfx_v9_0_mqd_init()
3521 	ring->wptr = 0;  in gfx_v9_0_mqd_init()
3522 	mqd->cp_hqd_dequeue_request = 0;  in gfx_v9_0_mqd_init()
3523 	mqd->cp_hqd_pq_rptr = 0;  in gfx_v9_0_mqd_init()
3524 	mqd->cp_hqd_pq_wptr_lo = 0;  in gfx_v9_0_mqd_init()
3525 	mqd->cp_hqd_pq_wptr_hi = 0;  in gfx_v9_0_mqd_init()
3528 	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3531 	/* set MQD vmid to 0 */  in gfx_v9_0_mqd_init()
3532 	tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);  in gfx_v9_0_mqd_init()
3533 	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);  in gfx_v9_0_mqd_init()
3542 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);  in gfx_v9_0_mqd_init()
3550 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);  in gfx_v9_0_mqd_init()
3551 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);  in gfx_v9_0_mqd_init()
3558 	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3560 		upper_32_bits(wb_gpu_addr) & 0xffff;  in gfx_v9_0_mqd_init()
3564 	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3565 	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;  in gfx_v9_0_mqd_init()
3567 	tmp = 0;  in gfx_v9_0_mqd_init()
3570 		tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);  in gfx_v9_0_mqd_init()
3577 					 DOORBELL_SOURCE, 0);  in gfx_v9_0_mqd_init()
3579 					 DOORBELL_HIT, 0);  in gfx_v9_0_mqd_init()
3585 	ring->wptr = 0;  in gfx_v9_0_mqd_init()
3586 	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);  in gfx_v9_0_mqd_init()
3589 	mqd->cp_hqd_vmid = 0;  in gfx_v9_0_mqd_init()
3591 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);  in gfx_v9_0_mqd_init()
3592 	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);  in gfx_v9_0_mqd_init()
3596 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);  in gfx_v9_0_mqd_init()
3602 	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);  in gfx_v9_0_mqd_init()
3610 	return 0;  in gfx_v9_0_mqd_init()
3620 	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);  in gfx_v9_0_kiq_init_register()
3622 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,  in gfx_v9_0_kiq_init_register()
3624 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,  in gfx_v9_0_kiq_init_register()
3628 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,  in gfx_v9_0_kiq_init_register()
3632 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,  in gfx_v9_0_kiq_init_register()
3636 	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {  in gfx_v9_0_kiq_init_register()
3637 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);  in gfx_v9_0_kiq_init_register()
3638 		for (j = 0; j < adev->usec_timeout; j++) {  in gfx_v9_0_kiq_init_register()
3639 			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))  in gfx_v9_0_kiq_init_register()
3643 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,  in gfx_v9_0_kiq_init_register()
3645 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,  in gfx_v9_0_kiq_init_register()
3647 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,  in gfx_v9_0_kiq_init_register()
3649 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,  in gfx_v9_0_kiq_init_register()
3654 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,  in gfx_v9_0_kiq_init_register()
3656 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,  in gfx_v9_0_kiq_init_register()
3659 	/* set MQD vmid to 0 */  in gfx_v9_0_kiq_init_register()
3660 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,  in gfx_v9_0_kiq_init_register()
3664 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,  in gfx_v9_0_kiq_init_register()
3666 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,  in gfx_v9_0_kiq_init_register()
3670 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,  in gfx_v9_0_kiq_init_register()
3674 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,  in gfx_v9_0_kiq_init_register()
3676 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,  in gfx_v9_0_kiq_init_register()
3680 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,  in gfx_v9_0_kiq_init_register()
3682 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,  in gfx_v9_0_kiq_init_register()
3687 		WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,  in gfx_v9_0_kiq_init_register()
3695 			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,  in gfx_v9_0_kiq_init_register()
3698 			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,  in gfx_v9_0_kiq_init_register()
3702 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,  in gfx_v9_0_kiq_init_register()
3706 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,  in gfx_v9_0_kiq_init_register()
3708 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,  in gfx_v9_0_kiq_init_register()
3712 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);  in gfx_v9_0_kiq_init_register()
3714 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,  in gfx_v9_0_kiq_init_register()
3718 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,  in gfx_v9_0_kiq_init_register()
3722 		WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);  in gfx_v9_0_kiq_init_register()
3724 	return 0;  in gfx_v9_0_kiq_init_register()
3733 	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {  in gfx_v9_0_kiq_fini_register()
3735 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);  in gfx_v9_0_kiq_fini_register()
3737 		for (j = 0; j < adev->usec_timeout; j++) {  in gfx_v9_0_kiq_fini_register()
3738 			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))  in gfx_v9_0_kiq_fini_register()
3747 			WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);  in gfx_v9_0_kiq_fini_register()
3750 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,  in gfx_v9_0_kiq_fini_register()
3751 		      0);  in gfx_v9_0_kiq_fini_register()
3754 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);  in gfx_v9_0_kiq_fini_register()
3755 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);  in gfx_v9_0_kiq_fini_register()
3756 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);  in gfx_v9_0_kiq_fini_register()
3757 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);  in gfx_v9_0_kiq_fini_register()
3758 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);  in gfx_v9_0_kiq_fini_register()
3759 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);  in gfx_v9_0_kiq_fini_register()
3760 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);  in gfx_v9_0_kiq_fini_register()
3761 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);  in gfx_v9_0_kiq_fini_register()
3763 	return 0;  in gfx_v9_0_kiq_fini_register()
3778 	 * check mqd->cp_hqd_pq_control since this value should not be 0  in gfx_v9_0_kiq_init_queue()
3787 		ring->wptr = 0;  in gfx_v9_0_kiq_init_queue()
3791 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);  in gfx_v9_0_kiq_init_queue()
3793 		soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_kiq_init_queue()
3796 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kiq_init_queue()
3797 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;  in gfx_v9_0_kiq_init_queue()
3798 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;  in gfx_v9_0_kiq_init_queue()
3800 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);  in gfx_v9_0_kiq_init_queue()
3803 		soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_kiq_init_queue()
3810 	return 0;  in gfx_v9_0_kiq_init_queue()
3817 	int mqd_idx = ring - &adev->gfx.compute_ring[0];  in gfx_v9_0_kcq_init_queue()
3827 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kcq_init_queue()
3828 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;  in gfx_v9_0_kcq_init_queue()
3829 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;  in gfx_v9_0_kcq_init_queue()
3831 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);  in gfx_v9_0_kcq_init_queue()
3833 		soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_kcq_init_queue()
3844 		ring->wptr = 0;  in gfx_v9_0_kcq_init_queue()
3845 		atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);  in gfx_v9_0_kcq_init_queue()
3851 	return 0;  in gfx_v9_0_kcq_init_queue()
3862 	if (unlikely(r != 0))  in gfx_v9_0_kiq_resume()
3866 	if (unlikely(r != 0))  in gfx_v9_0_kiq_resume()
3874 	return 0;  in gfx_v9_0_kiq_resume()
3880 	int r = 0, i;  in gfx_v9_0_kcq_resume()
3884 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_kcq_resume()
3888 		if (unlikely(r != 0))  in gfx_v9_0_kcq_resume()
3942 		ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_resume()
3948 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_cp_resume()
3955 	return 0;  in gfx_v9_0_cp_resume()
3966 	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);  in gfx_v9_0_init_tcp_config()
3973 	WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);  in gfx_v9_0_init_tcp_config()
4013 	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);  in gfx_v9_0_hw_fini()
4014 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_hw_fini()
4015 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_hw_fini()
4029 		WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);  in gfx_v9_0_hw_fini()
4030 		return 0;  in gfx_v9_0_hw_fini()
4040 				adev->gfx.kiq.ring.queue, 0);  in gfx_v9_0_hw_fini()
4042 		soc15_grbm_select(adev, 0, 0, 0, 0);  in gfx_v9_0_hw_fini()
4051 		return 0;  in gfx_v9_0_hw_fini()
4055 	return 0;  in gfx_v9_0_hw_fini()
4072 	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),  in gfx_v9_0_is_idle()
4084 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_wait_for_idle()
4086 			return 0;  in gfx_v9_0_wait_for_idle()
4094 	u32 grbm_soft_reset = 0;  in gfx_v9_0_soft_reset()
4099 	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);  in gfx_v9_0_soft_reset()
4118 	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);  in gfx_v9_0_soft_reset()
4136 			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);  in gfx_v9_0_soft_reset()
4138 			dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in gfx_v9_0_soft_reset()
4139 			WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);  in gfx_v9_0_soft_reset()
4140 			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);  in gfx_v9_0_soft_reset()
4145 			WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);  in gfx_v9_0_soft_reset()
4146 			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);  in gfx_v9_0_soft_reset()
4152 	return 0;  in gfx_v9_0_soft_reset()
4157 	signed long r, cnt = 0;  in gfx_v9_0_kiq_read_clock()
4159 	uint32_t seq, reg_val_offs = 0;  in gfx_v9_0_kiq_read_clock()
4160 	uint64_t value = 0;  in gfx_v9_0_kiq_read_clock()
4177 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_kiq_read_clock()
4178 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_kiq_read_clock()
4226 	return ~0;  in gfx_v9_0_kiq_read_clock()
4238 		WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);  in gfx_v9_0_get_gpu_clock_counter()
4239 		clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |  in gfx_v9_0_get_gpu_clock_counter()
4240 			((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);  in gfx_v9_0_get_gpu_clock_counter()
4256 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4257 				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,  in gfx_v9_0_ring_emit_gds_switch()
4261 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4262 				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,  in gfx_v9_0_ring_emit_gds_switch()
4266 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4267 				   SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,  in gfx_v9_0_ring_emit_gds_switch()
4271 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4272 				   SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,  in gfx_v9_0_ring_emit_gds_switch()
4278 	0xb07c0000, 0xbe8000ff,
4279 	0x000000f8, 0xbf110800,
4280 	0x7e000280, 0x7e020280,
4281 	0x7e040280, 0x7e060280,
4282 	0x7e080280, 0x7e0a0280,
4283 	0x7e0c0280, 0x7e0e0280,
4284 	0x80808800, 0xbe803200,
4285 	0xbf84fff5, 0xbf9c0000,
4286 	0xd28c0001, 0x0001007f,
4287 	0xd28d0001, 0x0002027e,
4288 	0x10020288, 0xb8810904,
4289 	0xb7814000, 0xd1196a01,
4290 	0x00000301, 0xbe800087,
4291 	0xbefc00c1, 0xd89c4000,
4292 	0x00020201, 0xd89cc080,
4293 	0x00040401, 0x320202ff,
4294 	0x00000800, 0x80808100,
4295 	0xbf84fff8, 0x7e020280,
4296 	0xbf810000, 0x00000000,
4301 	0xb07c0000, 0xbe8000ff,
4302 	0x0000005f, 0xbee50080,
4303 	0xbe812c65, 0xbe822c65,
4304 	0xbe832c65, 0xbe842c65,
4305 	0xbe852c65, 0xb77c0005,
4306 	0x80808500, 0xbf84fff8,
4307 	0xbe800080, 0xbf810000,
4311 	0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4312 	0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4313 	0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4314 	0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4315 	0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4316 	0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4317 	0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4318 	0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4319 	0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4320 	0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4321 	0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4322 	0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4323 	0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4324 	0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4325 	0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4326 	0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4327 	0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4328 	0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4329 	0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4330 	0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4331 	0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4332 	0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4333 	0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4334 	0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4335 	0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4336 	0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4337 	0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4338 	0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4339 	0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4340 	0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4341 	0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4342 	0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4343 	0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4344 	0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4345 	0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4346 	0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4347 	0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4348 	0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4349 	0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4350 	0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4351 	0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4352 	0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4353 	0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4354 	0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4355 	0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4356 	0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4357 	0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4358 	0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4359 	0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4360 	0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4361 	0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4362 	0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4363 	0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4364 	0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4365 	0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4366 	0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4367 	0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4368 	0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4369 	0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4370 	0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4371 	0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4372 	0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4373 	0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4374 	0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4375 	0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4376 	0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4377 	0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4378 	0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4379 	0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4380 	0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4381 	0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4382 	0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4383 	0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4384 	0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4385 	0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4386 	0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4387 	0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4388 	0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4389 	0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4390 	0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4391 	0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4392 	0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4393 	0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4394 	0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4395 	0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4396 	0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4397 	0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4398 	0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4399 	0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4400 	0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4401 	0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4402 	0xbf84fff8, 0xbf810000,
4409    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4410    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4411    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4412    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4413    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4414    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4415    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4416    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4417    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4418    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4419    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4420    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4421    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4422    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4426    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4427    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4429    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4430    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4431    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4432    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4433    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4434    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4435    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4436    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4437    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4438    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4439    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4443    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4444    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4446    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4447    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4448    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4449    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4450    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4451    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4452    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4453    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4454    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4455    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4456    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4460    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4461    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4462    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4463    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4464    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4465    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4466    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4467    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4468    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4469    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4470    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4471    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4472    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4473    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4477    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4478    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4479    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4480    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4481    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4482    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4483    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4484    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4485    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4486    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4487    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4488    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4489    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4490    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4491    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4492    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4493    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4494    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4495    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4496    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4497    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4498    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4499    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4500    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4501    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4502    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4503    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4504    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4505    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4506    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4507    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4508    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4509    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4514 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gds_workarounds()
4519 		return 0;  in gfx_v9_0_do_edc_gds_workarounds()
4528 	WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);  in gfx_v9_0_do_edc_gds_workarounds()
4529 	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);  in gfx_v9_0_do_edc_gds_workarounds()
4535 				PACKET3_DMA_DATA_ENGINE(0)));  in gfx_v9_0_do_edc_gds_workarounds()
4536 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4537 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4538 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4539 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4545 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_do_edc_gds_workarounds()
4554 	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);  in gfx_v9_0_do_edc_gds_workarounds()
4561 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gpr_workarounds()
4579 		return 0;  in gfx_v9_0_do_edc_gpr_workarounds()
4583 		return 0;  in gfx_v9_0_do_edc_gpr_workarounds()
4608 	memset(&ib, 0, sizeof(ib));  in gfx_v9_0_do_edc_gpr_workarounds()
4617 	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)  in gfx_v9_0_do_edc_gpr_workarounds()
4620 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)  in gfx_v9_0_do_edc_gpr_workarounds()
4623 	/* init the ib length to 0 */  in gfx_v9_0_do_edc_gpr_workarounds()
4624 	ib.length_dw = 0;  in gfx_v9_0_do_edc_gpr_workarounds()
4628 	for (i = 0; i < gpr_reg_size; i++) {  in gfx_v9_0_do_edc_gpr_workarounds()
4637 	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)  in gfx_v9_0_do_edc_gpr_workarounds()
4648 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);  in gfx_v9_0_do_edc_gpr_workarounds()
4651 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);  in gfx_v9_0_do_edc_gpr_workarounds()
4656 	for (i = 0; i < gpr_reg_size; i++) {  in gfx_v9_0_do_edc_gpr_workarounds()
4665 	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)  in gfx_v9_0_do_edc_gpr_workarounds()
4676 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);  in gfx_v9_0_do_edc_gpr_workarounds()
4679 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);  in gfx_v9_0_do_edc_gpr_workarounds()
4684 	for (i = 0; i < gpr_reg_size; i++) {  in gfx_v9_0_do_edc_gpr_workarounds()
4693 	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)  in gfx_v9_0_do_edc_gpr_workarounds()
4704 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);  in gfx_v9_0_do_edc_gpr_workarounds()
4707 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);  in gfx_v9_0_do_edc_gpr_workarounds()
4737 		adev->gfx.num_gfx_rings = 0;  in gfx_v9_0_early_init()
4748 	return 0;  in gfx_v9_0_early_init()
4789 	return 0;  in gfx_v9_0_ecc_late_init()
4797 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_late_init()
4801 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_late_init()
4809 	return 0;  in gfx_v9_0_late_init()
4817 	rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);  in gfx_v9_0_is_rlc_enabled()
4831 	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);  in gfx_v9_0_set_safe_mode()
4834 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_set_safe_mode()
4835 		if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))  in gfx_v9_0_set_safe_mode()
4846 	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);  in gfx_v9_0_unset_safe_mode()
4896 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);  in gfx_v9_0_update_medium_grain_clock_gating()
4909 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4915 				def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);  in gfx_v9_0_update_medium_grain_clock_gating()
4918 					WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4922 				def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);  in gfx_v9_0_update_medium_grain_clock_gating()
4925 					WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4930 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);  in gfx_v9_0_update_medium_grain_clock_gating()
4941 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4944 		data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);  in gfx_v9_0_update_medium_grain_clock_gating()
4947 			WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4951 		data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);  in gfx_v9_0_update_medium_grain_clock_gating()
4954 			WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);  in gfx_v9_0_update_medium_grain_clock_gating()
4974 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);  in gfx_v9_0_update_3d_clock_gating()
4979 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);  in gfx_v9_0_update_3d_clock_gating()
4981 		/* enable 3Dcgcg FSM(0x0000363f) */  in gfx_v9_0_update_3d_clock_gating()
4982 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);  in gfx_v9_0_update_3d_clock_gating()
4985 			data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |  in gfx_v9_0_update_3d_clock_gating()
4988 			data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;  in gfx_v9_0_update_3d_clock_gating()
4991 			data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |  in gfx_v9_0_update_3d_clock_gating()
4994 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);  in gfx_v9_0_update_3d_clock_gating()
4996 		/* set IDLE_POLL_COUNT(0x00900100) */  in gfx_v9_0_update_3d_clock_gating()
4997 		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);  in gfx_v9_0_update_3d_clock_gating()
4998 		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |  in gfx_v9_0_update_3d_clock_gating()
4999 			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);  in gfx_v9_0_update_3d_clock_gating()
5001 			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);  in gfx_v9_0_update_3d_clock_gating()
5004 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);  in gfx_v9_0_update_3d_clock_gating()
5010 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);  in gfx_v9_0_update_3d_clock_gating()
5024 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);  in gfx_v9_0_update_coarse_grain_clock_gating()
5033 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);  in gfx_v9_0_update_coarse_grain_clock_gating()
5035 		/* enable cgcg FSM(0x0000363F) */  in gfx_v9_0_update_coarse_grain_clock_gating()
5036 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);  in gfx_v9_0_update_coarse_grain_clock_gating()
5039 			data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |  in gfx_v9_0_update_coarse_grain_clock_gating()
5042 			data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |  in gfx_v9_0_update_coarse_grain_clock_gating()
5045 			data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |  in gfx_v9_0_update_coarse_grain_clock_gating()
5048 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);  in gfx_v9_0_update_coarse_grain_clock_gating()
5050 		/* set IDLE_POLL_COUNT(0x00900100) */  in gfx_v9_0_update_coarse_grain_clock_gating()
5051 		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);  in gfx_v9_0_update_coarse_grain_clock_gating()
5052 		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |  in gfx_v9_0_update_coarse_grain_clock_gating()
5053 			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);  in gfx_v9_0_update_coarse_grain_clock_gating()
5055 			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);  in gfx_v9_0_update_coarse_grain_clock_gating()
5057 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);  in gfx_v9_0_update_coarse_grain_clock_gating()
5062 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);  in gfx_v9_0_update_coarse_grain_clock_gating()
5090 	return 0;  in gfx_v9_0_update_gfx_clock_gating()
5097 	reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);  in gfx_v9_0_update_spm_vmid()
5107 		WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);  in gfx_v9_0_update_spm_vmid()
5109 		WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);  in gfx_v9_0_update_spm_vmid()
5122 	for (i = 0; i < arr_size; i++) {  in gfx_v9_0_check_rlcg_range()
5199 	return 0;  in gfx_v9_0_set_powergating_state()
5208 		return 0;  in gfx_v9_0_set_clockgating_state()
5224 	return 0;  in gfx_v9_0_set_clockgating_state()
5233 		*flags = 0;  in gfx_v9_0_get_clockgating_state()
5236 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));  in gfx_v9_0_get_clockgating_state()
5241 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));  in gfx_v9_0_get_clockgating_state()
5250 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));  in gfx_v9_0_get_clockgating_state()
5255 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));  in gfx_v9_0_get_clockgating_state()
5261 		data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));  in gfx_v9_0_get_clockgating_state()
5285 		wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);  in gfx_v9_0_ring_get_wptr_gfx()
5286 		wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;  in gfx_v9_0_ring_get_wptr_gfx()
5301 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));  in gfx_v9_0_ring_set_wptr_gfx()
5302 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));  in gfx_v9_0_ring_set_wptr_gfx()
5323 		reg_mem_engine = 0;  in gfx_v9_0_ring_emit_hdp_flush()
5329 	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,  in gfx_v9_0_ring_emit_hdp_flush()
5332 			      ref_and_mask, ref_and_mask, 0x20);  in gfx_v9_0_ring_emit_hdp_flush()
5341 	u32 header, control = 0;  in gfx_v9_0_ring_emit_ib_gfx()
5358 	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */  in gfx_v9_0_ring_emit_ib_gfx()
5361 		(2 << 0) |  in gfx_v9_0_ring_emit_ib_gfx()
5384 	 * GDS to 0 for this ring (me/pipe).  in gfx_v9_0_ring_emit_ib_compute()
5393 	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */  in gfx_v9_0_ring_emit_ib_compute()
5396 				(2 << 0) |  in gfx_v9_0_ring_emit_ib_compute()
5420 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));  in gfx_v9_0_ring_emit_fence()
5427 		BUG_ON(addr & 0x7);  in gfx_v9_0_ring_emit_fence()
5429 		BUG_ON(addr & 0x3);  in gfx_v9_0_ring_emit_fence()
5434 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_fence()
5443 	gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,  in gfx_v9_0_ring_emit_pipeline_sync()
5445 			      seq, 0xffffffff, 4);  in gfx_v9_0_ring_emit_pipeline_sync()
5456 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));  in gfx_v9_0_ring_emit_vm_flush()
5457 		amdgpu_ring_write(ring, 0x0);  in gfx_v9_0_ring_emit_vm_flush()
5501 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |  in gfx_v9_0_ring_emit_fence_kiq()
5510 		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |  in gfx_v9_0_ring_emit_fence_kiq()
5511 					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));  in gfx_v9_0_ring_emit_fence_kiq()
5512 		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));  in gfx_v9_0_ring_emit_fence_kiq()
5513 		amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_fence_kiq()
5514 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */  in gfx_v9_0_ring_emit_fence_kiq()
5520 	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));  in gfx_v9_ring_emit_sb()
5521 	amdgpu_ring_write(ring, 0);  in gfx_v9_ring_emit_sb()
5526 	struct v9_ce_ib_state ce_payload = {0};  in gfx_v9_0_ring_emit_ce_meta()
5537 				 WRITE_DATA_CACHE_POLICY(0));  in gfx_v9_0_ring_emit_ce_meta()
5545 	struct v9_de_ib_state de_payload = {0};  in gfx_v9_0_ring_emit_de_meta()
5559 				 WRITE_DATA_CACHE_POLICY(0));  in gfx_v9_0_ring_emit_de_meta()
5568 	uint32_t v = secure ? FRAME_TMZ : 0;  in gfx_v9_0_ring_emit_frame_cntl()
5570 	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));  in gfx_v9_0_ring_emit_frame_cntl()
5571 	amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));  in gfx_v9_0_ring_emit_frame_cntl()
5576 	uint32_t dw2 = 0;  in gfx_v9_ring_emit_cntxcntl()
5581 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */  in gfx_v9_ring_emit_cntxcntl()
5584 		dw2 |= 0x8001;  in gfx_v9_ring_emit_cntxcntl()
5586 		dw2 |= 0x01000000;  in gfx_v9_ring_emit_cntxcntl()
5588 		dw2 |= 0x10002;  in gfx_v9_ring_emit_cntxcntl()
5592 			dw2 |= 0x10000000;  in gfx_v9_ring_emit_cntxcntl()
5598 			dw2 |= 0x10000000;  in gfx_v9_ring_emit_cntxcntl()
5603 	amdgpu_ring_write(ring, 0);  in gfx_v9_ring_emit_cntxcntl()
5612 	amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */  in gfx_v9_0_ring_emit_init_cond_exec()
5614 	amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */  in gfx_v9_0_ring_emit_init_cond_exec()
5622 	BUG_ON(ring->ring[offset] != 0x55aa55aa);  in gfx_v9_0_ring_emit_patch_cond_exec()
5637 	amdgpu_ring_write(ring, 0 |	/* src: register*/  in gfx_v9_0_ring_emit_rreg()
5641 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_rreg()
5651 	uint32_t cmd = 0;  in gfx_v9_0_ring_emit_wreg()
5667 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_wreg()
5674 	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);  in gfx_v9_0_ring_emit_reg_wait()
5687 		gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5688 				      ref, mask, 0x20);  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5697 	uint32_t value = 0;  in gfx_v9_0_ring_soft_recovery()
5699 	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);  in gfx_v9_0_ring_soft_recovery()
5700 	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);  in gfx_v9_0_ring_soft_recovery()
5703 	WREG32_SOC15(GC, 0, mmSQ_CMD, value);  in gfx_v9_0_ring_soft_recovery()
5712 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,  in gfx_v9_0_set_gfx_eop_interrupt_state()
5714 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);  in gfx_v9_0_set_gfx_eop_interrupt_state()
5735 		case 0:  in gfx_v9_0_set_compute_eop_interrupt_state()
5736 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);  in gfx_v9_0_set_compute_eop_interrupt_state()
5739 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);  in gfx_v9_0_set_compute_eop_interrupt_state()
5742 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);  in gfx_v9_0_set_compute_eop_interrupt_state()
5745 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);  in gfx_v9_0_set_compute_eop_interrupt_state()
5760 					     TIME_STAMP_INT_ENABLE, 0);  in gfx_v9_0_set_compute_eop_interrupt_state()
5782 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,  in gfx_v9_0_set_priv_reg_fault_state()
5784 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);  in gfx_v9_0_set_priv_reg_fault_state()
5790 	return 0;  in gfx_v9_0_set_priv_reg_fault_state()
5801 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,  in gfx_v9_0_set_priv_inst_fault_state()
5803 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);  in gfx_v9_0_set_priv_inst_fault_state()
5809 	return 0;  in gfx_v9_0_set_priv_inst_fault_state()
5813 	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5817 	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5818 			CP_ECC_ERROR_INT_ENABLE, 0)
5827 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,  in gfx_v9_0_set_cp_ecc_error_state()
5828 				CP_ECC_ERROR_INT_ENABLE, 0);  in gfx_v9_0_set_cp_ecc_error_state()
5829 		DISABLE_ECC_ON_ME_PIPE(1, 0);  in gfx_v9_0_set_cp_ecc_error_state()
5836 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,  in gfx_v9_0_set_cp_ecc_error_state()
5838 		ENABLE_ECC_ON_ME_PIPE(1, 0);  in gfx_v9_0_set_cp_ecc_error_state()
5847 	return 0;  in gfx_v9_0_set_cp_ecc_error_state()
5861 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);  in gfx_v9_0_set_eop_interrupt_state()
5873 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);  in gfx_v9_0_set_eop_interrupt_state()
5887 	return 0;  in gfx_v9_0_set_eop_interrupt_state()
5899 	me_id = (entry->ring_id & 0x0c) >> 2;  in gfx_v9_0_eop_irq()
5900 	pipe_id = (entry->ring_id & 0x03) >> 0;  in gfx_v9_0_eop_irq()
5901 	queue_id = (entry->ring_id & 0x70) >> 4;  in gfx_v9_0_eop_irq()
5904 	case 0:  in gfx_v9_0_eop_irq()
5905 		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);  in gfx_v9_0_eop_irq()
5909 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_eop_irq()
5919 	return 0;  in gfx_v9_0_eop_irq()
5929 	me_id = (entry->ring_id & 0x0c) >> 2;  in gfx_v9_0_fault()
5930 	pipe_id = (entry->ring_id & 0x03) >> 0;  in gfx_v9_0_fault()
5931 	queue_id = (entry->ring_id & 0x70) >> 4;  in gfx_v9_0_fault()
5934 	case 0:  in gfx_v9_0_fault()
5935 		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);  in gfx_v9_0_fault()
5939 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_fault()
5955 	return 0;  in gfx_v9_0_priv_reg_irq()
5964 	return 0;  in gfx_v9_0_priv_inst_irq()
5969 	{ "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5973 	{ "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5977 	{ "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5979 	  0, 0
5981 	{ "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5983 	  0, 0
5985 	{ "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5989 	{ "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5991 	  0, 0
5993 	{ "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5997 	{ "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6001 	{ "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6003 	  0, 0
6005 	{ "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6007 	  0, 0
6009 	{ "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6011 	  0, 0
6013 	{ "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6017 	{ "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6019 	  0, 0
6021 	{ "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6026 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6031 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6033 	  0, 0
6036 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6041 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6046 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6051 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6055 	{ "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6057 	  0, 0
6059 	{ "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6063 	{ "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6065 	  0, 0
6067 	{ "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6069 	  0, 0
6071 	{ "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6073 	  0, 0
6075 	{ "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6077 	  0, 0
6079 	{ "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6081 	  0, 0
6083 	{ "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6085 	  0, 0
6087 	{ "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6091 	{ "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6095 	{ "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6099 	{ "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6103 	{ "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6107 	{ "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6109 	  0, 0
6111 	{ "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6113 	  0, 0
6115 	{ "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6117 	  0, 0
6119 	{ "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6121 	  0, 0
6123 	{ "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6125 	  0, 0
6127 	{ "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6129 	  0, 0
6131 	{ "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6133 	  0, 0
6135 	{ "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6137 	  0, 0
6139 	{ "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6141 	  0, 0
6143 	{ "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6145 	  0, 0
6147 	{ "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6149 	  0, 0
6151 	{ "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6153 	  0, 0
6155 	{ "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6157 	  0, 0
6159 	{ "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6161 	  0, 0
6163 	{ "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6167 	{ "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6171 	{ "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6173 	  0, 0
6175 	{ "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6177 	  0, 0
6179 	{ "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6181 	  0, 0
6183 	{ "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6187 	{ "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6191 	{ "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6195 	{ "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6199 	{ "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6201 	  0, 0
6203 	{ "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6207 	{ "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6211 	{ "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6215 	{ "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6219 	{ "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6223 	{ "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6227 	{ "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6231 	{ "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6235 	{ "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6239 	{ "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6243 	{ "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6247 	{ "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6251 	{ "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6255 	{ "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6259 	{ "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6263 	{ "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6267 	{ "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6271 	{ "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6273 	  0, 0
6275 	{ "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6277 	  0, 0
6279 	{ "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6281 	  0, 0
6283 	{ "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6285 	  0, 0
6287 	{ "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6289 	  0, 0
6291 	{ "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6295 	{ "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6299 	{ "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6303 	{ "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6307 	{ "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6311 	{ "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6313 	  0, 0
6315 	{ "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6317 	  0, 0
6319 	{ "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6321 	  0, 0
6323 	{ "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6325 	  0, 0
6327 	{ "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6329 	  0, 0
6331 	{ "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6335 	{ "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6339 	{ "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6343 	{ "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6347 	{ "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6351 	{ "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6353 	  0, 0
6355 	{ "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6357 	  0, 0
6359 	{ "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6361 	  0, 0
6363 	{ "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6365 	  0, 0
6367 	{ "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6369 	  0, 0
6371 	{ "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6375 	{ "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6379 	{ "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6383 	{ "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6385 	  0, 0
6387 	{ "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6389 	  0, 0
6391 	{ "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6393 	  0, 0
6395 	{ "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6397 	  0, 0
6399 	{ "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6401 	  0, 0
6403 	{ "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6405 	  0, 0
6414 	struct ta_ras_trigger_error_input block_info = { 0 };  in gfx_v9_0_ras_error_inject()
6427 		DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",  in gfx_v9_0_ras_error_inject()
6435 		DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",  in gfx_v9_0_ras_error_inject()
6532 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6533 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);  in gfx_v9_0_query_utc_edc_status()
6534 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6535 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);  in gfx_v9_0_query_utc_edc_status()
6536 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6537 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);  in gfx_v9_0_query_utc_edc_status()
6538 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6539 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);  in gfx_v9_0_query_utc_edc_status()
6541 	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {  in gfx_v9_0_query_utc_edc_status()
6542 		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);  in gfx_v9_0_query_utc_edc_status()
6543 		data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);  in gfx_v9_0_query_utc_edc_status()
6560 	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {  in gfx_v9_0_query_utc_edc_status()
6561 		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);  in gfx_v9_0_query_utc_edc_status()
6562 		data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);  in gfx_v9_0_query_utc_edc_status()
6581 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {  in gfx_v9_0_query_utc_edc_status()
6582 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);  in gfx_v9_0_query_utc_edc_status()
6583 		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);  in gfx_v9_0_query_utc_edc_status()
6585 		sec_count = (data & 0x00006000L) >> 0xd;  in gfx_v9_0_query_utc_edc_status()
6594 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {  in gfx_v9_0_query_utc_edc_status()
6595 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);  in gfx_v9_0_query_utc_edc_status()
6596 		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);  in gfx_v9_0_query_utc_edc_status()
6598 		sec_count = (data & 0x00006000L) >> 0xd;  in gfx_v9_0_query_utc_edc_status()
6606 		ded_count = (data & 0x00018000L) >> 0xf;  in gfx_v9_0_query_utc_edc_status()
6615 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6616 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6617 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6618 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);  in gfx_v9_0_query_utc_edc_status()
6620 	return 0;  in gfx_v9_0_query_utc_edc_status()
6631 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {  in gfx_v9_0_ras_error_count()
6662 	return 0;  in gfx_v9_0_ras_error_count()
6674 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {  in gfx_v9_0_reset_ras_error_count()
6675 		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {  in gfx_v9_0_reset_ras_error_count()
6676 			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {  in gfx_v9_0_reset_ras_error_count()
6677 				gfx_v9_0_select_se_sh(adev, j, 0x0, k);  in gfx_v9_0_reset_ras_error_count()
6682 	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);  in gfx_v9_0_reset_ras_error_count()
6685 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6686 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);  in gfx_v9_0_reset_ras_error_count()
6687 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6688 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);  in gfx_v9_0_reset_ras_error_count()
6689 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6690 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);  in gfx_v9_0_reset_ras_error_count()
6691 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6692 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);  in gfx_v9_0_reset_ras_error_count()
6694 	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {  in gfx_v9_0_reset_ras_error_count()
6695 		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);  in gfx_v9_0_reset_ras_error_count()
6696 		RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);  in gfx_v9_0_reset_ras_error_count()
6699 	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {  in gfx_v9_0_reset_ras_error_count()
6700 		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);  in gfx_v9_0_reset_ras_error_count()
6701 		RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);  in gfx_v9_0_reset_ras_error_count()
6704 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {  in gfx_v9_0_reset_ras_error_count()
6705 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);  in gfx_v9_0_reset_ras_error_count()
6706 		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);  in gfx_v9_0_reset_ras_error_count()
6709 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {  in gfx_v9_0_reset_ras_error_count()
6710 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);  in gfx_v9_0_reset_ras_error_count()
6711 		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);  in gfx_v9_0_reset_ras_error_count()
6714 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6715 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6716 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6717 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);  in gfx_v9_0_reset_ras_error_count()
6724 	uint32_t sec_count = 0, ded_count = 0;  in gfx_v9_0_query_ras_error_count()
6731 	err_data->ue_count = 0;  in gfx_v9_0_query_ras_error_count()
6732 	err_data->ce_count = 0;  in gfx_v9_0_query_ras_error_count()
6736 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {  in gfx_v9_0_query_ras_error_count()
6737 		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {  in gfx_v9_0_query_ras_error_count()
6738 			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {  in gfx_v9_0_query_ras_error_count()
6739 				gfx_v9_0_select_se_sh(adev, j, 0, k);  in gfx_v9_0_query_ras_error_count()
6754 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_query_ras_error_count()
6759 	return 0;  in gfx_v9_0_query_ras_error_count()
6774 	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */  in gfx_v9_0_emit_mem_sync()
6775 	amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */  in gfx_v9_0_emit_mem_sync()
6776 	amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */  in gfx_v9_0_emit_mem_sync()
6777 	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */  in gfx_v9_0_emit_mem_sync()
6778 	amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */  in gfx_v9_0_emit_mem_sync()
6788 	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */  in gfx_v9_0_emit_wave_limit_cs()
6789 	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;  in gfx_v9_0_emit_wave_limit_cs()
6792 	case 0:  in gfx_v9_0_emit_wave_limit_cs()
6793 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);  in gfx_v9_0_emit_wave_limit_cs()
6796 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);  in gfx_v9_0_emit_wave_limit_cs()
6799 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);  in gfx_v9_0_emit_wave_limit_cs()
6802 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);  in gfx_v9_0_emit_wave_limit_cs()
6823 	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;  in gfx_v9_0_emit_wave_limit()
6825 			      SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),  in gfx_v9_0_emit_wave_limit()
6831 	 * amdgpu controls only 1st ME(0-3 CS pipes).  in gfx_v9_0_emit_wave_limit()
6833 	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {  in gfx_v9_0_emit_wave_limit()
6860 	.align_mask = 0xff,
6861 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6913 	.align_mask = 0xff,
6914 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6952 	.align_mask = 0xff,
6953 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6985 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_set_ring_funcs()
6988 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_set_ring_funcs()
7052 		adev->gds.gds_size = 0x10000;  in gfx_v9_0_set_gds_init()
7056 		adev->gds.gds_size = 0x1000;  in gfx_v9_0_set_gds_init()
7062 		adev->gds.gds_size = 0;  in gfx_v9_0_set_gds_init()
7065 		adev->gds.gds_size = 0x10000;  in gfx_v9_0_set_gds_init()
7072 		adev->gds.gds_compute_max_wave_id = 0x7ff;  in gfx_v9_0_set_gds_init()
7075 		adev->gds.gds_compute_max_wave_id = 0x27f;  in gfx_v9_0_set_gds_init()
7079 			adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */  in gfx_v9_0_set_gds_init()
7081 			adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */  in gfx_v9_0_set_gds_init()
7084 		adev->gds.gds_compute_max_wave_id = 0xfff;  in gfx_v9_0_set_gds_init()
7088 		adev->gds.gds_compute_max_wave_id = 0;  in gfx_v9_0_set_gds_init()
7092 		adev->gds.gds_compute_max_wave_id = 0x7ff;  in gfx_v9_0_set_gds_init()
7111 	WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);  in gfx_v9_0_set_user_cu_inactive_bitmap()
7118 	data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);  in gfx_v9_0_get_cu_active_bitmap()
7119 	data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);  in gfx_v9_0_get_cu_active_bitmap()
7132 	int i, j, k, counter, active_cu_number = 0;  in gfx_v9_0_get_cu_info()
7133 	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;  in gfx_v9_0_get_cu_info()
7151 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_get_cu_info()
7152 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_get_cu_info()
7154 			ao_bitmap = 0;  in gfx_v9_0_get_cu_info()
7155 			counter = 0;  in gfx_v9_0_get_cu_info()
7156 			gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);  in gfx_v9_0_get_cu_info()
7168 			 *    SE4,SH0 --> bitmap[0][1]  in gfx_v9_0_get_cu_info()
7175 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_get_cu_info()
7189 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);  in gfx_v9_0_get_cu_info()
7196 	return 0;  in gfx_v9_0_get_cu_info()
7203 	.minor = 0,
7204 	.rev = 0,