Lines Matching refs:kfd
333 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
335 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
337 static int kfd_resume(struct kfd_dev *kfd);
359 struct kfd_dev *kfd; in kgd2kfd_probe() local
383 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); in kgd2kfd_probe()
384 if (!kfd) in kgd2kfd_probe()
387 kfd->kgd = kgd; in kgd2kfd_probe()
388 kfd->device_info = device_info; in kgd2kfd_probe()
389 kfd->pdev = pdev; in kgd2kfd_probe()
390 kfd->init_complete = false; in kgd2kfd_probe()
391 kfd->kfd2kgd = f2g; in kgd2kfd_probe()
393 mutex_init(&kfd->doorbell_mutex); in kgd2kfd_probe()
394 memset(&kfd->doorbell_available_index, 0, in kgd2kfd_probe()
395 sizeof(kfd->doorbell_available_index)); in kgd2kfd_probe()
397 return kfd; in kgd2kfd_probe()
400 static void kfd_cwsr_init(struct kfd_dev *kfd) in kfd_cwsr_init() argument
402 if (cwsr_enable && kfd->device_info->supports_cwsr) { in kfd_cwsr_init()
403 if (kfd->device_info->asic_family < CHIP_VEGA10) { in kfd_cwsr_init()
405 kfd->cwsr_isa = cwsr_trap_gfx8_hex; in kfd_cwsr_init()
406 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); in kfd_cwsr_init()
409 kfd->cwsr_isa = cwsr_trap_gfx9_hex; in kfd_cwsr_init()
410 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); in kfd_cwsr_init()
413 kfd->cwsr_enabled = true; in kfd_cwsr_init()
417 bool kgd2kfd_device_init(struct kfd_dev *kfd, in kgd2kfd_device_init() argument
422 kfd->shared_resources = *gpu_resources; in kgd2kfd_device_init()
424 kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; in kgd2kfd_device_init()
425 kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; in kgd2kfd_device_init()
426 kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd in kgd2kfd_device_init()
427 - kfd->vm_info.first_vmid_kfd + 1; in kgd2kfd_device_init()
431 || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { in kgd2kfd_device_init()
434 hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, in kgd2kfd_device_init()
435 kfd->vm_info.vmid_num_kfd); in kgd2kfd_device_init()
436 kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; in kgd2kfd_device_init()
438 kfd->max_proc_per_quantum = hws_max_conc_proc; in kgd2kfd_device_init()
442 kfd->device_info->mqd_size_aligned; in kgd2kfd_device_init()
458 if (kfd->kfd2kgd->init_gtt_mem_allocation( in kgd2kfd_device_init()
459 kfd->kgd, size, &kfd->gtt_mem, in kgd2kfd_device_init()
460 &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, in kgd2kfd_device_init()
469 if (kfd_gtt_sa_init(kfd, size, 512) != 0) { in kgd2kfd_device_init()
474 if (kfd_doorbell_init(kfd)) { in kgd2kfd_device_init()
480 if (kfd_topology_add_device(kfd)) { in kgd2kfd_device_init()
485 if (kfd_interrupt_init(kfd)) { in kgd2kfd_device_init()
490 kfd->dqm = device_queue_manager_init(kfd); in kgd2kfd_device_init()
491 if (!kfd->dqm) { in kgd2kfd_device_init()
496 if (kfd_iommu_device_init(kfd)) { in kgd2kfd_device_init()
501 kfd_cwsr_init(kfd); in kgd2kfd_device_init()
503 if (kfd_resume(kfd)) in kgd2kfd_device_init()
506 kfd->dbgmgr = NULL; in kgd2kfd_device_init()
508 kfd->init_complete = true; in kgd2kfd_device_init()
509 dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, in kgd2kfd_device_init()
510 kfd->pdev->device); in kgd2kfd_device_init()
513 kfd->dqm->sched_policy); in kgd2kfd_device_init()
519 device_queue_manager_uninit(kfd->dqm); in kgd2kfd_device_init()
521 kfd_interrupt_exit(kfd); in kgd2kfd_device_init()
523 kfd_topology_remove_device(kfd); in kgd2kfd_device_init()
525 kfd_doorbell_fini(kfd); in kgd2kfd_device_init()
527 kfd_gtt_sa_fini(kfd); in kgd2kfd_device_init()
529 kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem); in kgd2kfd_device_init()
532 kfd->pdev->vendor, kfd->pdev->device); in kgd2kfd_device_init()
534 return kfd->init_complete; in kgd2kfd_device_init()
537 void kgd2kfd_device_exit(struct kfd_dev *kfd) in kgd2kfd_device_exit() argument
539 if (kfd->init_complete) { in kgd2kfd_device_exit()
540 kgd2kfd_suspend(kfd); in kgd2kfd_device_exit()
541 device_queue_manager_uninit(kfd->dqm); in kgd2kfd_device_exit()
542 kfd_interrupt_exit(kfd); in kgd2kfd_device_exit()
543 kfd_topology_remove_device(kfd); in kgd2kfd_device_exit()
544 kfd_doorbell_fini(kfd); in kgd2kfd_device_exit()
545 kfd_gtt_sa_fini(kfd); in kgd2kfd_device_exit()
546 kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem); in kgd2kfd_device_exit()
549 kfree(kfd); in kgd2kfd_device_exit()
552 int kgd2kfd_pre_reset(struct kfd_dev *kfd) in kgd2kfd_pre_reset() argument
554 if (!kfd->init_complete) in kgd2kfd_pre_reset()
556 kgd2kfd_suspend(kfd); in kgd2kfd_pre_reset()
559 dqm_lock(kfd->dqm); in kgd2kfd_pre_reset()
561 kfd_signal_reset_event(kfd); in kgd2kfd_pre_reset()
571 int kgd2kfd_post_reset(struct kfd_dev *kfd) in kgd2kfd_post_reset() argument
575 if (!kfd->init_complete) in kgd2kfd_post_reset()
578 dqm_unlock(kfd->dqm); in kgd2kfd_post_reset()
580 ret = kfd_resume(kfd); in kgd2kfd_post_reset()
593 void kgd2kfd_suspend(struct kfd_dev *kfd) in kgd2kfd_suspend() argument
595 if (!kfd->init_complete) in kgd2kfd_suspend()
602 kfd->dqm->ops.stop(kfd->dqm); in kgd2kfd_suspend()
604 kfd_iommu_suspend(kfd); in kgd2kfd_suspend()
607 int kgd2kfd_resume(struct kfd_dev *kfd) in kgd2kfd_resume() argument
611 if (!kfd->init_complete) in kgd2kfd_resume()
614 ret = kfd_resume(kfd); in kgd2kfd_resume()
626 static int kfd_resume(struct kfd_dev *kfd) in kfd_resume() argument
630 err = kfd_iommu_resume(kfd); in kfd_resume()
634 kfd->pdev->vendor, kfd->pdev->device); in kfd_resume()
638 err = kfd->dqm->ops.start(kfd->dqm); in kfd_resume()
642 kfd->pdev->vendor, kfd->pdev->device); in kfd_resume()
649 kfd_iommu_suspend(kfd); in kfd_resume()
654 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) in kgd2kfd_interrupt() argument
659 if (!kfd->init_complete) in kgd2kfd_interrupt()
662 if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { in kgd2kfd_interrupt()
667 spin_lock(&kfd->interrupt_lock); in kgd2kfd_interrupt()
669 if (kfd->interrupts_active in kgd2kfd_interrupt()
670 && interrupt_is_wanted(kfd, ih_ring_entry, in kgd2kfd_interrupt()
672 && enqueue_ih_ring_entry(kfd, in kgd2kfd_interrupt()
674 queue_work(kfd->ih_wq, &kfd->interrupt_work); in kgd2kfd_interrupt()
676 spin_unlock(&kfd->interrupt_lock); in kgd2kfd_interrupt()
765 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, in kfd_gtt_sa_init() argument
777 kfd->gtt_sa_chunk_size = chunk_size; in kfd_gtt_sa_init()
778 kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; in kfd_gtt_sa_init()
780 num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / in kfd_gtt_sa_init()
783 kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); in kfd_gtt_sa_init()
785 if (!kfd->gtt_sa_bitmap) in kfd_gtt_sa_init()
789 kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); in kfd_gtt_sa_init()
791 mutex_init(&kfd->gtt_sa_lock); in kfd_gtt_sa_init()
797 static void kfd_gtt_sa_fini(struct kfd_dev *kfd) in kfd_gtt_sa_fini() argument
799 mutex_destroy(&kfd->gtt_sa_lock); in kfd_gtt_sa_fini()
800 kfree(kfd->gtt_sa_bitmap); in kfd_gtt_sa_fini()
817 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, in kfd_gtt_sa_allocate() argument
825 if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) in kfd_gtt_sa_allocate()
836 mutex_lock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
840 found = find_next_zero_bit(kfd->gtt_sa_bitmap, in kfd_gtt_sa_allocate()
841 kfd->gtt_sa_num_of_chunks, in kfd_gtt_sa_allocate()
847 if (found == kfd->gtt_sa_num_of_chunks) in kfd_gtt_sa_allocate()
854 kfd->gtt_start_gpu_addr, in kfd_gtt_sa_allocate()
856 kfd->gtt_sa_chunk_size); in kfd_gtt_sa_allocate()
858 kfd->gtt_start_cpu_ptr, in kfd_gtt_sa_allocate()
860 kfd->gtt_sa_chunk_size); in kfd_gtt_sa_allocate()
866 if (size <= kfd->gtt_sa_chunk_size) { in kfd_gtt_sa_allocate()
868 set_bit(found, kfd->gtt_sa_bitmap); in kfd_gtt_sa_allocate()
873 cur_size = size - kfd->gtt_sa_chunk_size; in kfd_gtt_sa_allocate()
876 find_next_zero_bit(kfd->gtt_sa_bitmap, in kfd_gtt_sa_allocate()
877 kfd->gtt_sa_num_of_chunks, ++found); in kfd_gtt_sa_allocate()
891 if (found == kfd->gtt_sa_num_of_chunks) in kfd_gtt_sa_allocate()
895 if (cur_size <= kfd->gtt_sa_chunk_size) in kfd_gtt_sa_allocate()
898 cur_size -= kfd->gtt_sa_chunk_size; in kfd_gtt_sa_allocate()
909 set_bit(found, kfd->gtt_sa_bitmap); in kfd_gtt_sa_allocate()
912 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
917 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
922 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) in kfd_gtt_sa_free() argument
933 mutex_lock(&kfd->gtt_sa_lock); in kfd_gtt_sa_free()
939 clear_bit(bit, kfd->gtt_sa_bitmap); in kfd_gtt_sa_free()
941 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_free()