184  * @mfence: migrate fence
269 			 struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_vram()  argument
272 	uint64_t npages = migrate->cpages;  in svm_migrate_copy_to_vram()
297 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_vram()
300 			migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);  in svm_migrate_copy_to_vram()
301 			svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
302 			migrate->dst[i] = migrate_pfn(migrate->dst[i]);  in svm_migrate_copy_to_vram()
303 			migrate->dst[i] |= MIGRATE_PFN_LOCKED;  in svm_migrate_copy_to_vram()
353 			migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
362 		migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
364 		migrate->dst[i + 1] = 0;  in svm_migrate_copy_to_vram()
366 		migrate->dst[i + 2] = 0;  in svm_migrate_copy_to_vram()
368 		migrate->dst[i + 3] = 0;  in svm_migrate_copy_to_vram()
383 	struct migrate_vma migrate;  in svm_migrate_vma_to_vram()  local
389 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_vram()
390 	migrate.vma = vma;  in svm_migrate_vma_to_vram()
391 	migrate.start = start;  in svm_migrate_vma_to_vram()
392 	migrate.end = end;  in svm_migrate_vma_to_vram()
393 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;  in svm_migrate_vma_to_vram()
394 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_vram()
396 	size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);  in svm_migrate_vma_to_vram()
402 	migrate.src = buf;  in svm_migrate_vma_to_vram()
403 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_vram()
404 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_vram()
406 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_vram()
408 		pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_vram()
412 	if (migrate.cpages != npages) {  in svm_migrate_vma_to_vram()
414 			 migrate.cpages,  in svm_migrate_vma_to_vram()
418 	if (migrate.cpages) {  in svm_migrate_vma_to_vram()
419 		r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_vram()
421 		migrate_vma_pages(&migrate);  in svm_migrate_vma_to_vram()
423 		migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_vram()
435 			WRITE_ONCE(pdd->page_in, pdd->page_in + migrate.cpages);  in svm_migrate_vma_to_vram()
442  * svm_migrate_ram_to_vram - migrate svm range from system to device
444  * @best_loc: the device to migrate to
492 			pr_debug("failed to migrate\n");  in svm_migrate_ram_to_vram()
516 			struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_ram()  argument
538 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_ram()
564 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);  in svm_migrate_copy_to_ram()
582 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));  in svm_migrate_copy_to_ram()
583 		migrate->dst[i] |= MIGRATE_PFN_LOCKED;  in svm_migrate_copy_to_ram()
595 			migrate->dst[i] = 0;  in svm_migrate_copy_to_ram()
609 	struct migrate_vma migrate;  in svm_migrate_vma_to_ram()  local
615 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_ram()
616 	migrate.vma = vma;  in svm_migrate_vma_to_ram()
617 	migrate.start = start;  in svm_migrate_vma_to_ram()
618 	migrate.end = end;  in svm_migrate_vma_to_ram()
619 	migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;  in svm_migrate_vma_to_ram()
620 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_ram()
622 	size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);  in svm_migrate_vma_to_ram()
628 	migrate.src = buf;  in svm_migrate_vma_to_ram()
629 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_ram()
630 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_ram()
632 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_ram()
634 		pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
639 	pr_debug("cpages %ld\n", migrate.cpages);  in svm_migrate_vma_to_ram()
641 	if (migrate.cpages) {  in svm_migrate_vma_to_ram()
642 		r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
644 		migrate_vma_pages(&migrate);  in svm_migrate_vma_to_ram()
646 		migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_ram()
648 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
661 				   pdd->page_out + migrate.cpages);  in svm_migrate_vma_to_ram()
667  * svm_migrate_vram_to_ram - migrate svm range from device to system
715 			pr_debug("failed %d to migrate\n", r);  in svm_migrate_vram_to_ram()
729  * svm_migrate_vram_to_vram - migrate svm range from device to device
731  * @best_loc: the device to migrate to
832 		pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,  in svm_migrate_to_ram()