27 #include <linux/migrate.h>
181  * @mfence: migrate fence
263 static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)  in svm_migrate_successful_pages()  argument
268 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_successful_pages()
269 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_successful_pages()
270 		    migrate->src[i] & MIGRATE_PFN_MIGRATE)  in svm_migrate_successful_pages()
276 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)  in svm_migrate_unsuccessful_pages()  argument
281 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_unsuccessful_pages()
282 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_unsuccessful_pages()
283 		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in svm_migrate_unsuccessful_pages()
291 			 struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_vram()  argument
294 	uint64_t npages = migrate->cpages;  in svm_migrate_copy_to_vram()
315 		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);  in svm_migrate_copy_to_vram()
316 		svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
317 		migrate->dst[i] = migrate_pfn(migrate->dst[i]);  in svm_migrate_copy_to_vram()
319 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_vram()
371 			migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
380 		migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
382 		migrate->dst[i + 1] = 0;  in svm_migrate_copy_to_vram()
384 		migrate->dst[i + 2] = 0;  in svm_migrate_copy_to_vram()
386 		migrate->dst[i + 3] = 0;  in svm_migrate_copy_to_vram()
403 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_vram()  local
409 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_vram()
410 	migrate.vma = vma;  in svm_migrate_vma_to_vram()
411 	migrate.start = start;  in svm_migrate_vma_to_vram()
412 	migrate.end = end;  in svm_migrate_vma_to_vram()
413 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;  in svm_migrate_vma_to_vram()
414 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_vram()
417 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_vram()
422 	migrate.src = buf;  in svm_migrate_vma_to_vram()
423 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_vram()
424 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_vram()
431 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_vram()
438 	cpages = migrate.cpages;  in svm_migrate_vma_to_vram()
440 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_vram()
450 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);  in svm_migrate_vma_to_vram()
451 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_vram()
454 		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);  in svm_migrate_vma_to_vram()
457 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_vram()
479  * svm_migrate_ram_to_vram - migrate svm range from system to device
481  * @best_loc: the device to migrate to
536 			pr_debug("failed %ld to migrate\n", r);  in svm_migrate_ram_to_vram()
567 			struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_ram()  argument
589 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_ram()
615 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);  in svm_migrate_copy_to_ram()
633 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));  in svm_migrate_copy_to_ram()
645 			migrate->dst[i] = 0;  in svm_migrate_copy_to_ram()
653  * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
659  * @node: kfd node device to migrate from
682 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_ram()  local
687 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_ram()
688 	migrate.vma = vma;  in svm_migrate_vma_to_ram()
689 	migrate.start = start;  in svm_migrate_vma_to_ram()
690 	migrate.end = end;  in svm_migrate_vma_to_ram()
691 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_ram()
693 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;  in svm_migrate_vma_to_ram()
695 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;  in svm_migrate_vma_to_ram()
698 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_ram()
703 	migrate.src = buf;  in svm_migrate_vma_to_ram()
704 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_ram()
705 	migrate.fault_page = fault_page;  in svm_migrate_vma_to_ram()
706 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_ram()
713 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_ram()
720 	cpages = migrate.cpages;  in svm_migrate_vma_to_ram()
722 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
724 		upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
733 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
735 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_ram()
737 	upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
739 		 upages, cpages, migrate.npages);  in svm_migrate_vma_to_ram()
742 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_ram()
762  * svm_migrate_vram_to_ram - migrate svm range from device to system
816 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
833  * svm_migrate_vram_to_vram - migrate svm range from device to device
835  * @best_loc: the device to migrate to
962 		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_to_ram()