27 #include <linux/migrate.h>
187  * @mfence: migrate fence
269 static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)  in svm_migrate_successful_pages()  argument
274 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_successful_pages()
275 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_successful_pages()
276 		    migrate->src[i] & MIGRATE_PFN_MIGRATE)  in svm_migrate_successful_pages()
282 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)  in svm_migrate_unsuccessful_pages()  argument
287 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_unsuccessful_pages()
288 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_unsuccessful_pages()
289 		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in svm_migrate_unsuccessful_pages()
297 			 struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_vram()  argument
300 	uint64_t npages = migrate->npages;  in svm_migrate_copy_to_vram()
326 		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);  in svm_migrate_copy_to_vram()
327 		svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
328 		migrate->dst[i] = migrate_pfn(migrate->dst[i]);  in svm_migrate_copy_to_vram()
330 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_vram()
382 			migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
391 		migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
393 		migrate->dst[i + 1] = 0;  in svm_migrate_copy_to_vram()
395 		migrate->dst[i + 2] = 0;  in svm_migrate_copy_to_vram()
397 		migrate->dst[i + 3] = 0;  in svm_migrate_copy_to_vram()
413 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_vram()  local
419 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_vram()
420 	migrate.vma = vma;  in svm_migrate_vma_to_vram()
421 	migrate.start = start;  in svm_migrate_vma_to_vram()
422 	migrate.end = end;  in svm_migrate_vma_to_vram()
423 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;  in svm_migrate_vma_to_vram()
424 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_vram()
427 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_vram()
432 	migrate.src = buf;  in svm_migrate_vma_to_vram()
433 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_vram()
434 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_vram()
441 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_vram()
448 	cpages = migrate.cpages;  in svm_migrate_vma_to_vram()
450 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_vram()
460 	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence, scratch);  in svm_migrate_vma_to_vram()
461 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_vram()
464 		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);  in svm_migrate_vma_to_vram()
467 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_vram()
490  * svm_migrate_ram_to_vram - migrate svm range from system to device
492  * @best_loc: the device to migrate to
539 			pr_debug("failed %ld to migrate\n", r);  in svm_migrate_ram_to_vram()
565 			struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_ram()  argument
587 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_ram()
613 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);  in svm_migrate_copy_to_ram()
631 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));  in svm_migrate_copy_to_ram()
643 			migrate->dst[i] = 0;  in svm_migrate_copy_to_ram()
651  * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
653  * @adev: amdgpu device to migrate from
677 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_ram()  local
682 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_ram()
683 	migrate.vma = vma;  in svm_migrate_vma_to_ram()
684 	migrate.start = start;  in svm_migrate_vma_to_ram()
685 	migrate.end = end;  in svm_migrate_vma_to_ram()
686 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_ram()
688 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;  in svm_migrate_vma_to_ram()
690 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;  in svm_migrate_vma_to_ram()
693 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_ram()
698 	migrate.src = buf;  in svm_migrate_vma_to_ram()
699 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_ram()
700 	migrate.fault_page = fault_page;  in svm_migrate_vma_to_ram()
701 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_ram()
708 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_ram()
715 	cpages = migrate.cpages;  in svm_migrate_vma_to_ram()
717 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
719 		upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
728 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
730 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_ram()
732 	upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
734 		 upages, cpages, migrate.npages);  in svm_migrate_vma_to_ram()
737 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_ram()
757  * svm_migrate_vram_to_ram - migrate svm range from device to system
812 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
829  * svm_migrate_vram_to_vram - migrate svm range from device to device
831  * @best_loc: the device to migrate to
958 		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_to_ram()