147 static struct dev_dax *to_dev_dax(struct device *dev)  in to_dev_dax()
149 	return container_of(dev, struct dev_dax, dev);  in to_dev_dax()
155 	struct dev_dax *dev_dax = to_dev_dax(dev);  in size_show()  local
159 	for (i = 0; i < dev_dax->num_resources; i++)  in size_show()
160 		size += resource_size(&dev_dax->res[i]);  in size_show()
180 static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,  in check_vma()  argument
183 	struct dax_region *dax_region = dev_dax->region;  in check_vma()
184 	struct device *dev = &dev_dax->dev;  in check_vma()
187 	if (!dax_alive(dev_dax->dax_dev))  in check_vma()
226 __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,  in dax_pgoff_to_phys()  argument
234 	for (i = 0; i < dev_dax->num_resources; i++) {  in dax_pgoff_to_phys()
235 		res = &dev_dax->res[i];  in dax_pgoff_to_phys()
242 	if (i < dev_dax->num_resources) {  in dax_pgoff_to_phys()
243 		res = &dev_dax->res[i];  in dax_pgoff_to_phys()
251 static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,  in __dev_dax_pte_fault()  argument
254 	struct device *dev = &dev_dax->dev;  in __dev_dax_pte_fault()
259 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pte_fault()
262 	dax_region = dev_dax->region;  in __dev_dax_pte_fault()
272 	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);  in __dev_dax_pte_fault()
283 static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,  in __dev_dax_pmd_fault()  argument
287 	struct device *dev = &dev_dax->dev;  in __dev_dax_pmd_fault()
293 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pmd_fault()
296 	dax_region = dev_dax->region;  in __dev_dax_pmd_fault()
320 	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);  in __dev_dax_pmd_fault()
333 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,  in __dev_dax_pud_fault()  argument
337 	struct device *dev = &dev_dax->dev;  in __dev_dax_pud_fault()
344 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pud_fault()
347 	dax_region = dev_dax->region;  in __dev_dax_pud_fault()
371 	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);  in __dev_dax_pud_fault()
383 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,  in __dev_dax_pud_fault()  argument
398 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_huge_fault()  local
400 	dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,  in dev_dax_huge_fault()
408 		rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);  in dev_dax_huge_fault()
412 		rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);  in dev_dax_huge_fault()
416 		rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);  in dev_dax_huge_fault()
457 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_split()  local
458 	struct dax_region *dax_region = dev_dax->region;  in dev_dax_split()
468 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_pagesize()  local
469 	struct dax_region *dax_region = dev_dax->region;  in dev_dax_pagesize()
483 	struct dev_dax *dev_dax = filp->private_data;  in dax_mmap()  local
486 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_mmap()
493 	rc = check_vma(dev_dax, vma, __func__);  in dax_mmap()
509 	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;  in dax_get_unmapped_area()  local
512 	if (!dev_dax || addr)  in dax_get_unmapped_area()
515 	dax_region = dev_dax->region;  in dax_get_unmapped_area()
547 	struct dev_dax *dev_dax = dax_get_private(dax_dev);  in dax_open()  local
549 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_open()
555 	filp->private_data = dev_dax;  in dax_open()
563 	struct dev_dax *dev_dax = filp->private_data;  in dax_release()  local
565 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_release()
581 	struct dev_dax *dev_dax = to_dev_dax(dev);  in dev_dax_release()  local
582 	struct dax_region *dax_region = dev_dax->region;  in dev_dax_release()
583 	struct dax_device *dax_dev = dev_dax->dax_dev;  in dev_dax_release()
585 	if (dev_dax->id >= 0)  in dev_dax_release()
586 		ida_simple_remove(&dax_region->ida, dev_dax->id);  in dev_dax_release()
589 	kfree(dev_dax);  in dev_dax_release()
592 static void kill_dev_dax(struct dev_dax *dev_dax)  in kill_dev_dax()  argument
594 	struct dax_device *dax_dev = dev_dax->dax_dev;  in kill_dev_dax()
603 	struct dev_dax *dev_dax = to_dev_dax(dev);  in unregister_dev_dax()  local
604 	struct dax_device *dax_dev = dev_dax->dax_dev;  in unregister_dev_dax()
610 	kill_dev_dax(dev_dax);  in unregister_dev_dax()
615 struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,  in devm_create_dev_dax()
620 	struct dev_dax *dev_dax;  in devm_create_dev_dax()  local
629 	dev_dax = kzalloc(struct_size(dev_dax, res, count), GFP_KERNEL);  in devm_create_dev_dax()
630 	if (!dev_dax)  in devm_create_dev_dax()
640 		dev_dax->res[i].start = res[i].start;  in devm_create_dev_dax()
641 		dev_dax->res[i].end = res[i].end;  in devm_create_dev_dax()
649 		dev_dax->id = id;  in devm_create_dev_dax()
656 		dev_dax->id = -1;  in devm_create_dev_dax()
663 	dax_dev = alloc_dax(dev_dax, NULL, NULL);  in devm_create_dev_dax()
670 	dev = &dev_dax->dev;  in devm_create_dev_dax()
678 	dev_dax->num_resources = count;  in devm_create_dev_dax()
679 	dev_dax->dax_dev = dax_dev;  in devm_create_dev_dax()
680 	dev_dax->region = dax_region;  in devm_create_dev_dax()
692 		kill_dev_dax(dev_dax);  in devm_create_dev_dax()
701 	return dev_dax;  in devm_create_dev_dax()
704 	if (dev_dax->id >= 0)  in devm_create_dev_dax()
705 		ida_simple_remove(&dax_region->ida, dev_dax->id);  in devm_create_dev_dax()
707 	kfree(dev_dax);  in devm_create_dev_dax()