33 static struct device *to_dev(struct pmem_device *pmem)  in to_dev()  argument
39 	return pmem->bb.dev;  in to_dev()
42 static struct nd_region *to_region(struct pmem_device *pmem)  in to_region()  argument
44 	return to_nd_region(to_dev(pmem)->parent);  in to_region()
47 static void hwpoison_clear(struct pmem_device *pmem,  in hwpoison_clear()  argument
53 	if (is_vmalloc_addr(pmem->virt_addr))  in hwpoison_clear()
71 static blk_status_t pmem_clear_poison(struct pmem_device *pmem,  in pmem_clear_poison()  argument
74 	struct device *dev = to_dev(pmem);  in pmem_clear_poison()
79 	sector = (offset - pmem->data_offset) / 512;  in pmem_clear_poison()
81 	cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len);  in pmem_clear_poison()
85 		hwpoison_clear(pmem, pmem->phys_addr + offset, cleared);  in pmem_clear_poison()
90 		badblocks_clear(&pmem->bb, sector, cleared);  in pmem_clear_poison()
91 		if (pmem->bb_state)  in pmem_clear_poison()
92 			sysfs_notify_dirent(pmem->bb_state);  in pmem_clear_poison()
95 	arch_invalidate_pmem(pmem->virt_addr + offset, len);  in pmem_clear_poison()
140 static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page,  in pmem_do_bvec()  argument
146 	phys_addr_t pmem_off = sector * 512 + pmem->data_offset;  in pmem_do_bvec()
147 	void *pmem_addr = pmem->virt_addr + pmem_off;  in pmem_do_bvec()
149 	if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))  in pmem_do_bvec()
177 			rc = pmem_clear_poison(pmem, pmem_off, len);  in pmem_do_bvec()
193 	struct pmem_device *pmem = q->queuedata;  in pmem_make_request()  local
194 	struct nd_region *nd_region = to_region(pmem);  in pmem_make_request()
201 		rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len,  in pmem_make_request()
224 	struct pmem_device *pmem = bdev->bd_queue->queuedata;  in pmem_rw_page()  local
227 	rc = pmem_do_bvec(pmem, page, hpage_nr_pages(page) * PAGE_SIZE,  in pmem_rw_page()
243 __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,  in __pmem_direct_access()  argument
246 	resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;  in __pmem_direct_access()
248 	if (unlikely(is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) / 512,  in __pmem_direct_access()
253 		*kaddr = pmem->virt_addr + offset;  in __pmem_direct_access()
255 		*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);  in __pmem_direct_access()
261 	if (unlikely(pmem->bb.count))  in __pmem_direct_access()
263 	return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);  in __pmem_direct_access()
275 	struct pmem_device *pmem = dax_get_private(dax_dev);  in pmem_dax_direct_access()  local
277 	return __pmem_direct_access(pmem, pgoff, nr_pages, kaddr, pfn);  in pmem_dax_direct_access()
333 	struct pmem_device *pmem = __pmem;  in pmem_release_disk()  local
335 	kill_dax(pmem->dax_dev);  in pmem_release_disk()
336 	put_dax(pmem->dax_dev);  in pmem_release_disk()
337 	del_gendisk(pmem->disk);  in pmem_release_disk()
338 	put_disk(pmem->disk);  in pmem_release_disk()
363 	struct pmem_device *pmem;  in pmem_attach_disk()  local
371 	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);  in pmem_attach_disk()
372 	if (!pmem)  in pmem_attach_disk()
378 		rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap);  in pmem_attach_disk()
386 	dev_set_drvdata(dev, pmem);  in pmem_attach_disk()
387 	pmem->phys_addr = res->start;  in pmem_attach_disk()
388 	pmem->size = resource_size(res);  in pmem_attach_disk()
405 	pmem->pfn_flags = PFN_DEV;  in pmem_attach_disk()
406 	pmem->pgmap.ref = &q->q_usage_counter;  in pmem_attach_disk()
408 		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;  in pmem_attach_disk()
409 		pmem->pgmap.ops = &fsdax_pagemap_ops;  in pmem_attach_disk()
410 		addr = devm_memremap_pages(dev, &pmem->pgmap);  in pmem_attach_disk()
412 		pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);  in pmem_attach_disk()
413 		pmem->pfn_pad = resource_size(res) -  in pmem_attach_disk()
414 			resource_size(&pmem->pgmap.res);  in pmem_attach_disk()
415 		pmem->pfn_flags |= PFN_MAP;  in pmem_attach_disk()
416 		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));  in pmem_attach_disk()
417 		bb_res.start += pmem->data_offset;  in pmem_attach_disk()
419 		memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));  in pmem_attach_disk()
420 		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;  in pmem_attach_disk()
421 		pmem->pgmap.ops = &fsdax_pagemap_ops;  in pmem_attach_disk()
422 		addr = devm_memremap_pages(dev, &pmem->pgmap);  in pmem_attach_disk()
423 		pmem->pfn_flags |= PFN_MAP;  in pmem_attach_disk()
424 		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));  in pmem_attach_disk()
427 					&pmem->pgmap))  in pmem_attach_disk()
429 		addr = devm_memremap(dev, pmem->phys_addr,  in pmem_attach_disk()
430 				pmem->size, ARCH_MEMREMAP_PMEM);  in pmem_attach_disk()
436 	pmem->virt_addr = addr;  in pmem_attach_disk()
444 	if (pmem->pfn_flags & PFN_MAP)  in pmem_attach_disk()
446 	q->queuedata = pmem;  in pmem_attach_disk()
451 	pmem->disk = disk;  in pmem_attach_disk()
458 	set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)  in pmem_attach_disk()
460 	if (devm_init_badblocks(dev, &pmem->bb))  in pmem_attach_disk()
462 	nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_res);  in pmem_attach_disk()
463 	disk->bb = &pmem->bb;  in pmem_attach_disk()
467 	dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags);  in pmem_attach_disk()
473 	pmem->dax_dev = dax_dev;  in pmem_attach_disk()
478 	if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))  in pmem_attach_disk()
483 	pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,  in pmem_attach_disk()
485 	if (!pmem->bb_state)  in pmem_attach_disk()
540 	struct pmem_device *pmem = dev_get_drvdata(dev);  in nd_pmem_remove()  local
549 		sysfs_put(pmem->bb_state);  in nd_pmem_remove()
550 		pmem->bb_state = NULL;  in nd_pmem_remove()
584 		struct pmem_device *pmem = dev_get_drvdata(dev);  in nd_pmem_notify()  local
586 		nd_region = to_region(pmem);  in nd_pmem_notify()
587 		bb = &pmem->bb;  in nd_pmem_notify()
588 		bb_state = pmem->bb_state;  in nd_pmem_notify()
595 			offset = pmem->data_offset +  in nd_pmem_notify()