62 int nd_region_activate(struct nd_region *nd_region)  in nd_region_activate()  argument
66 	struct device *dev = &nd_region->dev;  in nd_region_activate()
69 	nvdimm_bus_lock(&nd_region->dev);  in nd_region_activate()
70 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_activate()
71 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_activate()
75 			nvdimm_bus_unlock(&nd_region->dev);  in nd_region_activate()
86 	nvdimm_bus_unlock(&nd_region->dev);  in nd_region_activate()
97 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_activate()
98 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_activate()
100 		int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd);  in nd_region_activate()
110 	for (i = 0; i < nd_region->ndr_mappings - 1; i++) {  in nd_region_activate()
115 		for (j = i + 1; j < nd_region->ndr_mappings; j++)  in nd_region_activate()
126 	struct nd_region *nd_region = to_nd_region(dev);  in nd_region_release()  local
129 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_release()
130 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_release()
135 	free_percpu(nd_region->lane);  in nd_region_release()
136 	if (!test_bit(ND_REGION_CXL, &nd_region->flags))  in nd_region_release()
137 		memregion_free(nd_region->id);  in nd_region_release()
138 	kfree(nd_region);  in nd_region_release()
141 struct nd_region *to_nd_region(struct device *dev)  in to_nd_region()
143 	struct nd_region *nd_region = container_of(dev, struct nd_region, dev);  in to_nd_region()  local
146 	return nd_region;  in to_nd_region()
150 struct device *nd_region_dev(struct nd_region *nd_region)  in nd_region_dev()  argument
152 	if (!nd_region)  in nd_region_dev()
154 	return &nd_region->dev;  in nd_region_dev()
158 void *nd_region_provider_data(struct nd_region *nd_region)  in nd_region_provider_data()  argument
160 	return nd_region->provider_data;  in nd_region_provider_data()
172 int nd_region_to_nstype(struct nd_region *nd_region)  in nd_region_to_nstype()  argument
174 	if (is_memory(&nd_region->dev)) {  in nd_region_to_nstype()
177 		for (i = 0, label = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_to_nstype()
178 			struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_to_nstype()
194 static unsigned long long region_size(struct nd_region *nd_region)  in region_size()  argument
196 	if (is_memory(&nd_region->dev)) {  in region_size()
197 		return nd_region->ndr_size;  in region_size()
198 	} else if (nd_region->ndr_mappings == 1) {  in region_size()
199 		struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in region_size()
210 	struct nd_region *nd_region = to_nd_region(dev);  in size_show()  local
212 	return sprintf(buf, "%llu\n", region_size(nd_region));  in size_show()
219 	struct nd_region *nd_region = to_nd_region(dev);  in deep_flush_show()  local
225 	return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region));  in deep_flush_show()
233 	struct nd_region *nd_region = to_nd_region(dev);  in deep_flush_store()  local
239 	rc = nvdimm_flush(nd_region, NULL);  in deep_flush_store()
250 	struct nd_region *nd_region = to_nd_region(dev);  in mappings_show()  local
252 	return sprintf(buf, "%d\n", nd_region->ndr_mappings);  in mappings_show()
259 	struct nd_region *nd_region = to_nd_region(dev);  in nstype_show()  local
261 	return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region));  in nstype_show()
268 	struct nd_region *nd_region = to_nd_region(dev);  in set_cookie_show()  local
269 	struct nd_interleave_set *nd_set = nd_region->nd_set;  in set_cookie_show()
286 	if (nd_region->ndr_mappings) {  in set_cookie_show()
287 		struct nd_mapping *nd_mapping = &nd_region->mapping[0];  in set_cookie_show()
295 					nd_region_interleave_set_cookie(nd_region,  in set_cookie_show()
308 resource_size_t nd_region_available_dpa(struct nd_region *nd_region)  in nd_region_available_dpa()  argument
313 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_available_dpa()
316 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_available_dpa()
317 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_available_dpa()
324 		available += nd_pmem_available_dpa(nd_region, nd_mapping);  in nd_region_available_dpa()
330 resource_size_t nd_region_allocatable_dpa(struct nd_region *nd_region)  in nd_region_allocatable_dpa()  argument
335 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));  in nd_region_allocatable_dpa()
336 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nd_region_allocatable_dpa()
337 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nd_region_allocatable_dpa()
340 						    nd_region, nd_mapping));  in nd_region_allocatable_dpa()
342 	return avail * nd_region->ndr_mappings;  in nd_region_allocatable_dpa()
348 	struct nd_region *nd_region = to_nd_region(dev);  in available_size_show()  local
360 	available = nd_region_available_dpa(nd_region);  in available_size_show()
371 	struct nd_region *nd_region = to_nd_region(dev);  in max_available_extent_show()  local
377 	available = nd_region_allocatable_dpa(nd_region);  in max_available_extent_show()
405 	struct nd_region *nd_region = to_nd_region(dev);  in namespace_seed_show()  local
409 	if (nd_region->ns_seed)  in namespace_seed_show()
410 		rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed));  in namespace_seed_show()
421 	struct nd_region *nd_region = to_nd_region(dev);  in btt_seed_show()  local
425 	if (nd_region->btt_seed)  in btt_seed_show()
426 		rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed));  in btt_seed_show()
438 	struct nd_region *nd_region = to_nd_region(dev);  in pfn_seed_show()  local
442 	if (nd_region->pfn_seed)  in pfn_seed_show()
443 		rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed));  in pfn_seed_show()
455 	struct nd_region *nd_region = to_nd_region(dev);  in dax_seed_show()  local
459 	if (nd_region->dax_seed)  in dax_seed_show()
460 		rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed));  in dax_seed_show()
472 	struct nd_region *nd_region = to_nd_region(dev);  in read_only_show()  local
474 	return sprintf(buf, "%d\n", nd_region->ro);  in read_only_show()
488 	struct nd_region *nd_region = to_nd_region(dev);  in read_only_store()  local
493 	nd_region->ro = ro;  in read_only_store()
502 	struct nd_region *nd_region = to_nd_region(dev);  in align_show()  local
504 	return sprintf(buf, "%#lx\n", nd_region->align);  in align_show()
510 	struct nd_region *nd_region = to_nd_region(dev);  in align_store()  local
526 	mappings = max_t(u32, 1, nd_region->ndr_mappings);  in align_store()
529 			|| val > region_size(nd_region) || remainder)  in align_store()
538 	nd_region->align = val;  in align_store()
548 	struct nd_region *nd_region = to_nd_region(dev);  in region_badblocks_show()  local
553 		rc = badblocks_show(&nd_region->bb, buf, 0);  in region_badblocks_show()
565 	struct nd_region *nd_region = to_nd_region(dev);  in resource_show()  local
567 	return sprintf(buf, "%#llx\n", nd_region->ndr_start);  in resource_show()
574 	struct nd_region *nd_region = to_nd_region(dev);  in persistence_domain_show()  local
576 	if (test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags))  in persistence_domain_show()
578 	else if (test_bit(ND_REGION_PERSIST_MEMCTRL, &nd_region->flags))  in persistence_domain_show()
609 	struct nd_region *nd_region = to_nd_region(dev);  in region_visible()  local
610 	struct nd_interleave_set *nd_set = nd_region->nd_set;  in region_visible()
611 	int type = nd_region_to_nstype(nd_region);  in region_visible()
626 		int has_flush = nvdimm_has_flush(nd_region);  in region_visible()
637 		if ((nd_region->flags & (BIT(ND_REGION_PERSIST_CACHE)  in region_visible()
661 	struct nd_region *nd_region = to_nd_region(dev);  in mappingN()  local
665 	if (n >= nd_region->ndr_mappings)  in mappingN()
667 	nd_mapping = &nd_region->mapping[n];  in mappingN()
723 	struct nd_region *nd_region = to_nd_region(dev);  in mapping_visible()  local
725 	if (n < nd_region->ndr_mappings)  in mapping_visible()
806 u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,  in nd_region_interleave_set_cookie()  argument
809 	struct nd_interleave_set *nd_set = nd_region->nd_set;  in nd_region_interleave_set_cookie()
820 u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region)  in nd_region_interleave_set_altcookie()  argument
822 	struct nd_interleave_set *nd_set = nd_region->nd_set;  in nd_region_interleave_set_altcookie()
844 void nd_region_advance_seeds(struct nd_region *nd_region, struct device *dev)  in nd_region_advance_seeds()  argument
847 	if (nd_region->ns_seed == dev) {  in nd_region_advance_seeds()
848 		nd_region_create_ns_seed(nd_region);  in nd_region_advance_seeds()
852 		if (nd_region->btt_seed == dev)  in nd_region_advance_seeds()
853 			nd_region_create_btt_seed(nd_region);  in nd_region_advance_seeds()
854 		if (nd_region->ns_seed == &nd_btt->ndns->dev)  in nd_region_advance_seeds()
855 			nd_region_create_ns_seed(nd_region);  in nd_region_advance_seeds()
859 		if (nd_region->pfn_seed == dev)  in nd_region_advance_seeds()
860 			nd_region_create_pfn_seed(nd_region);  in nd_region_advance_seeds()
861 		if (nd_region->ns_seed == &nd_pfn->ndns->dev)  in nd_region_advance_seeds()
862 			nd_region_create_ns_seed(nd_region);  in nd_region_advance_seeds()
866 		if (nd_region->dax_seed == dev)  in nd_region_advance_seeds()
867 			nd_region_create_dax_seed(nd_region);  in nd_region_advance_seeds()
868 		if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev)  in nd_region_advance_seeds()
869 			nd_region_create_ns_seed(nd_region);  in nd_region_advance_seeds()
891 unsigned int nd_region_acquire_lane(struct nd_region *nd_region)  in nd_region_acquire_lane()  argument
896 	if (nd_region->num_lanes < nr_cpu_ids) {  in nd_region_acquire_lane()
899 		lane = cpu % nd_region->num_lanes;  in nd_region_acquire_lane()
900 		ndl_count = per_cpu_ptr(nd_region->lane, cpu);  in nd_region_acquire_lane()
901 		ndl_lock = per_cpu_ptr(nd_region->lane, lane);  in nd_region_acquire_lane()
911 void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane)  in nd_region_release_lane()  argument
913 	if (nd_region->num_lanes < nr_cpu_ids) {  in nd_region_release_lane()
917 		ndl_count = per_cpu_ptr(nd_region->lane, cpu);  in nd_region_release_lane()
918 		ndl_lock = per_cpu_ptr(nd_region->lane, lane);  in nd_region_release_lane()
933 static unsigned long default_align(struct nd_region *nd_region)  in default_align()  argument
940 	if (nd_region->ndr_size < MEMREMAP_COMPAT_ALIGN_MAX)  in default_align()
943 	mappings = max_t(u16, 1, nd_region->ndr_mappings);  in default_align()
953 static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,  in nd_region_create()
957 	struct nd_region *nd_region;  in nd_region_create()  local
978 	nd_region =  in nd_region_create()
979 		kzalloc(struct_size(nd_region, mapping, ndr_desc->num_mappings),  in nd_region_create()
982 	if (!nd_region)  in nd_region_create()
986 		nd_region->id = ndr_desc->memregion;  in nd_region_create()
988 		nd_region->id = memregion_alloc(GFP_KERNEL);  in nd_region_create()
989 		if (nd_region->id < 0)  in nd_region_create()
993 	nd_region->lane = alloc_percpu(struct nd_percpu_lane);  in nd_region_create()
994 	if (!nd_region->lane)  in nd_region_create()
1000 		ndl = per_cpu_ptr(nd_region->lane, i);  in nd_region_create()
1009 		nd_region->mapping[i].nvdimm = nvdimm;  in nd_region_create()
1010 		nd_region->mapping[i].start = mapping->start;  in nd_region_create()
1011 		nd_region->mapping[i].size = mapping->size;  in nd_region_create()
1012 		nd_region->mapping[i].position = mapping->position;  in nd_region_create()
1013 		INIT_LIST_HEAD(&nd_region->mapping[i].labels);  in nd_region_create()
1014 		mutex_init(&nd_region->mapping[i].lock);  in nd_region_create()
1018 	nd_region->ndr_mappings = ndr_desc->num_mappings;  in nd_region_create()
1019 	nd_region->provider_data = ndr_desc->provider_data;  in nd_region_create()
1020 	nd_region->nd_set = ndr_desc->nd_set;  in nd_region_create()
1021 	nd_region->num_lanes = ndr_desc->num_lanes;  in nd_region_create()
1022 	nd_region->flags = ndr_desc->flags;  in nd_region_create()
1023 	nd_region->ro = ro;  in nd_region_create()
1024 	nd_region->numa_node = ndr_desc->numa_node;  in nd_region_create()
1025 	nd_region->target_node = ndr_desc->target_node;  in nd_region_create()
1026 	ida_init(&nd_region->ns_ida);  in nd_region_create()
1027 	ida_init(&nd_region->btt_ida);  in nd_region_create()
1028 	ida_init(&nd_region->pfn_ida);  in nd_region_create()
1029 	ida_init(&nd_region->dax_ida);  in nd_region_create()
1030 	dev = &nd_region->dev;  in nd_region_create()
1031 	dev_set_name(dev, "region%d", nd_region->id);  in nd_region_create()
1036 	nd_region->ndr_size = resource_size(ndr_desc->res);  in nd_region_create()
1037 	nd_region->ndr_start = ndr_desc->res->start;  in nd_region_create()
1038 	nd_region->align = default_align(nd_region);  in nd_region_create()
1040 		nd_region->flush = ndr_desc->flush;  in nd_region_create()
1042 		nd_region->flush = NULL;  in nd_region_create()
1048 	return nd_region;  in nd_region_create()
1052 		memregion_free(nd_region->id);  in nd_region_create()
1054 	kfree(nd_region);  in nd_region_create()
1058 struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus,  in nvdimm_pmem_region_create()
1067 struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus,  in nvdimm_volatile_region_create()
1076 void nvdimm_region_delete(struct nd_region *nd_region)  in nvdimm_region_delete()  argument
1078 	if (nd_region)  in nvdimm_region_delete()
1079 		nd_device_unregister(&nd_region->dev, ND_SYNC);  in nvdimm_region_delete()
1083 int nvdimm_flush(struct nd_region *nd_region, struct bio *bio)  in nvdimm_flush()  argument
1087 	if (!nd_region->flush)  in nvdimm_flush()
1088 		rc = generic_nvdimm_flush(nd_region);  in nvdimm_flush()
1090 		if (nd_region->flush(nd_region, bio))  in nvdimm_flush()
1100 int generic_nvdimm_flush(struct nd_region *nd_region)  in generic_nvdimm_flush()  argument
1102 	struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);  in generic_nvdimm_flush()
1120 	for (i = 0; i < nd_region->ndr_mappings; i++)  in generic_nvdimm_flush()
1137 int nvdimm_has_flush(struct nd_region *nd_region)  in nvdimm_has_flush()  argument
1142 	if (nd_region->ndr_mappings == 0  in nvdimm_has_flush()
1147 	if (test_bit(ND_REGION_ASYNC, &nd_region->flags) && nd_region->flush)  in nvdimm_has_flush()
1151 	for (i = 0; i < nd_region->ndr_mappings; i++) {  in nvdimm_has_flush()
1152 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];  in nvdimm_has_flush()
1168 int nvdimm_has_cache(struct nd_region *nd_region)  in nvdimm_has_cache()  argument
1170 	return is_nd_pmem(&nd_region->dev) &&  in nvdimm_has_cache()
1171 		!test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags);  in nvdimm_has_cache()
1175 bool is_nvdimm_sync(struct nd_region *nd_region)  in is_nvdimm_sync()  argument
1177 	if (is_nd_volatile(&nd_region->dev))  in is_nvdimm_sync()
1180 	return is_nd_pmem(&nd_region->dev) &&  in is_nvdimm_sync()
1181 		!test_bit(ND_REGION_ASYNC, &nd_region->flags);  in is_nvdimm_sync()
1186 	struct nd_region *nd_region;  member
1192 	struct nd_region *nd_region;  in region_conflict()  local
1199 	nd_region = to_nd_region(dev);  in region_conflict()
1200 	if (nd_region == ctx->nd_region)  in region_conflict()
1204 	region_start = nd_region->ndr_start;  in region_conflict()
1205 	region_end = region_start + nd_region->ndr_size;  in region_conflict()
1213 int nd_region_conflict(struct nd_region *nd_region, resource_size_t start,  in nd_region_conflict()  argument
1216 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);  in nd_region_conflict()
1218 		.nd_region = nd_region,  in nd_region_conflict()