45 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)  in devmap_managed_enable_put()  argument
47 	if (pgmap->type == MEMORY_DEVICE_FS_DAX)  in devmap_managed_enable_put()
51 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)  in devmap_managed_enable_get()  argument
53 	if (pgmap->type == MEMORY_DEVICE_FS_DAX)  in devmap_managed_enable_get()
57 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)  in devmap_managed_enable_get()  argument
60 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)  in devmap_managed_enable_put()  argument
72 static unsigned long pfn_first(struct dev_pagemap *pgmap, int range_id)  in pfn_first()  argument
74 	struct range *range = &pgmap->ranges[range_id];  in pfn_first()
79 	return pfn + vmem_altmap_offset(pgmap_altmap(pgmap));  in pfn_first()
82 bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)  in pgmap_pfn_valid()  argument
86 	for (i = 0; i < pgmap->nr_range; i++) {  in pgmap_pfn_valid()
87 		struct range *range = &pgmap->ranges[i];  in pgmap_pfn_valid()
91 			return pfn >= pfn_first(pgmap, i);  in pgmap_pfn_valid()
97 static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id)  in pfn_end()  argument
99 	const struct range *range = &pgmap->ranges[range_id];  in pfn_end()
104 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)  in pfn_len()  argument
106 	return (pfn_end(pgmap, range_id) -  in pfn_len()
107 		pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;  in pfn_len()
110 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)  in pageunmap_range()  argument
112 	struct range *range = &pgmap->ranges[range_id];  in pageunmap_range()
116 	first_page = pfn_to_page(pfn_first(pgmap, range_id));  in pageunmap_range()
122 	if (pgmap->type == MEMORY_DEVICE_PRIVATE) {  in pageunmap_range()
127 				pgmap_altmap(pgmap));  in pageunmap_range()
136 void memunmap_pages(struct dev_pagemap *pgmap)  in memunmap_pages()  argument
140 	percpu_ref_kill(&pgmap->ref);  in memunmap_pages()
141 	if (pgmap->type != MEMORY_DEVICE_PRIVATE &&  in memunmap_pages()
142 	    pgmap->type != MEMORY_DEVICE_COHERENT)  in memunmap_pages()
143 		for (i = 0; i < pgmap->nr_range; i++)  in memunmap_pages()
144 			percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));  in memunmap_pages()
146 	wait_for_completion(&pgmap->done);  in memunmap_pages()
148 	for (i = 0; i < pgmap->nr_range; i++)  in memunmap_pages()
149 		pageunmap_range(pgmap, i);  in memunmap_pages()
150 	percpu_ref_exit(&pgmap->ref);  in memunmap_pages()
152 	WARN_ONCE(pgmap->altmap.alloc, "failed to free all reserved pages\n");  in memunmap_pages()
153 	devmap_managed_enable_put(pgmap);  in memunmap_pages()
164 	struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref);  in dev_pagemap_percpu_release()  local
166 	complete(&pgmap->done);  in dev_pagemap_percpu_release()
169 static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params,  in pagemap_range()  argument
172 	const bool is_private = pgmap->type == MEMORY_DEVICE_PRIVATE;  in pagemap_range()
173 	struct range *range = &pgmap->ranges[range_id];  in pagemap_range()
177 	if (WARN_ONCE(pgmap_altmap(pgmap) && range_id > 0,  in pagemap_range()
206 				PHYS_PFN(range->end), pgmap, GFP_KERNEL));  in pagemap_range()
269 				PHYS_PFN(range_len(range)), pgmap);  in pagemap_range()
270 	if (pgmap->type != MEMORY_DEVICE_PRIVATE &&  in pagemap_range()
271 	    pgmap->type != MEMORY_DEVICE_COHERENT)  in pagemap_range()
272 		percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id));  in pagemap_range()
291 void *memremap_pages(struct dev_pagemap *pgmap, int nid)  in memremap_pages()  argument
294 		.altmap = pgmap_altmap(pgmap),  in memremap_pages()
295 		.pgmap = pgmap,  in memremap_pages()
298 	const int nr_range = pgmap->nr_range;  in memremap_pages()
304 	switch (pgmap->type) {  in memremap_pages()
310 		if (!pgmap->ops || !pgmap->ops->migrate_to_ram) {  in memremap_pages()
314 		if (!pgmap->ops->page_free) {  in memremap_pages()
318 		if (!pgmap->owner) {  in memremap_pages()
324 		if (!pgmap->ops->page_free) {  in memremap_pages()
328 		if (!pgmap->owner) {  in memremap_pages()
346 		WARN(1, "Invalid pgmap type %d\n", pgmap->type);  in memremap_pages()
350 	init_completion(&pgmap->done);  in memremap_pages()
351 	error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0,  in memremap_pages()
356 	devmap_managed_enable_get(pgmap);  in memremap_pages()
363 	pgmap->nr_range = 0;  in memremap_pages()
366 		error = pagemap_range(pgmap, &params, i, nid);  in memremap_pages()
369 		pgmap->nr_range++;  in memremap_pages()
373 		memunmap_pages(pgmap);  in memremap_pages()
374 		pgmap->nr_range = nr_range;  in memremap_pages()
378 	return __va(pgmap->ranges[0].start);  in memremap_pages()
402 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)  in devm_memremap_pages()  argument
407 	ret = memremap_pages(pgmap, dev_to_node(dev));  in devm_memremap_pages()
412 			pgmap);  in devm_memremap_pages()
419 void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)  in devm_memunmap_pages()  argument
421 	devm_release_action(dev, devm_memremap_pages_release, pgmap);  in devm_memunmap_pages()
447 		struct dev_pagemap *pgmap)  in get_dev_pagemap()  argument
454 	if (pgmap) {  in get_dev_pagemap()
455 		if (phys >= pgmap->range.start && phys <= pgmap->range.end)  in get_dev_pagemap()
456 			return pgmap;  in get_dev_pagemap()
457 		put_dev_pagemap(pgmap);  in get_dev_pagemap()
462 	pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));  in get_dev_pagemap()
463 	if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref))  in get_dev_pagemap()
464 		pgmap = NULL;  in get_dev_pagemap()
467 	return pgmap;  in get_dev_pagemap()
473 	if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))  in free_zone_device_page()
509 	page->pgmap->ops->page_free(page);  in free_zone_device_page()
511 	if (page->pgmap->type != MEMORY_DEVICE_PRIVATE &&  in free_zone_device_page()
512 	    page->pgmap->type != MEMORY_DEVICE_COHERENT)  in free_zone_device_page()
519 		put_dev_pagemap(page->pgmap);  in free_zone_device_page()
528 	WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref));  in zone_device_page_init()
537 	if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)  in __put_devmap_managed_page_refs()