929 	struct hmm_devmem *devmem;  in hmm_devmem_ref_release()  local
931 	devmem = container_of(ref, struct hmm_devmem, ref);  in hmm_devmem_ref_release()
932 	complete(&devmem->completion);  in hmm_devmem_ref_release()
938 	struct hmm_devmem *devmem;  in hmm_devmem_ref_exit()  local
940 	devmem = container_of(ref, struct hmm_devmem, ref);  in hmm_devmem_ref_exit()
942 	devm_remove_action(devmem->device, &hmm_devmem_ref_exit, data);  in hmm_devmem_ref_exit()
948 	struct hmm_devmem *devmem;  in hmm_devmem_ref_kill()  local
950 	devmem = container_of(ref, struct hmm_devmem, ref);  in hmm_devmem_ref_kill()
952 	wait_for_completion(&devmem->completion);  in hmm_devmem_ref_kill()
953 	devm_remove_action(devmem->device, &hmm_devmem_ref_kill, data);  in hmm_devmem_ref_kill()
962 	struct hmm_devmem *devmem = page->pgmap->data;  in hmm_devmem_fault()  local
964 	return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);  in hmm_devmem_fault()
969 	struct hmm_devmem *devmem = data;  in hmm_devmem_free()  local
973 	devmem->ops->free(devmem, page);  in hmm_devmem_free()
993 	struct hmm_devmem *devmem = data;  in hmm_devmem_release()  local
994 	struct resource *resource = devmem->resource;  in hmm_devmem_release()
999 	if (percpu_ref_tryget_live(&devmem->ref)) {  in hmm_devmem_release()
1001 		percpu_ref_put(&devmem->ref);  in hmm_devmem_release()
1022 static int hmm_devmem_pages_create(struct hmm_devmem *devmem)  in hmm_devmem_pages_create()  argument
1025 	struct device *device = devmem->device;  in hmm_devmem_pages_create()
1029 	align_start = devmem->resource->start & ~(PA_SECTION_SIZE - 1);  in hmm_devmem_pages_create()
1030 	align_size = ALIGN(devmem->resource->start +  in hmm_devmem_pages_create()
1031 			   resource_size(devmem->resource),  in hmm_devmem_pages_create()
1039 				__func__, devmem->resource);  in hmm_devmem_pages_create()
1045 	if (devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY)  in hmm_devmem_pages_create()
1046 		devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;  in hmm_devmem_pages_create()
1048 		devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;  in hmm_devmem_pages_create()
1050 	devmem->pagemap.res = *devmem->resource;  in hmm_devmem_pages_create()
1051 	devmem->pagemap.page_fault = hmm_devmem_fault;  in hmm_devmem_pages_create()
1052 	devmem->pagemap.page_free = hmm_devmem_free;  in hmm_devmem_pages_create()
1053 	devmem->pagemap.dev = devmem->device;  in hmm_devmem_pages_create()
1054 	devmem->pagemap.ref = &devmem->ref;  in hmm_devmem_pages_create()
1055 	devmem->pagemap.data = devmem;  in hmm_devmem_pages_create()
1073 					devmem);  in hmm_devmem_pages_create()
1097 	if (devmem->pagemap.type == MEMORY_DEVICE_PUBLIC)  in hmm_devmem_pages_create()
1112 	for (pfn = devmem->pfn_first; pfn < devmem->pfn_last; pfn++) {  in hmm_devmem_pages_create()
1115 		page->pgmap = &devmem->pagemap;  in hmm_devmem_pages_create()
1122 	hmm_devmem_radix_release(devmem->resource);  in hmm_devmem_pages_create()
1129 	struct hmm_devmem *devmem = data;  in hmm_devmem_match()  local
1131 	return devmem->resource == match_data;  in hmm_devmem_match()
1134 static void hmm_devmem_pages_remove(struct hmm_devmem *devmem)  in hmm_devmem_pages_remove()  argument
1136 	devres_release(devmem->device, &hmm_devmem_release,  in hmm_devmem_pages_remove()
1137 		       &hmm_devmem_match, devmem->resource);  in hmm_devmem_pages_remove()
1161 	struct hmm_devmem *devmem;  in hmm_devmem_add()  local
1167 	devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),  in hmm_devmem_add()
1169 	if (!devmem)  in hmm_devmem_add()
1172 	init_completion(&devmem->completion);  in hmm_devmem_add()
1173 	devmem->pfn_first = -1UL;  in hmm_devmem_add()
1174 	devmem->pfn_last = -1UL;  in hmm_devmem_add()
1175 	devmem->resource = NULL;  in hmm_devmem_add()
1176 	devmem->device = device;  in hmm_devmem_add()
1177 	devmem->ops = ops;  in hmm_devmem_add()
1179 	ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,  in hmm_devmem_add()
1184 	ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);  in hmm_devmem_add()
1204 		devmem->resource = devm_request_mem_region(device, addr, size,  in hmm_devmem_add()
1206 		if (!devmem->resource) {  in hmm_devmem_add()
1212 	if (!devmem->resource) {  in hmm_devmem_add()
1217 	devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;  in hmm_devmem_add()
1218 	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;  in hmm_devmem_add()
1219 	devmem->pfn_last = devmem->pfn_first +  in hmm_devmem_add()
1220 			   (resource_size(devmem->resource) >> PAGE_SHIFT);  in hmm_devmem_add()
1222 	ret = hmm_devmem_pages_create(devmem);  in hmm_devmem_add()
1226 	devres_add(device, devmem);  in hmm_devmem_add()
1228 	ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);  in hmm_devmem_add()
1230 		hmm_devmem_remove(devmem);  in hmm_devmem_add()
1234 	return devmem;  in hmm_devmem_add()
1237 	devm_release_mem_region(device, devmem->resource->start,  in hmm_devmem_add()
1238 				resource_size(devmem->resource));  in hmm_devmem_add()
1241 	hmm_devmem_ref_kill(&devmem->ref);  in hmm_devmem_add()
1242 	hmm_devmem_ref_exit(&devmem->ref);  in hmm_devmem_add()
1244 	devres_free(devmem);  in hmm_devmem_add()
1253 	struct hmm_devmem *devmem;  in hmm_devmem_add_resource()  local
1261 	devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),  in hmm_devmem_add_resource()
1263 	if (!devmem)  in hmm_devmem_add_resource()
1266 	init_completion(&devmem->completion);  in hmm_devmem_add_resource()
1267 	devmem->pfn_first = -1UL;  in hmm_devmem_add_resource()
1268 	devmem->pfn_last = -1UL;  in hmm_devmem_add_resource()
1269 	devmem->resource = res;  in hmm_devmem_add_resource()
1270 	devmem->device = device;  in hmm_devmem_add_resource()
1271 	devmem->ops = ops;  in hmm_devmem_add_resource()
1273 	ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,  in hmm_devmem_add_resource()
1278 	ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);  in hmm_devmem_add_resource()
1283 	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;  in hmm_devmem_add_resource()
1284 	devmem->pfn_last = devmem->pfn_first +  in hmm_devmem_add_resource()
1285 			   (resource_size(devmem->resource) >> PAGE_SHIFT);  in hmm_devmem_add_resource()
1287 	ret = hmm_devmem_pages_create(devmem);  in hmm_devmem_add_resource()
1291 	devres_add(device, devmem);  in hmm_devmem_add_resource()
1293 	ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);  in hmm_devmem_add_resource()
1295 		hmm_devmem_remove(devmem);  in hmm_devmem_add_resource()
1299 	return devmem;  in hmm_devmem_add_resource()
1302 	hmm_devmem_ref_kill(&devmem->ref);  in hmm_devmem_add_resource()
1303 	hmm_devmem_ref_exit(&devmem->ref);  in hmm_devmem_add_resource()
1305 	devres_free(devmem);  in hmm_devmem_add_resource()
1319 void hmm_devmem_remove(struct hmm_devmem *devmem)  in hmm_devmem_remove()  argument
1325 	if (!devmem)  in hmm_devmem_remove()
1328 	device = devmem->device;  in hmm_devmem_remove()
1329 	start = devmem->resource->start;  in hmm_devmem_remove()
1330 	size = resource_size(devmem->resource);  in hmm_devmem_remove()
1332 	cdm = devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY;  in hmm_devmem_remove()
1333 	hmm_devmem_ref_kill(&devmem->ref);  in hmm_devmem_remove()
1334 	hmm_devmem_ref_exit(&devmem->ref);  in hmm_devmem_remove()
1335 	hmm_devmem_pages_remove(devmem);  in hmm_devmem_remove()