52 static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in create_direct_mr()  argument
64 	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);  in create_direct_mr()
70 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_direct_mr()
79 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mr, in, inlen);  in create_direct_mr()
82 		mlx5_vdpa_warn(mvdev, "Failed to create direct MR\n");  in create_direct_mr()
89 static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in destroy_direct_mr()  argument
91 	mlx5_vdpa_destroy_mkey(mvdev, &mr->mr);  in destroy_direct_mr()
145 static void fill_indir(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey, void *in)  in fill_indir()  argument
169 			klm->key = cpu_to_be32(mvdev->res.null_mkey);  in fill_indir()
182 static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_indirect_key()  argument
201 	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);  in create_indirect_key()
207 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_indirect_key()
212 	fill_indir(mvdev, mr, in);  in create_indirect_key()
213 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_indirect_key()
218 static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)  in destroy_indirect_key()  argument
220 	mlx5_vdpa_destroy_mkey(mvdev, &mkey->mkey);  in destroy_indirect_key()
223 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,  in map_direct_mr()  argument
238 	struct device *dma = mvdev->vdev.dma_dev;  in map_direct_mr()
262 				mlx5_vdpa_warn(mvdev, "sg null. start 0x%llx, end 0x%llx\n",  in map_direct_mr()
282 	err = create_direct_mr(mvdev, mr);  in map_direct_mr()
295 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in unmap_direct_mr()  argument
297 	struct device *dma = mvdev->vdev.dma_dev;  in unmap_direct_mr()
299 	destroy_direct_mr(mvdev, mr);  in unmap_direct_mr()
304 static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, u64 start, u64 size, u8 perm,  in add_direct_chain()  argument
307 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in add_direct_chain()
328 		err = map_direct_mr(mvdev, dmr, iotlb);  in add_direct_chain()
347 		unmap_direct_mr(mvdev, dmr);  in add_direct_chain()
359 static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)  in create_user_mr()  argument
361 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in create_user_mr()
389 				err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);  in create_user_mr()
398 	err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);  in create_user_mr()
406 	err = create_indirect_key(mvdev, mr);  in create_user_mr()
416 		unmap_direct_mr(mvdev, dmr);  in create_user_mr()
422 static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_dma_mr()  argument
439 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_dma_mr()
442 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_dma_mr()
450 static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_dma_mr()  argument
452 	mlx5_vdpa_destroy_mkey(mvdev, &mr->mkey);  in destroy_dma_mr()
455 static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)  in dup_iotlb()  argument
462 		err = vhost_iotlb_add_range(mvdev->cvq.iotlb, start, last, start, VHOST_ACCESS_RW);  in dup_iotlb()
468 		err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, map->last,  in dup_iotlb()
476 static void prune_iotlb(struct mlx5_vdpa_dev *mvdev)  in prune_iotlb()  argument
478 	vhost_iotlb_del_range(mvdev->cvq.iotlb, 0, ULLONG_MAX);  in prune_iotlb()
481 static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_user_mr()  argument
486 	destroy_indirect_key(mvdev, mr);  in destroy_user_mr()
489 		unmap_direct_mr(mvdev, dmr);  in destroy_user_mr()
494 void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev)  in mlx5_vdpa_destroy_mr()  argument
496 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in mlx5_vdpa_destroy_mr()
502 	prune_iotlb(mvdev);  in mlx5_vdpa_destroy_mr()
504 		destroy_user_mr(mvdev, mr);  in mlx5_vdpa_destroy_mr()
506 		destroy_dma_mr(mvdev, mr);  in mlx5_vdpa_destroy_mr()
514 static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)  in _mlx5_vdpa_create_mr()  argument
516 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in _mlx5_vdpa_create_mr()
523 		err = create_user_mr(mvdev, iotlb);  in _mlx5_vdpa_create_mr()
525 		err = create_dma_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
530 	err = dup_iotlb(mvdev, iotlb);  in _mlx5_vdpa_create_mr()
539 		destroy_user_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
541 		destroy_dma_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
546 int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)  in mlx5_vdpa_create_mr()  argument
550 	mutex_lock(&mvdev->mr.mkey_mtx);  in mlx5_vdpa_create_mr()
551 	err = _mlx5_vdpa_create_mr(mvdev, iotlb);  in mlx5_vdpa_create_mr()
552 	mutex_unlock(&mvdev->mr.mkey_mtx);  in mlx5_vdpa_create_mr()
556 int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,  in mlx5_vdpa_handle_set_map()  argument
559 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in mlx5_vdpa_handle_set_map()
565 		mlx5_vdpa_info(mvdev, "memory map update\n");  in mlx5_vdpa_handle_set_map()
569 		err = _mlx5_vdpa_create_mr(mvdev, iotlb);  in mlx5_vdpa_handle_set_map()