52 static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in create_direct_mr()  argument
64 	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);  in create_direct_mr()
70 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_direct_mr()
79 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mr, in, inlen);  in create_direct_mr()
82 		mlx5_vdpa_warn(mvdev, "Failed to create direct MR\n");  in create_direct_mr()
89 static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in destroy_direct_mr()  argument
91 	mlx5_vdpa_destroy_mkey(mvdev, mr->mr);  in destroy_direct_mr()
145 static void fill_indir(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey, void *in)  in fill_indir()  argument
169 			klm->key = cpu_to_be32(mvdev->res.null_mkey);  in fill_indir()
182 static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_indirect_key()  argument
201 	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);  in create_indirect_key()
207 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_indirect_key()
212 	fill_indir(mvdev, mr, in);  in create_indirect_key()
213 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_indirect_key()
218 static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)  in destroy_indirect_key()  argument
220 	mlx5_vdpa_destroy_mkey(mvdev, mkey->mkey);  in destroy_indirect_key()
223 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,  in map_direct_mr()  argument
238 	struct device *dma = mvdev->vdev.dma_dev;  in map_direct_mr()
262 				mlx5_vdpa_warn(mvdev, "sg null. start 0x%llx, end 0x%llx\n",  in map_direct_mr()
282 	err = create_direct_mr(mvdev, mr);  in map_direct_mr()
295 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in unmap_direct_mr()  argument
297 	struct device *dma = mvdev->vdev.dma_dev;  in unmap_direct_mr()
299 	destroy_direct_mr(mvdev, mr);  in unmap_direct_mr()
304 static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, u64 start, u64 size, u8 perm,  in add_direct_chain()  argument
307 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in add_direct_chain()
327 		err = map_direct_mr(mvdev, dmr, iotlb);  in add_direct_chain()
345 		unmap_direct_mr(mvdev, dmr);  in add_direct_chain()
357 static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)  in create_user_mr()  argument
359 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in create_user_mr()
387 				err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);  in create_user_mr()
396 	err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);  in create_user_mr()
404 	err = create_indirect_key(mvdev, mr);  in create_user_mr()
414 		unmap_direct_mr(mvdev, dmr);  in create_user_mr()
420 static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_dma_mr()  argument
437 	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);  in create_dma_mr()
440 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_dma_mr()
448 static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_dma_mr()  argument
450 	mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);  in destroy_dma_mr()
453 static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)  in dup_iotlb()  argument
460 		err = vhost_iotlb_add_range(mvdev->cvq.iotlb, start, last, start, VHOST_ACCESS_RW);  in dup_iotlb()
466 		err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, map->last,  in dup_iotlb()
474 static void prune_iotlb(struct mlx5_vdpa_dev *mvdev)  in prune_iotlb()  argument
476 	vhost_iotlb_del_range(mvdev->cvq.iotlb, 0, ULLONG_MAX);  in prune_iotlb()
479 static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_user_mr()  argument
484 	destroy_indirect_key(mvdev, mr);  in destroy_user_mr()
487 		unmap_direct_mr(mvdev, dmr);  in destroy_user_mr()
492 static void _mlx5_vdpa_destroy_cvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)  in _mlx5_vdpa_destroy_cvq_mr()  argument
494 	if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)  in _mlx5_vdpa_destroy_cvq_mr()
497 	prune_iotlb(mvdev);  in _mlx5_vdpa_destroy_cvq_mr()
500 static void _mlx5_vdpa_destroy_dvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)  in _mlx5_vdpa_destroy_dvq_mr()  argument
502 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in _mlx5_vdpa_destroy_dvq_mr()
504 	if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)  in _mlx5_vdpa_destroy_dvq_mr()
511 		destroy_user_mr(mvdev, mr);  in _mlx5_vdpa_destroy_dvq_mr()
513 		destroy_dma_mr(mvdev, mr);  in _mlx5_vdpa_destroy_dvq_mr()
518 void mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev *mvdev, unsigned int asid)  in mlx5_vdpa_destroy_mr_asid()  argument
520 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in mlx5_vdpa_destroy_mr_asid()
524 	_mlx5_vdpa_destroy_dvq_mr(mvdev, asid);  in mlx5_vdpa_destroy_mr_asid()
525 	_mlx5_vdpa_destroy_cvq_mr(mvdev, asid);  in mlx5_vdpa_destroy_mr_asid()
530 void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev)  in mlx5_vdpa_destroy_mr()  argument
532 	mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_CVQ_GROUP]);  in mlx5_vdpa_destroy_mr()
533 	mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]);  in mlx5_vdpa_destroy_mr()
536 static int _mlx5_vdpa_create_cvq_mr(struct mlx5_vdpa_dev *mvdev,  in _mlx5_vdpa_create_cvq_mr()  argument
540 	if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)  in _mlx5_vdpa_create_cvq_mr()
543 	return dup_iotlb(mvdev, iotlb);  in _mlx5_vdpa_create_cvq_mr()
546 static int _mlx5_vdpa_create_dvq_mr(struct mlx5_vdpa_dev *mvdev,  in _mlx5_vdpa_create_dvq_mr()  argument
550 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in _mlx5_vdpa_create_dvq_mr()
553 	if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)  in _mlx5_vdpa_create_dvq_mr()
560 		err = create_user_mr(mvdev, iotlb);  in _mlx5_vdpa_create_dvq_mr()
562 		err = create_dma_mr(mvdev, mr);  in _mlx5_vdpa_create_dvq_mr()
572 static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,  in _mlx5_vdpa_create_mr()  argument
577 	err = _mlx5_vdpa_create_dvq_mr(mvdev, iotlb, asid);  in _mlx5_vdpa_create_mr()
581 	err = _mlx5_vdpa_create_cvq_mr(mvdev, iotlb, asid);  in _mlx5_vdpa_create_mr()
588 	_mlx5_vdpa_destroy_dvq_mr(mvdev, asid);  in _mlx5_vdpa_create_mr()
593 int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,  in mlx5_vdpa_create_mr()  argument
598 	mutex_lock(&mvdev->mr.mkey_mtx);  in mlx5_vdpa_create_mr()
599 	err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);  in mlx5_vdpa_create_mr()
600 	mutex_unlock(&mvdev->mr.mkey_mtx);  in mlx5_vdpa_create_mr()
604 int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,  in mlx5_vdpa_handle_set_map()  argument
607 	struct mlx5_vdpa_mr *mr = &mvdev->mr;  in mlx5_vdpa_handle_set_map()
613 		mlx5_vdpa_info(mvdev, "memory map update\n");  in mlx5_vdpa_handle_set_map()
617 		err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);  in mlx5_vdpa_handle_set_map()