487 	struct hns_roce_mtr *mtr = &mr->pbl_mtr;  in hns_roce_map_mr_sg()  local
503 	mtr->hem_cfg.region[0].offset = 0;  in hns_roce_map_mr_sg()
504 	mtr->hem_cfg.region[0].count = mr->npages;  in hns_roce_map_mr_sg()
505 	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;  in hns_roce_map_mr_sg()
506 	mtr->hem_cfg.region_count = 1;  in hns_roce_map_mr_sg()
507 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
632 static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_region()  argument
651 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in mtr_map_region()
724 static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_bufs()  argument
727 	if (mtr->umem) {  in mtr_free_bufs()
728 		ib_umem_release(mtr->umem);  in mtr_free_bufs()
729 		mtr->umem = NULL;  in mtr_free_bufs()
733 	if (mtr->kmem) {  in mtr_free_bufs()
734 		hns_roce_buf_free(hr_dev, mtr->kmem);  in mtr_free_bufs()
735 		kfree(mtr->kmem);  in mtr_free_bufs()
736 		mtr->kmem = NULL;  in mtr_free_bufs()
740 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_bufs()  argument
761 		mtr->kmem = NULL;  in mtr_alloc_bufs()
762 		mtr->umem = ib_umem_get(ibdev, user_addr, total_size,  in mtr_alloc_bufs()
764 		if (IS_ERR_OR_NULL(mtr->umem)) {  in mtr_alloc_bufs()
766 				  PTR_ERR(mtr->umem));  in mtr_alloc_bufs()
774 		page_size = ib_umem_find_best_pgsz(mtr->umem, pgsz_bitmap,  in mtr_alloc_bufs()
779 		all_pg_count = ib_umem_num_dma_blocks(mtr->umem, page_size);  in mtr_alloc_bufs()
782 		mtr->umem = NULL;  in mtr_alloc_bufs()
783 		mtr->kmem = kzalloc(sizeof(*mtr->kmem), GFP_KERNEL);  in mtr_alloc_bufs()
784 		if (!mtr->kmem) {  in mtr_alloc_bufs()
791 					 mtr->kmem, buf_attr->page_shift);  in mtr_alloc_bufs()
797 		all_pg_count = mtr->kmem->npages;  in mtr_alloc_bufs()
808 	mtr->hem_cfg.buf_pg_shift = best_pg_shift;  in mtr_alloc_bufs()
809 	mtr->hem_cfg.buf_pg_count = all_pg_count;  in mtr_alloc_bufs()
813 	mtr_free_bufs(hr_dev, mtr);  in mtr_alloc_bufs()
817 static int mtr_get_pages(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_get_pages()  argument
824 	if (mtr->umem)  in mtr_get_pages()
826 					       mtr->umem, page_shift);  in mtr_get_pages()
829 					       mtr->kmem);  in mtr_get_pages()
831 	if (mtr->hem_cfg.is_direct && npage > 1) {  in mtr_get_pages()
835 				  mtr->umem ? "user" : "kernel", err);  in mtr_get_pages()
843 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_map()  argument
855 	if (mtr->hem_cfg.is_direct) {  in hns_roce_mtr_map()
856 		mtr->hem_cfg.root_ba = pages[0];  in hns_roce_mtr_map()
860 	for (i = 0; i < mtr->hem_cfg.region_count; i++) {  in hns_roce_mtr_map()
861 		r = &mtr->hem_cfg.region[i];  in hns_roce_mtr_map()
870 		err = mtr_map_region(hr_dev, mtr, &pages[r->offset], r);  in hns_roce_mtr_map()
882 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_find()  argument
885 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in hns_roce_mtr_find()
922 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in hns_roce_mtr_find()
1012 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_create()  argument
1017 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in hns_roce_mtr_create()
1033 		mtr->umem = NULL;  in hns_roce_mtr_create()
1034 		mtr->kmem = NULL;  in hns_roce_mtr_create()
1036 		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, cfg->is_direct,  in hns_roce_mtr_create()
1052 	hns_roce_hem_list_init(&mtr->hem_list);  in hns_roce_mtr_create()
1054 		ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,  in hns_roce_mtr_create()
1062 		cfg->root_ba = mtr->hem_list.root_ba;  in hns_roce_mtr_create()
1081 	get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt,  in hns_roce_mtr_create()
1091 	ret = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);  in hns_roce_mtr_create()
1103 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in hns_roce_mtr_create()
1105 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_create()
1109 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in hns_roce_mtr_destroy()  argument
1112 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in hns_roce_mtr_destroy()
1115 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_destroy()