51 static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in alloc_mr_key()  argument
53 	struct hns_roce_ida *mtpt_ida = &hr_dev->mr_table.mtpt_ida;  in alloc_mr_key()
54 	struct ib_device *ibdev = &hr_dev->ib_dev;  in alloc_mr_key()
68 	err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table,  in alloc_mr_key()
81 static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_key()  argument
85 	hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, obj);  in free_mr_key()
86 	ida_free(&hr_dev->mr_table.mtpt_ida.ida, (int)obj);  in free_mr_key()
89 static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,  in alloc_mr_pbl()  argument
92 	struct ib_device *ibdev = &hr_dev->ib_dev;  in alloc_mr_pbl()
97 	mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num;  in alloc_mr_pbl()
99 			      hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT;  in alloc_mr_pbl()
107 	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,  in alloc_mr_pbl()
108 				  hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT,  in alloc_mr_pbl()
118 static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_pbl()  argument
120 	hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);  in free_mr_pbl()
123 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in hns_roce_mr_free()  argument
125 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_mr_free()
129 		ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,  in hns_roce_mr_free()
131 					      (hr_dev->caps.num_mtpts - 1));  in hns_roce_mr_free()
137 	free_mr_pbl(hr_dev, mr);  in hns_roce_mr_free()
138 	free_mr_key(hr_dev, mr);  in hns_roce_mr_free()
141 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,  in hns_roce_mr_enable()  argument
146 	struct device *dev = hr_dev->dev;  in hns_roce_mr_enable()
150 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);  in hns_roce_mr_enable()
155 		ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr);  in hns_roce_mr_enable()
157 		ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr);  in hns_roce_mr_enable()
163 	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,  in hns_roce_mr_enable()
164 				     mtpt_idx & (hr_dev->caps.num_mtpts - 1));  in hns_roce_mr_enable()
173 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);  in hns_roce_mr_enable()
178 void hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)  in hns_roce_init_mr_table()  argument
180 	struct hns_roce_ida *mtpt_ida = &hr_dev->mr_table.mtpt_ida;  in hns_roce_init_mr_table()
183 	mtpt_ida->max = hr_dev->caps.num_mtpts - 1;  in hns_roce_init_mr_table()
184 	mtpt_ida->min = hr_dev->caps.reserved_mrws;  in hns_roce_init_mr_table()
189 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);  in hns_roce_get_dma_mr()  local
203 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
207 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_get_dma_mr()
215 	free_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
226 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);  in hns_roce_reg_user_mr()  local
240 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
244 	ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_reg_user_mr()
248 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_reg_user_mr()
257 	free_mr_pbl(hr_dev, mr);  in hns_roce_reg_user_mr()
259 	free_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
270 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);  in hns_roce_rereg_user_mr()  local
271 	struct ib_device *ib_dev = &hr_dev->ib_dev;  in hns_roce_rereg_user_mr()
280 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);  in hns_roce_rereg_user_mr()
284 	mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);  in hns_roce_rereg_user_mr()
286 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,  in hns_roce_rereg_user_mr()
291 	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,  in hns_roce_rereg_user_mr()
307 		free_mr_pbl(hr_dev, mr);  in hns_roce_rereg_user_mr()
308 		ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_rereg_user_mr()
316 	ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, mailbox->buf);  in hns_roce_rereg_user_mr()
322 	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,  in hns_roce_rereg_user_mr()
332 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);  in hns_roce_rereg_user_mr()
341 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);  in hns_roce_dereg_mr()  local
344 	if (hr_dev->hw->dereg_mr)  in hns_roce_dereg_mr()
345 		hr_dev->hw->dereg_mr(hr_dev);  in hns_roce_dereg_mr()
347 	hns_roce_mr_free(hr_dev, mr);  in hns_roce_dereg_mr()
356 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);  in hns_roce_alloc_mr()  local
357 	struct device *dev = hr_dev->dev;  in hns_roce_alloc_mr()
379 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
383 	ret = alloc_mr_pbl(hr_dev, mr, NULL, 0);  in hns_roce_alloc_mr()
387 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_alloc_mr()
397 	free_mr_pbl(hr_dev, mr);  in hns_roce_alloc_mr()
399 	free_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
420 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);  in hns_roce_map_mr_sg()  local
421 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_map_mr_sg()
443 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
459 static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,  in hns_roce_mw_free()  argument
462 	struct device *dev = hr_dev->dev;  in hns_roce_mw_free()
466 		ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,  in hns_roce_mw_free()
468 					      (hr_dev->caps.num_mtpts - 1));  in hns_roce_mw_free()
472 		hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,  in hns_roce_mw_free()
476 	ida_free(&hr_dev->mr_table.mtpt_ida.ida,  in hns_roce_mw_free()
480 static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,  in hns_roce_mw_enable()  argument
483 	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;  in hns_roce_mw_enable()
485 	struct device *dev = hr_dev->dev;  in hns_roce_mw_enable()
490 	ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);  in hns_roce_mw_enable()
494 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);  in hns_roce_mw_enable()
500 	ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw);  in hns_roce_mw_enable()
506 	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,  in hns_roce_mw_enable()
507 				     mtpt_idx & (hr_dev->caps.num_mtpts - 1));  in hns_roce_mw_enable()
515 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);  in hns_roce_mw_enable()
520 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);  in hns_roce_mw_enable()
523 	hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);  in hns_roce_mw_enable()
530 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);  in hns_roce_alloc_mw()  local
531 	struct hns_roce_ida *mtpt_ida = &hr_dev->mr_table.mtpt_ida;  in hns_roce_alloc_mw()
532 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_alloc_mw()
549 	mw->pbl_hop_num = hr_dev->caps.pbl_hop_num;  in hns_roce_alloc_mw()
550 	mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;  in hns_roce_alloc_mw()
551 	mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;  in hns_roce_alloc_mw()
553 	ret = hns_roce_mw_enable(hr_dev, mw);  in hns_roce_alloc_mw()
560 	hns_roce_mw_free(hr_dev, mw);  in hns_roce_alloc_mw()
566 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);  in hns_roce_dealloc_mw()  local
569 	hns_roce_mw_free(hr_dev, mw);  in hns_roce_dealloc_mw()
573 static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_region()  argument
588 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in mtr_map_region()
649 static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_bufs()  argument
659 		hns_roce_buf_free(hr_dev, mtr->kmem);  in mtr_free_bufs()
664 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_bufs()  argument
668 	struct ib_device *ibdev = &hr_dev->ib_dev;  in mtr_alloc_bufs()
684 		mtr->kmem = hns_roce_buf_alloc(hr_dev, total_size,  in mtr_alloc_bufs()
698 static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_bufs()  argument
701 	struct ib_device *ibdev = &hr_dev->ib_dev;  in mtr_map_bufs()
712 		npage = hns_roce_get_umem_bufs(hr_dev, pages, page_count,  in mtr_map_bufs()
715 		npage = hns_roce_get_kmem_bufs(hr_dev, pages, page_count,  in mtr_map_bufs()
735 	ret = hns_roce_mtr_map(hr_dev, mtr, pages, page_count);  in mtr_map_bufs()
745 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_map()  argument
748 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_mtr_map()
779 		ret = mtr_map_region(hr_dev, mtr, r, &pages[r->offset],  in hns_roce_mtr_map()
800 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_find()  argument
836 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in hns_roce_mtr_find()
855 static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,  in mtr_init_buf_cfg()  argument
918 static unsigned int cal_best_bt_pg_sz(struct hns_roce_dev *hr_dev,  in cal_best_bt_pg_sz()  argument
922 	unsigned long cap = hr_dev->caps.page_size_cap;  in cal_best_bt_pg_sz()
951 static int mtr_alloc_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_mtt()  argument
959 		ba_page_shift = cal_best_bt_pg_sz(hr_dev, mtr, ba_page_shift);  in mtr_alloc_mtt()
963 		ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,  in mtr_alloc_mtt()
977 static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_mtt()  argument
979 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in mtr_free_mtt()
992 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_create()  argument
997 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_mtr_create()
1002 	buf_page_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, &mtr->hem_cfg,  in hns_roce_mtr_create()
1011 	ret = mtr_alloc_mtt(hr_dev, mtr, ba_page_shift);  in hns_roce_mtr_create()
1026 	ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);  in hns_roce_mtr_create()
1033 	ret = mtr_map_bufs(hr_dev, mtr, buf_page_cnt, buf_page_shift);  in hns_roce_mtr_create()
1039 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_create()
1041 	mtr_free_mtt(hr_dev, mtr);  in hns_roce_mtr_create()
1045 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in hns_roce_mtr_destroy()  argument
1048 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in hns_roce_mtr_destroy()
1051 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_destroy()