48 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)  in hns_roce_check_whether_mhop()  argument
54 		hop_num = hr_dev->caps.qpc_hop_num;  in hns_roce_check_whether_mhop()
57 		hop_num = hr_dev->caps.mpt_hop_num;  in hns_roce_check_whether_mhop()
60 		hop_num = hr_dev->caps.cqc_hop_num;  in hns_roce_check_whether_mhop()
63 		hop_num = hr_dev->caps.srqc_hop_num;  in hns_roce_check_whether_mhop()
66 		hop_num = hr_dev->caps.sccc_hop_num;  in hns_roce_check_whether_mhop()
69 		hop_num = hr_dev->caps.qpc_timer_hop_num;  in hns_roce_check_whether_mhop()
72 		hop_num = hr_dev->caps.cqc_timer_hop_num;  in hns_roce_check_whether_mhop()
75 		hop_num = hr_dev->caps.gmv_hop_num;  in hns_roce_check_whether_mhop()
122 static int get_hem_table_config(struct hns_roce_dev *hr_dev,  in get_hem_table_config()  argument
126 	struct device *dev = hr_dev->dev;  in get_hem_table_config()
130 		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz  in get_hem_table_config()
132 		mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz  in get_hem_table_config()
134 		mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;  in get_hem_table_config()
135 		mhop->hop_num = hr_dev->caps.qpc_hop_num;  in get_hem_table_config()
138 		mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz  in get_hem_table_config()
140 		mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz  in get_hem_table_config()
142 		mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;  in get_hem_table_config()
143 		mhop->hop_num = hr_dev->caps.mpt_hop_num;  in get_hem_table_config()
146 		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz  in get_hem_table_config()
148 		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz  in get_hem_table_config()
150 		mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;  in get_hem_table_config()
151 		mhop->hop_num = hr_dev->caps.cqc_hop_num;  in get_hem_table_config()
154 		mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz  in get_hem_table_config()
156 		mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz  in get_hem_table_config()
158 		mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;  in get_hem_table_config()
159 		mhop->hop_num = hr_dev->caps.sccc_hop_num;  in get_hem_table_config()
162 		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz  in get_hem_table_config()
164 		mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz  in get_hem_table_config()
166 		mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;  in get_hem_table_config()
167 		mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;  in get_hem_table_config()
170 		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz  in get_hem_table_config()
172 		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz  in get_hem_table_config()
174 		mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;  in get_hem_table_config()
175 		mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;  in get_hem_table_config()
178 		mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz  in get_hem_table_config()
180 		mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz  in get_hem_table_config()
182 		mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;  in get_hem_table_config()
183 		mhop->hop_num = hr_dev->caps.srqc_hop_num;  in get_hem_table_config()
186 		mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +  in get_hem_table_config()
188 		mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +  in get_hem_table_config()
190 		mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;  in get_hem_table_config()
191 		mhop->hop_num = hr_dev->caps.gmv_hop_num;  in get_hem_table_config()
202 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,  in hns_roce_calc_hem_mhop()  argument
206 	struct device *dev = hr_dev->dev;  in hns_roce_calc_hem_mhop()
212 	if (get_hem_table_config(hr_dev, mhop, table->type))  in hns_roce_calc_hem_mhop()
251 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,  in hns_roce_alloc_hem()  argument
295 		buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,  in hns_roce_alloc_hem()
311 	hns_roce_free_hem(hr_dev, hem);  in hns_roce_alloc_hem()
315 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)  in hns_roce_free_hem()  argument
325 			dma_free_coherent(hr_dev->dev,  in hns_roce_free_hem()
335 static int calc_hem_config(struct hns_roce_dev *hr_dev,  in calc_hem_config()  argument
340 	struct ib_device *ibdev = &hr_dev->ib_dev;  in calc_hem_config()
347 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);  in calc_hem_config()
385 static void free_mhop_hem(struct hns_roce_dev *hr_dev,  in free_mhop_hem()  argument
391 	struct device *dev = hr_dev->dev;  in free_mhop_hem()
394 		hns_roce_free_hem(hr_dev, table->hem[index->buf]);  in free_mhop_hem()
411 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,  in alloc_mhop_hem()  argument
417 	struct device *dev = hr_dev->dev;  in alloc_mhop_hem()
459 	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,  in alloc_mhop_hem()
480 	free_mhop_hem(hr_dev, table, mhop, index);  in alloc_mhop_hem()
485 static int set_mhop_hem(struct hns_roce_dev *hr_dev,  in set_mhop_hem()  argument
490 	struct ib_device *ibdev = &hr_dev->ib_dev;  in set_mhop_hem()
495 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);  in set_mhop_hem()
503 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);  in set_mhop_hem()
515 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);  in set_mhop_hem()
523 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,  in hns_roce_table_mhop_get()  argument
527 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_table_mhop_get()
532 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
544 	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
552 		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
563 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
569 int hns_roce_table_get(struct hns_roce_dev *hr_dev,  in hns_roce_table_get()  argument
572 	struct device *dev = hr_dev->dev;  in hns_roce_table_get()
576 	if (hns_roce_check_whether_mhop(hr_dev, table->type))  in hns_roce_table_get()
577 		return hns_roce_table_mhop_get(hr_dev, table, obj);  in hns_roce_table_get()
588 	table->hem[i] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_get()
598 	ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_get()
600 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_get()
613 static void clear_mhop_hem(struct hns_roce_dev *hr_dev,  in clear_mhop_hem()  argument
618 	struct ib_device *ibdev = &hr_dev->ib_dev;  in clear_mhop_hem()
646 		ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx);  in clear_mhop_hem()
652 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1);  in clear_mhop_hem()
659 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in clear_mhop_hem()
667 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,  in hns_roce_table_mhop_put()  argument
672 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_table_mhop_put()
677 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
689 	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
690 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_put()
695 void hns_roce_table_put(struct hns_roce_dev *hr_dev,  in hns_roce_table_put()  argument
698 	struct device *dev = hr_dev->dev;  in hns_roce_table_put()
702 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put()
703 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);  in hns_roce_table_put()
713 	ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_put()
718 	hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_put()
724 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,  in hns_roce_table_find()  argument
742 	if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_find()
750 		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))  in hns_roce_table_find()
793 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,  in hns_roce_init_hem_table()  argument
800 	if (!hns_roce_check_whether_mhop(hr_dev, type)) {  in hns_roce_init_hem_table()
801 		table->table_chunk_size = hr_dev->caps.chunk_sz;  in hns_roce_init_hem_table()
816 		if (get_hem_table_config(hr_dev, &mhop, type))  in hns_roce_init_hem_table()
896 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,  in hns_roce_cleanup_mhop_hem_table()  argument
904 	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_cleanup_mhop_hem_table()
912 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);  in hns_roce_cleanup_mhop_hem_table()
927 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,  in hns_roce_cleanup_hem_table()  argument
930 	struct device *dev = hr_dev->dev;  in hns_roce_cleanup_hem_table()
935 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_cleanup_hem_table()
936 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);  in hns_roce_cleanup_hem_table()
943 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in hns_roce_cleanup_hem_table()
948 			hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_cleanup_hem_table()
954 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)  in hns_roce_cleanup_hem()  argument
956 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)  in hns_roce_cleanup_hem()
957 		hns_roce_cleanup_hem_table(hr_dev,  in hns_roce_cleanup_hem()
958 					   &hr_dev->srq_table.table);  in hns_roce_cleanup_hem()
959 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);  in hns_roce_cleanup_hem()
960 	if (hr_dev->caps.qpc_timer_entry_sz)  in hns_roce_cleanup_hem()
961 		hns_roce_cleanup_hem_table(hr_dev,  in hns_roce_cleanup_hem()
962 					   &hr_dev->qpc_timer_table);  in hns_roce_cleanup_hem()
963 	if (hr_dev->caps.cqc_timer_entry_sz)  in hns_roce_cleanup_hem()
964 		hns_roce_cleanup_hem_table(hr_dev,  in hns_roce_cleanup_hem()
965 					   &hr_dev->cqc_timer_table);  in hns_roce_cleanup_hem()
966 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)  in hns_roce_cleanup_hem()
967 		hns_roce_cleanup_hem_table(hr_dev,  in hns_roce_cleanup_hem()
968 					   &hr_dev->qp_table.sccc_table);  in hns_roce_cleanup_hem()
969 	if (hr_dev->caps.trrl_entry_sz)  in hns_roce_cleanup_hem()
970 		hns_roce_cleanup_hem_table(hr_dev,  in hns_roce_cleanup_hem()
971 					   &hr_dev->qp_table.trrl_table);  in hns_roce_cleanup_hem()
973 	if (hr_dev->caps.gmv_entry_sz)  in hns_roce_cleanup_hem()
974 		hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);  in hns_roce_cleanup_hem()
976 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);  in hns_roce_cleanup_hem()
977 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);  in hns_roce_cleanup_hem()
978 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);  in hns_roce_cleanup_hem()
999 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,  in hem_list_alloc_item()  argument
1009 		hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,  in hem_list_alloc_item()
1026 static void hem_list_free_item(struct hns_roce_dev *hr_dev,  in hem_list_free_item()  argument
1030 		dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,  in hem_list_free_item()
1035 static void hem_list_free_all(struct hns_roce_dev *hr_dev,  in hem_list_free_all()  argument
1042 		hem_list_free_item(hr_dev, hem, exist_bt);  in hem_list_free_all()
1046 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,  in hem_list_link_bt()  argument
1053 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,  in hem_list_assign_bt()  argument
1156 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,  in hem_list_alloc_mid_bt()  argument
1177 		dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);  in hem_list_alloc_mid_bt()
1182 		dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",  in hem_list_alloc_mid_bt()
1208 		cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,  in hem_list_alloc_mid_bt()
1223 			hem_list_link_bt(hr_dev, pre->addr + step,  in hem_list_alloc_mid_bt()
1236 		hem_list_free_all(hr_dev, &temp_list[level], true);  in hem_list_alloc_mid_bt()
1242 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,  in alloc_root_hem()  argument
1260 	hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,  in alloc_root_hem()
1270 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,  in alloc_fake_root_bt()  argument
1277 	hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,  in alloc_fake_root_bt()
1282 	hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);  in alloc_fake_root_bt()
1289 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,  in setup_middle_bt()  argument
1305 		hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);  in setup_middle_bt()
1313 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,  in setup_root_hem()  argument
1343 			ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,  in setup_root_hem()
1346 			ret = setup_middle_bt(hr_dev, cpu_base, unit, r,  in setup_root_hem()
1363 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,  in hem_list_alloc_root_bt()  argument
1379 	root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,  in hem_list_alloc_root_bt()
1392 	ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,  in hem_list_alloc_root_bt()
1396 			hem_list_free_all(hr_dev, &head.branch[i], false);  in hem_list_alloc_root_bt()
1398 		hem_list_free_all(hr_dev, &head.root, true);  in hem_list_alloc_root_bt()
1405 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,  in hns_roce_hem_list_request()  argument
1417 		dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",  in hns_roce_hem_list_request()
1430 			ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,  in hns_roce_hem_list_request()
1434 				dev_err(hr_dev->dev,  in hns_roce_hem_list_request()
1441 	ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,  in hns_roce_hem_list_request()
1444 		dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret);  in hns_roce_hem_list_request()
1449 	hns_roce_hem_list_release(hr_dev, hem_list);  in hns_roce_hem_list_request()
1454 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,  in hns_roce_hem_list_release()  argument
1461 			hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],  in hns_roce_hem_list_release()
1464 	hem_list_free_all(hr_dev, &hem_list->root_bt, true);  in hns_roce_hem_list_release()
1480 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,  in hns_roce_hem_list_find_mtt()  argument