219 		dev_err(dev, "Table %d not support multi-hop addressing!\n",  in get_hem_table_config()
228 			   struct hns_roce_hem_table *table, unsigned long *obj,  in hns_roce_calc_hem_mhop()  argument
237 	if (get_hem_table_config(hr_dev, mhop, table->type))  in hns_roce_calc_hem_mhop()
247 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
249 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :  in hns_roce_calc_hem_mhop()
251 	table_idx = (*obj & (table->num_obj - 1)) /  in hns_roce_calc_hem_mhop()
252 		     (chunk_size / table->obj_size);  in hns_roce_calc_hem_mhop()
267 		dev_err(dev, "Table %d not support hop_num = %d!\n",  in hns_roce_calc_hem_mhop()
268 			     table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
363 			    struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_set_hem()  argument
378 	unsigned long i = (obj & (table->num_obj - 1)) /  in hns_roce_set_hem()
379 			  (table->table_chunk_size / table->obj_size);  in hns_roce_set_hem()
381 	switch (table->type) {  in hns_roce_set_hem()
387 			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);  in hns_roce_set_hem()
399 	for (hns_roce_hem_first(table->hem[i], &iter);  in hns_roce_set_hem()
438 				   struct hns_roce_hem_table *table,  in hns_roce_table_mhop_get()  argument
460 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);  in hns_roce_table_mhop_get()
469 	bt_num = hns_roce_get_bt_num(table->type, hop_num);  in hns_roce_table_mhop_get()
485 		dev_err(dev, "Table %d not support hop_num = %d!\n",  in hns_roce_table_mhop_get()
486 			     table->type, hop_num);  in hns_roce_table_mhop_get()
490 	if (unlikely(hem_idx >= table->num_hem)) {  in hns_roce_table_mhop_get()
491 		dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",  in hns_roce_table_mhop_get()
492 			     table->type, hem_idx, table->num_hem);  in hns_roce_table_mhop_get()
496 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_get()
498 	if (table->hem[hem_idx]) {  in hns_roce_table_mhop_get()
499 		++table->hem[hem_idx]->refcount;  in hns_roce_table_mhop_get()
504 	if ((check_whether_bt_num_3(table->type, hop_num) ||  in hns_roce_table_mhop_get()
505 		check_whether_bt_num_2(table->type, hop_num)) &&  in hns_roce_table_mhop_get()
506 		!table->bt_l0[bt_l0_idx]) {  in hns_roce_table_mhop_get()
507 		table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,  in hns_roce_table_mhop_get()
508 					    &(table->bt_l0_dma_addr[bt_l0_idx]),  in hns_roce_table_mhop_get()
510 		if (!table->bt_l0[bt_l0_idx]) {  in hns_roce_table_mhop_get()
517 		if (table->type < HEM_TYPE_MTT) {  in hns_roce_table_mhop_get()
519 			if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {  in hns_roce_table_mhop_get()
528 	if (check_whether_bt_num_3(table->type, hop_num) &&  in hns_roce_table_mhop_get()
529 	    !table->bt_l1[bt_l1_idx])  {  in hns_roce_table_mhop_get()
530 		table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,  in hns_roce_table_mhop_get()
531 					    &(table->bt_l1_dma_addr[bt_l1_idx]),  in hns_roce_table_mhop_get()
533 		if (!table->bt_l1[bt_l1_idx]) {  in hns_roce_table_mhop_get()
538 		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =  in hns_roce_table_mhop_get()
539 					       table->bt_l1_dma_addr[bt_l1_idx];  in hns_roce_table_mhop_get()
543 		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {  in hns_roce_table_mhop_get()
554 	size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;  in hns_roce_table_mhop_get()
555 	table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_mhop_get()
558 						(table->lowmem ? GFP_KERNEL :  in hns_roce_table_mhop_get()
560 	if (!table->hem[hem_idx]) {  in hns_roce_table_mhop_get()
565 	hns_roce_hem_first(table->hem[hem_idx], &iter);  in hns_roce_table_mhop_get()
568 	if (table->type < HEM_TYPE_MTT) {  in hns_roce_table_mhop_get()
570 			*(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;  in hns_roce_table_mhop_get()
573 			*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;  in hns_roce_table_mhop_get()
583 		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {  in hns_roce_table_mhop_get()
589 		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;  in hns_roce_table_mhop_get()
592 	++table->hem[hem_idx]->refcount;  in hns_roce_table_mhop_get()
597 		dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],  in hns_roce_table_mhop_get()
598 				  table->bt_l1_dma_addr[bt_l1_idx]);  in hns_roce_table_mhop_get()
599 		table->bt_l1[bt_l1_idx] = NULL;  in hns_roce_table_mhop_get()
604 		dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],  in hns_roce_table_mhop_get()
605 				  table->bt_l0_dma_addr[bt_l0_idx]);  in hns_roce_table_mhop_get()
606 		table->bt_l0[bt_l0_idx] = NULL;  in hns_roce_table_mhop_get()
610 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_get()
615 		       struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_get()  argument
621 	if (hns_roce_check_whether_mhop(hr_dev, table->type))  in hns_roce_table_get()
622 		return hns_roce_table_mhop_get(hr_dev, table, obj);  in hns_roce_table_get()
624 	i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /  in hns_roce_table_get()
625 	     table->obj_size);  in hns_roce_table_get()
627 	mutex_lock(&table->mutex);  in hns_roce_table_get()
629 	if (table->hem[i]) {  in hns_roce_table_get()
630 		++table->hem[i]->refcount;  in hns_roce_table_get()
634 	table->hem[i] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_get()
635 				       table->table_chunk_size >> PAGE_SHIFT,  in hns_roce_table_get()
636 				       table->table_chunk_size,  in hns_roce_table_get()
637 				       (table->lowmem ? GFP_KERNEL :  in hns_roce_table_get()
639 	if (!table->hem[i]) {  in hns_roce_table_get()
645 	if (hns_roce_set_hem(hr_dev, table, obj)) {  in hns_roce_table_get()
646 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_get()
647 		table->hem[i] = NULL;  in hns_roce_table_get()
653 	++table->hem[i]->refcount;  in hns_roce_table_get()
655 	mutex_unlock(&table->mutex);  in hns_roce_table_get()
660 				    struct hns_roce_hem_table *table,  in hns_roce_table_mhop_put()  argument
676 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);  in hns_roce_table_mhop_put()
684 	bt_num = hns_roce_get_bt_num(table->type, hop_num);  in hns_roce_table_mhop_put()
698 		dev_err(dev, "Table %d not support hop_num = %d!\n",  in hns_roce_table_mhop_put()
699 			     table->type, hop_num);  in hns_roce_table_mhop_put()
703 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_put()
705 	if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {  in hns_roce_table_mhop_put()
706 		mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
710 	if (table->type < HEM_TYPE_MTT && hop_num == 1) {  in hns_roce_table_mhop_put()
711 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))  in hns_roce_table_mhop_put()
713 	} else if (table->type < HEM_TYPE_MTT && hop_num == 2) {  in hns_roce_table_mhop_put()
714 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))  in hns_roce_table_mhop_put()
716 	} else if (table->type < HEM_TYPE_MTT &&  in hns_roce_table_mhop_put()
718 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))  in hns_roce_table_mhop_put()
726 	hns_roce_free_hem(hr_dev, table->hem[hem_idx]);  in hns_roce_table_mhop_put()
727 	table->hem[hem_idx] = NULL;  in hns_roce_table_mhop_put()
729 	if (check_whether_bt_num_2(table->type, hop_num)) {  in hns_roce_table_mhop_put()
731 		if (hns_roce_check_hem_null(table->hem, start_idx,  in hns_roce_table_mhop_put()
732 					    chunk_ba_num, table->num_hem)) {  in hns_roce_table_mhop_put()
733 			if (table->type < HEM_TYPE_MTT &&  in hns_roce_table_mhop_put()
734 			    hr_dev->hw->clear_hem(hr_dev, table, obj, 0))  in hns_roce_table_mhop_put()
738 					  table->bt_l0[mhop.l0_idx],  in hns_roce_table_mhop_put()
739 					  table->bt_l0_dma_addr[mhop.l0_idx]);  in hns_roce_table_mhop_put()
740 			table->bt_l0[mhop.l0_idx] = NULL;  in hns_roce_table_mhop_put()
742 	} else if (check_whether_bt_num_3(table->type, hop_num)) {  in hns_roce_table_mhop_put()
745 		if (hns_roce_check_hem_null(table->hem, start_idx,  in hns_roce_table_mhop_put()
746 					    chunk_ba_num, table->num_hem)) {  in hns_roce_table_mhop_put()
747 			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))  in hns_roce_table_mhop_put()
751 					  table->bt_l1[bt_l1_idx],  in hns_roce_table_mhop_put()
752 					  table->bt_l1_dma_addr[bt_l1_idx]);  in hns_roce_table_mhop_put()
753 			table->bt_l1[bt_l1_idx] = NULL;  in hns_roce_table_mhop_put()
756 			if (hns_roce_check_bt_null(table->bt_l1, start_idx,  in hns_roce_table_mhop_put()
758 				if (hr_dev->hw->clear_hem(hr_dev, table, obj,  in hns_roce_table_mhop_put()
763 					    table->bt_l0[mhop.l0_idx],  in hns_roce_table_mhop_put()
764 					    table->bt_l0_dma_addr[mhop.l0_idx]);  in hns_roce_table_mhop_put()
765 				table->bt_l0[mhop.l0_idx] = NULL;  in hns_roce_table_mhop_put()
770 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
774 			struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_put()  argument
779 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put()
780 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);  in hns_roce_table_put()
784 	i = (obj & (table->num_obj - 1)) /  in hns_roce_table_put()
785 	    (table->table_chunk_size / table->obj_size);  in hns_roce_table_put()
787 	mutex_lock(&table->mutex);  in hns_roce_table_put()
789 	if (--table->hem[i]->refcount == 0) {  in hns_roce_table_put()
791 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))  in hns_roce_table_put()
794 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_put()
795 		table->hem[i] = NULL;  in hns_roce_table_put()
798 	mutex_unlock(&table->mutex);  in hns_roce_table_put()
802 			  struct hns_roce_hem_table *table,  in hns_roce_table_find()  argument
817 	if (!table->lowmem)  in hns_roce_table_find()
820 	mutex_lock(&table->mutex);  in hns_roce_table_find()
822 	if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_find()
823 		obj_per_chunk = table->table_chunk_size / table->obj_size;  in hns_roce_table_find()
824 		hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];  in hns_roce_table_find()
825 		idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;  in hns_roce_table_find()
826 		dma_offset = offset = idx_offset * table->obj_size;  in hns_roce_table_find()
830 		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))  in hns_roce_table_find()
841 		hem = table->hem[hem_idx];  in hns_roce_table_find()
842 		dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %  in hns_roce_table_find()
870 	mutex_unlock(&table->mutex);  in hns_roce_table_find()
875 			     struct hns_roce_hem_table *table,  in hns_roce_table_get_range()  argument
879 	unsigned long inc = table->table_chunk_size / table->obj_size;  in hns_roce_table_get_range()
883 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_get_range()
884 		ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);  in hns_roce_table_get_range()
887 		inc = mhop.bt_chunk_size / table->obj_size;  in hns_roce_table_get_range()
892 		ret = hns_roce_table_get(hr_dev, table, i);  in hns_roce_table_get_range()
902 		hns_roce_table_put(hr_dev, table, i);  in hns_roce_table_get_range()
908 			      struct hns_roce_hem_table *table,  in hns_roce_table_put_range()  argument
912 	unsigned long inc = table->table_chunk_size / table->obj_size;  in hns_roce_table_put_range()
915 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put_range()
916 		if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_table_put_range()
918 		inc = mhop.bt_chunk_size / table->obj_size;  in hns_roce_table_put_range()
922 		hns_roce_table_put(hr_dev, table, i);  in hns_roce_table_put_range()
926 			    struct hns_roce_hem_table *table, u32 type,  in hns_roce_init_hem_table()  argument
934 		table->table_chunk_size = hr_dev->caps.chunk_sz;  in hns_roce_init_hem_table()
935 		obj_per_chunk = table->table_chunk_size / obj_size;  in hns_roce_init_hem_table()
938 		table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);  in hns_roce_init_hem_table()
939 		if (!table->hem)  in hns_roce_init_hem_table()
963 		table->hem = kcalloc(num_hem, sizeof(*table->hem),  in hns_roce_init_hem_table()
965 		if (!table->hem)  in hns_roce_init_hem_table()
973 			table->bt_l1 = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
974 					       sizeof(*table->bt_l1),  in hns_roce_init_hem_table()
976 			if (!table->bt_l1)  in hns_roce_init_hem_table()
979 			table->bt_l1_dma_addr = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
980 						 sizeof(*table->bt_l1_dma_addr),  in hns_roce_init_hem_table()
983 			if (!table->bt_l1_dma_addr)  in hns_roce_init_hem_table()
989 			table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),  in hns_roce_init_hem_table()
991 			if (!table->bt_l0)  in hns_roce_init_hem_table()
994 			table->bt_l0_dma_addr = kcalloc(num_bt_l0,  in hns_roce_init_hem_table()
995 						 sizeof(*table->bt_l0_dma_addr),  in hns_roce_init_hem_table()
997 			if (!table->bt_l0_dma_addr)  in hns_roce_init_hem_table()
1002 	table->type = type;  in hns_roce_init_hem_table()
1003 	table->num_hem = num_hem;  in hns_roce_init_hem_table()
1004 	table->num_obj = nobj;  in hns_roce_init_hem_table()
1005 	table->obj_size = obj_size;  in hns_roce_init_hem_table()
1006 	table->lowmem = use_lowmem;  in hns_roce_init_hem_table()
1007 	mutex_init(&table->mutex);  in hns_roce_init_hem_table()
1012 	kfree(table->bt_l0);  in hns_roce_init_hem_table()
1013 	table->bt_l0 = NULL;  in hns_roce_init_hem_table()
1016 	kfree(table->bt_l1_dma_addr);  in hns_roce_init_hem_table()
1017 	table->bt_l1_dma_addr = NULL;  in hns_roce_init_hem_table()
1020 	kfree(table->bt_l1);  in hns_roce_init_hem_table()
1021 	table->bt_l1 = NULL;  in hns_roce_init_hem_table()
1024 	kfree(table->hem);  in hns_roce_init_hem_table()
1025 	table->hem = NULL;  in hns_roce_init_hem_table()
1032 					    struct hns_roce_hem_table *table)  in hns_roce_cleanup_mhop_hem_table()  argument
1039 	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_cleanup_mhop_hem_table()
1041 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :  in hns_roce_cleanup_mhop_hem_table()
1044 	for (i = 0; i < table->num_hem; ++i) {  in hns_roce_cleanup_mhop_hem_table()
1045 		obj = i * buf_chunk_size / table->obj_size;  in hns_roce_cleanup_mhop_hem_table()
1046 		if (table->hem[i])  in hns_roce_cleanup_mhop_hem_table()
1047 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);  in hns_roce_cleanup_mhop_hem_table()
1050 	kfree(table->hem);  in hns_roce_cleanup_mhop_hem_table()
1051 	table->hem = NULL;  in hns_roce_cleanup_mhop_hem_table()
1052 	kfree(table->bt_l1);  in hns_roce_cleanup_mhop_hem_table()
1053 	table->bt_l1 = NULL;  in hns_roce_cleanup_mhop_hem_table()
1054 	kfree(table->bt_l1_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
1055 	table->bt_l1_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
1056 	kfree(table->bt_l0);  in hns_roce_cleanup_mhop_hem_table()
1057 	table->bt_l0 = NULL;  in hns_roce_cleanup_mhop_hem_table()
1058 	kfree(table->bt_l0_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
1059 	table->bt_l0_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
1063 				struct hns_roce_hem_table *table)  in hns_roce_cleanup_hem_table()  argument
1068 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_cleanup_hem_table()
1069 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);  in hns_roce_cleanup_hem_table()
1073 	for (i = 0; i < table->num_hem; ++i)  in hns_roce_cleanup_hem_table()
1074 		if (table->hem[i]) {  in hns_roce_cleanup_hem_table()
1075 			if (hr_dev->hw->clear_hem(hr_dev, table,  in hns_roce_cleanup_hem_table()
1076 			    i * table->table_chunk_size / table->obj_size, 0))  in hns_roce_cleanup_hem_table()
1079 			hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_cleanup_hem_table()
1082 	kfree(table->hem);  in hns_roce_cleanup_hem_table()
1095 					   &hr_dev->srq_table.table);  in hns_roce_cleanup_hem()
1096 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);  in hns_roce_cleanup_hem()
1184 /* assign L0 table address to hem from root bt */
1218 	 * hopnum    base address table levels  in hem_list_is_bottom_bt()
1230  * @bt_level: base address table level
1470 /* construct the base address table and link them by address hop config */