187 		dev_err(dev, "Table %d not support multi-hop addressing!\n",  in get_hem_table_config()
196 			   struct hns_roce_hem_table *table, unsigned long *obj,  in hns_roce_calc_hem_mhop()  argument
205 	if (get_hem_table_config(hr_dev, mhop, table->type))  in hns_roce_calc_hem_mhop()
215 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
217 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :  in hns_roce_calc_hem_mhop()
219 	table_idx = (*obj & (table->num_obj - 1)) /  in hns_roce_calc_hem_mhop()
220 		     (chunk_size / table->obj_size);  in hns_roce_calc_hem_mhop()
235 		dev_err(dev, "Table %d not support hop_num = %d!\n",  in hns_roce_calc_hem_mhop()
236 			     table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
331 			    struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_set_hem()  argument
346 	unsigned long i = (obj & (table->num_obj - 1)) /  in hns_roce_set_hem()
347 			  (table->table_chunk_size / table->obj_size);  in hns_roce_set_hem()
349 	switch (table->type) {  in hns_roce_set_hem()
355 			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);  in hns_roce_set_hem()
367 	for (hns_roce_hem_first(table->hem[i], &iter);  in hns_roce_set_hem()
406 			   struct hns_roce_hem_table *table, unsigned long obj,  in calc_hem_config()  argument
417 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);  in calc_hem_config()
425 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in calc_hem_config()
441 		ibdev_err(ibdev, "Table %d not support mhop.hop_num = %d!\n",  in calc_hem_config()
442 			  table->type, mhop->hop_num);  in calc_hem_config()
446 	if (unlikely(index->buf >= table->num_hem)) {  in calc_hem_config()
447 		ibdev_err(ibdev, "Table %d exceed hem limt idx %llu,max %lu!\n",  in calc_hem_config()
448 			  table->type, index->buf, table->num_hem);  in calc_hem_config()
456 			  struct hns_roce_hem_table *table,  in free_mhop_hem()  argument
464 		hns_roce_free_hem(hr_dev, table->hem[index->buf]);  in free_mhop_hem()
465 		table->hem[index->buf] = NULL;  in free_mhop_hem()
469 		dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],  in free_mhop_hem()
470 				  table->bt_l1_dma_addr[index->l1]);  in free_mhop_hem()
471 		table->bt_l1[index->l1] = NULL;  in free_mhop_hem()
475 		dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],  in free_mhop_hem()
476 				  table->bt_l0_dma_addr[index->l0]);  in free_mhop_hem()
477 		table->bt_l0[index->l0] = NULL;  in free_mhop_hem()
482 			  struct hns_roce_hem_table *table,  in alloc_mhop_hem()  argument
495 	if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||  in alloc_mhop_hem()
496 	     check_whether_bt_num_2(table->type, mhop->hop_num)) &&  in alloc_mhop_hem()
497 	     !table->bt_l0[index->l0]) {  in alloc_mhop_hem()
498 		table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
499 					    &table->bt_l0_dma_addr[index->l0],  in alloc_mhop_hem()
501 		if (!table->bt_l0[index->l0]) {  in alloc_mhop_hem()
509 	if (check_whether_bt_num_3(table->type, mhop->hop_num) &&  in alloc_mhop_hem()
510 	    !table->bt_l1[index->l1])  {  in alloc_mhop_hem()
511 		table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
512 					    &table->bt_l1_dma_addr[index->l1],  in alloc_mhop_hem()
514 		if (!table->bt_l1[index->l1]) {  in alloc_mhop_hem()
519 		*(table->bt_l0[index->l0] + mhop->l1_idx) =  in alloc_mhop_hem()
520 					       table->bt_l1_dma_addr[index->l1];  in alloc_mhop_hem()
527 	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;  in alloc_mhop_hem()
528 	flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;  in alloc_mhop_hem()
529 	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,  in alloc_mhop_hem()
531 	if (!table->hem[index->buf]) {  in alloc_mhop_hem()
537 	hns_roce_hem_first(table->hem[index->buf], &iter);  in alloc_mhop_hem()
539 	if (table->type < HEM_TYPE_MTT) {  in alloc_mhop_hem()
541 			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;  in alloc_mhop_hem()
543 			*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
545 		*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
550 	free_mhop_hem(hr_dev, table, mhop, index);  in alloc_mhop_hem()
556 			struct hns_roce_hem_table *table, unsigned long obj,  in set_mhop_hem()  argument
565 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);  in set_mhop_hem()
573 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);  in set_mhop_hem()
585 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);  in set_mhop_hem()
594 				   struct hns_roce_hem_table *table,  in hns_roce_table_mhop_get()  argument
602 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
608 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_get()
609 	if (table->hem[index.buf]) {  in hns_roce_table_mhop_get()
610 		++table->hem[index.buf]->refcount;  in hns_roce_table_mhop_get()
614 	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
621 	if (table->type < HEM_TYPE_MTT) {  in hns_roce_table_mhop_get()
622 		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
629 	++table->hem[index.buf]->refcount;  in hns_roce_table_mhop_get()
633 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
635 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_get()
640 		       struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_get()  argument
646 	if (hns_roce_check_whether_mhop(hr_dev, table->type))  in hns_roce_table_get()
647 		return hns_roce_table_mhop_get(hr_dev, table, obj);  in hns_roce_table_get()
649 	i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /  in hns_roce_table_get()
650 	     table->obj_size);  in hns_roce_table_get()
652 	mutex_lock(&table->mutex);  in hns_roce_table_get()
654 	if (table->hem[i]) {  in hns_roce_table_get()
655 		++table->hem[i]->refcount;  in hns_roce_table_get()
659 	table->hem[i] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_get()
660 				       table->table_chunk_size >> PAGE_SHIFT,  in hns_roce_table_get()
661 				       table->table_chunk_size,  in hns_roce_table_get()
662 				       (table->lowmem ? GFP_KERNEL :  in hns_roce_table_get()
664 	if (!table->hem[i]) {  in hns_roce_table_get()
670 	if (hns_roce_set_hem(hr_dev, table, obj)) {  in hns_roce_table_get()
671 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_get()
672 		table->hem[i] = NULL;  in hns_roce_table_get()
678 	++table->hem[i]->refcount;  in hns_roce_table_get()
680 	mutex_unlock(&table->mutex);  in hns_roce_table_get()
685 			   struct hns_roce_hem_table *table, unsigned long obj,  in clear_mhop_hem()  argument
696 	if (check_whether_bt_num_2(table->type, hop_num)) {  in clear_mhop_hem()
697 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
698 					    chunk_ba_num, table->num_hem))  in clear_mhop_hem()
700 	} else if (check_whether_bt_num_3(table->type, hop_num)) {  in clear_mhop_hem()
701 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
702 					    chunk_ba_num, table->num_hem)) {  in clear_mhop_hem()
704 			if (hns_roce_check_bt_null(table->bt_l1, index->l1,  in clear_mhop_hem()
710 	if (table->type < HEM_TYPE_MTT) {  in clear_mhop_hem()
716 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))  in clear_mhop_hem()
720 			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))  in clear_mhop_hem()
724 			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))  in clear_mhop_hem()
730 				    struct hns_roce_hem_table *table,  in hns_roce_table_mhop_put()  argument
739 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
745 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_put()
746 	if (check_refcount && (--table->hem[index.buf]->refcount > 0)) {  in hns_roce_table_mhop_put()
747 		mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
751 	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
752 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_put()
754 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
758 			struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_put()  argument
763 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put()
764 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);  in hns_roce_table_put()
768 	i = (obj & (table->num_obj - 1)) /  in hns_roce_table_put()
769 	    (table->table_chunk_size / table->obj_size);  in hns_roce_table_put()
771 	mutex_lock(&table->mutex);  in hns_roce_table_put()
773 	if (--table->hem[i]->refcount == 0) {  in hns_roce_table_put()
775 		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))  in hns_roce_table_put()
778 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_put()
779 		table->hem[i] = NULL;  in hns_roce_table_put()
782 	mutex_unlock(&table->mutex);  in hns_roce_table_put()
786 			  struct hns_roce_hem_table *table,  in hns_roce_table_find()  argument
801 	if (!table->lowmem)  in hns_roce_table_find()
804 	mutex_lock(&table->mutex);  in hns_roce_table_find()
806 	if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_find()
807 		obj_per_chunk = table->table_chunk_size / table->obj_size;  in hns_roce_table_find()
808 		hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];  in hns_roce_table_find()
809 		idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;  in hns_roce_table_find()
810 		dma_offset = offset = idx_offset * table->obj_size;  in hns_roce_table_find()
814 		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))  in hns_roce_table_find()
825 		hem = table->hem[hem_idx];  in hns_roce_table_find()
826 		dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %  in hns_roce_table_find()
854 	mutex_unlock(&table->mutex);  in hns_roce_table_find()
859 			    struct hns_roce_hem_table *table, u32 type,  in hns_roce_init_hem_table()  argument
867 		table->table_chunk_size = hr_dev->caps.chunk_sz;  in hns_roce_init_hem_table()
868 		obj_per_chunk = table->table_chunk_size / obj_size;  in hns_roce_init_hem_table()
871 		table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);  in hns_roce_init_hem_table()
872 		if (!table->hem)  in hns_roce_init_hem_table()
896 		table->hem = kcalloc(num_hem, sizeof(*table->hem),  in hns_roce_init_hem_table()
898 		if (!table->hem)  in hns_roce_init_hem_table()
906 			table->bt_l1 = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
907 					       sizeof(*table->bt_l1),  in hns_roce_init_hem_table()
909 			if (!table->bt_l1)  in hns_roce_init_hem_table()
912 			table->bt_l1_dma_addr = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
913 						 sizeof(*table->bt_l1_dma_addr),  in hns_roce_init_hem_table()
916 			if (!table->bt_l1_dma_addr)  in hns_roce_init_hem_table()
922 			table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),  in hns_roce_init_hem_table()
924 			if (!table->bt_l0)  in hns_roce_init_hem_table()
927 			table->bt_l0_dma_addr = kcalloc(num_bt_l0,  in hns_roce_init_hem_table()
928 						 sizeof(*table->bt_l0_dma_addr),  in hns_roce_init_hem_table()
930 			if (!table->bt_l0_dma_addr)  in hns_roce_init_hem_table()
935 	table->type = type;  in hns_roce_init_hem_table()
936 	table->num_hem = num_hem;  in hns_roce_init_hem_table()
937 	table->num_obj = nobj;  in hns_roce_init_hem_table()
938 	table->obj_size = obj_size;  in hns_roce_init_hem_table()
939 	table->lowmem = use_lowmem;  in hns_roce_init_hem_table()
940 	mutex_init(&table->mutex);  in hns_roce_init_hem_table()
945 	kfree(table->bt_l0);  in hns_roce_init_hem_table()
946 	table->bt_l0 = NULL;  in hns_roce_init_hem_table()
949 	kfree(table->bt_l1_dma_addr);  in hns_roce_init_hem_table()
950 	table->bt_l1_dma_addr = NULL;  in hns_roce_init_hem_table()
953 	kfree(table->bt_l1);  in hns_roce_init_hem_table()
954 	table->bt_l1 = NULL;  in hns_roce_init_hem_table()
957 	kfree(table->hem);  in hns_roce_init_hem_table()
958 	table->hem = NULL;  in hns_roce_init_hem_table()
965 					    struct hns_roce_hem_table *table)  in hns_roce_cleanup_mhop_hem_table()  argument
972 	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_cleanup_mhop_hem_table()
974 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :  in hns_roce_cleanup_mhop_hem_table()
977 	for (i = 0; i < table->num_hem; ++i) {  in hns_roce_cleanup_mhop_hem_table()
978 		obj = i * buf_chunk_size / table->obj_size;  in hns_roce_cleanup_mhop_hem_table()
979 		if (table->hem[i])  in hns_roce_cleanup_mhop_hem_table()
980 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);  in hns_roce_cleanup_mhop_hem_table()
983 	kfree(table->hem);  in hns_roce_cleanup_mhop_hem_table()
984 	table->hem = NULL;  in hns_roce_cleanup_mhop_hem_table()
985 	kfree(table->bt_l1);  in hns_roce_cleanup_mhop_hem_table()
986 	table->bt_l1 = NULL;  in hns_roce_cleanup_mhop_hem_table()
987 	kfree(table->bt_l1_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
988 	table->bt_l1_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
989 	kfree(table->bt_l0);  in hns_roce_cleanup_mhop_hem_table()
990 	table->bt_l0 = NULL;  in hns_roce_cleanup_mhop_hem_table()
991 	kfree(table->bt_l0_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
992 	table->bt_l0_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
996 				struct hns_roce_hem_table *table)  in hns_roce_cleanup_hem_table()  argument
1001 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_cleanup_hem_table()
1002 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);  in hns_roce_cleanup_hem_table()
1006 	for (i = 0; i < table->num_hem; ++i)  in hns_roce_cleanup_hem_table()
1007 		if (table->hem[i]) {  in hns_roce_cleanup_hem_table()
1008 			if (hr_dev->hw->clear_hem(hr_dev, table,  in hns_roce_cleanup_hem_table()
1009 			    i * table->table_chunk_size / table->obj_size, 0))  in hns_roce_cleanup_hem_table()
1012 			hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_cleanup_hem_table()
1015 	kfree(table->hem);  in hns_roce_cleanup_hem_table()
1022 					   &hr_dev->srq_table.table);  in hns_roce_cleanup_hem()
1023 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);  in hns_roce_cleanup_hem()
1107 /* assign L0 table address to hem from root bt */
1141 	 * hopnum    base address table levels  in hem_list_is_bottom_bt()
1153  * @bt_level: base address table level
1399 /* construct the base address table and link them by address hop config */