13 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
45 			   struct hfi1_ctxtdata *uctxt)  in hfi1_user_exp_rcv_init()  argument
49 	fd->entry_to_rb = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
55 	if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) {  in hfi1_user_exp_rcv_init()
57 		fd->invalid_tids = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
82 	if (uctxt->subctxt_cnt && fd->use_mn) {  in hfi1_user_exp_rcv_init()
85 		fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
86 		remainder = uctxt->expected_count % uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
90 		fd->tid_limit = uctxt->expected_count;  in hfi1_user_exp_rcv_init()
99 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_free()  local
101 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
102 	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))  in hfi1_user_exp_rcv_free()
103 		unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);  in hfi1_user_exp_rcv_free()
104 	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))  in hfi1_user_exp_rcv_free()
105 		unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);  in hfi1_user_exp_rcv_free()
106 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
134 	struct hfi1_devdata *dd = fd->uctxt->dd;  in unpin_rcv_pages()
159 	struct hfi1_devdata *dd = fd->uctxt->dd;  in pin_rcv_pages()
166 	if (npages > fd->uctxt->expected_count) {  in pin_rcv_pages()
250 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_setup()  local
251 	struct hfi1_devdata *dd = uctxt->dd;  in hfi1_user_exp_rcv_setup()
266 	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),  in hfi1_user_exp_rcv_setup()
311 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
316 	while (ngroups && uctxt->tid_group_list.count) {  in hfi1_user_exp_rcv_setup()
318 			tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
329 			tid_group_add_tail(grp, &uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
335 		tid_group_add_tail(grp, &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
348 		if (!uctxt->tid_used_list.count || need_group) {  in hfi1_user_exp_rcv_setup()
349 			if (!uctxt->tid_group_list.count)  in hfi1_user_exp_rcv_setup()
352 			grp = tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
353 			tid_group_add_tail(grp, &uctxt->tid_used_list);  in hfi1_user_exp_rcv_setup()
361 		list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list,  in hfi1_user_exp_rcv_setup()
377 						       &uctxt->tid_used_list,  in hfi1_user_exp_rcv_setup()
378 						       &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
397 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
441 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_clear()  local
453 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
466 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
475 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_invalid()  local
476 	unsigned long *ev = uctxt->dd->events +  in hfi1_user_exp_rcv_invalid()
477 		(uctxt_offset(uctxt) + fd->subctxt);  in hfi1_user_exp_rcv_invalid()
487 	array = kcalloc(uctxt->expected_count, sizeof(*array), GFP_KERNEL);  in hfi1_user_exp_rcv_invalid()
617 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in program_rcvarray()  local
618 	struct hfi1_devdata *dd = uctxt->dd;  in program_rcvarray()
664 		tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) |  in program_rcvarray()
685 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in set_rcvarray_entry()  local
687 	struct hfi1_devdata *dd = uctxt->dd;  in set_rcvarray_entry()
730 	fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;  in set_rcvarray_entry()
733 	trace_hfi1_exp_tid_reg(uctxt->ctxt, fd->subctxt, rcventry, npages,  in set_rcvarray_entry()
751 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in unprogram_rcvarray()  local
752 	struct hfi1_devdata *dd = uctxt->dd;  in unprogram_rcvarray()
757 	if (tididx >= uctxt->expected_count) {  in unprogram_rcvarray()
759 			   tididx, uctxt->ctxt);  in unprogram_rcvarray()
769 	if (!node || node->rcventry != (uctxt->expected_base + rcventry))  in unprogram_rcvarray()
784 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in clear_tid_node()  local
785 	struct hfi1_devdata *dd = uctxt->dd;  in clear_tid_node()
787 	trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,  in clear_tid_node()
804 		tid_group_move(node->grp, &uctxt->tid_full_list,  in clear_tid_node()
805 			       &uctxt->tid_used_list);  in clear_tid_node()
807 		tid_group_move(node->grp, &uctxt->tid_used_list,  in clear_tid_node()
808 			       &uctxt->tid_group_list);  in clear_tid_node()
816 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,  in unlock_exp_tids()  argument
832 							  uctxt->expected_base];  in unlock_exp_tids()
852 	struct hfi1_ctxtdata *uctxt = fdata->uctxt;  in tid_rb_invalidate()  local
857 	trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,  in tid_rb_invalidate()
863 	if (fdata->invalid_tid_idx < uctxt->expected_count) {  in tid_rb_invalidate()
865 			rcventry2tidinfo(node->rcventry - uctxt->expected_base);  in tid_rb_invalidate()
880 			ev = uctxt->dd->events +  in tid_rb_invalidate()
881 				(uctxt_offset(uctxt) + fdata->subctxt);  in tid_rb_invalidate()
893 	u32 base = fdata->uctxt->expected_base;  in cacheless_tid_rb_remove()