82 static void init_subctxts(struct hfi1_ctxtdata *uctxt,
85 			  struct hfi1_ctxtdata *uctxt);
86 static void user_init(struct hfi1_ctxtdata *uctxt);
96 			   struct hfi1_ctxtdata *uctxt);
97 static int setup_subctxt(struct hfi1_ctxtdata *uctxt);
104 static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt);
107 static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,
109 static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg);
110 static int ctxt_reset(struct hfi1_ctxtdata *uctxt);
111 static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
226 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_file_ioctl()  local
233 	    !uctxt)  in hfi1_file_ioctl()
250 		if (uctxt)  in hfi1_file_ioctl()
251 			sc_return_credits(uctxt->sc);  in hfi1_file_ioctl()
267 		ret = manage_rcvq(uctxt, fd->subctxt, arg);  in hfi1_file_ioctl()
273 		uctxt->poll_type = (typeof(uctxt->poll_type))uval;  in hfi1_file_ioctl()
277 		ret = user_event_ack(uctxt, fd->subctxt, arg);  in hfi1_file_ioctl()
281 		ret = set_ctxt_pkey(uctxt, arg);  in hfi1_file_ioctl()
285 		ret = ctxt_reset(uctxt);  in hfi1_file_ioctl()
315 	trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim);  in hfi1_write_iter()
342 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_file_mmap()  local
353 	if (!is_valid_mmap(token) || !uctxt ||  in hfi1_file_mmap()
358 	dd = uctxt->dd;  in hfi1_file_mmap()
362 	if (ctxt != uctxt->ctxt || subctxt != fd->subctxt) {  in hfi1_file_mmap()
374 			   (uctxt->sc->hw_context * BIT(16))) +  in hfi1_file_mmap()
382 		memlen = PAGE_ALIGN(uctxt->sc->credits * PIO_BLOCK_SIZE);  in hfi1_file_mmap()
398 		memvirt = dd->cr_base[uctxt->numa_id].va;  in hfi1_file_mmap()
400 			(((u64)uctxt->sc->hw_free -  in hfi1_file_mmap()
401 			  (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK);  in hfi1_file_mmap()
414 		memlen = rcvhdrq_size(uctxt);  in hfi1_file_mmap()
415 		memvirt = uctxt->rcvhdrq;  in hfi1_file_mmap()
425 		memlen = uctxt->egrbufs.size;  in hfi1_file_mmap()
438 		for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) {  in hfi1_file_mmap()
439 			memlen = uctxt->egrbufs.buffers[i].len;  in hfi1_file_mmap()
440 			memvirt = uctxt->egrbufs.buffers[i].addr;  in hfi1_file_mmap()
465 			+ (uctxt->ctxt * RXE_PER_CONTEXT_SIZE);  in hfi1_file_mmap()
481 			(dd->events + uctxt_offset(uctxt)) & PAGE_MASK;  in hfi1_file_mmap()
508 		if ((flags & VM_WRITE) || !uctxt->rcvhdrtail_kvaddr) {  in hfi1_file_mmap()
513 		memvirt = (void *)uctxt->rcvhdrtail_kvaddr;  in hfi1_file_mmap()
517 		memaddr = (u64)uctxt->subctxt_uregbase;  in hfi1_file_mmap()
523 		memaddr = (u64)uctxt->subctxt_rcvhdr_base;  in hfi1_file_mmap()
524 		memlen = rcvhdrq_size(uctxt) * uctxt->subctxt_cnt;  in hfi1_file_mmap()
529 		memaddr = (u64)uctxt->subctxt_rcvegrbuf;  in hfi1_file_mmap()
530 		memlen = uctxt->egrbufs.size * uctxt->subctxt_cnt;  in hfi1_file_mmap()
555 			  uctxt->ctxt, fd->subctxt,  in hfi1_file_mmap()
610 	struct hfi1_ctxtdata *uctxt;  in hfi1_poll()  local
613 	uctxt = ((struct hfi1_filedata *)fp->private_data)->uctxt;  in hfi1_poll()
614 	if (!uctxt)  in hfi1_poll()
616 	else if (uctxt->poll_type == HFI1_POLL_TYPE_URGENT)  in hfi1_poll()
618 	else  if (uctxt->poll_type == HFI1_POLL_TYPE_ANYRCV)  in hfi1_poll()
629 	struct hfi1_ctxtdata *uctxt = fdata->uctxt;  in hfi1_file_close()  local
637 	if (!uctxt)  in hfi1_file_close()
640 	hfi1_cdbg(PROC, "closing ctxt %u:%u", uctxt->ctxt, fdata->subctxt);  in hfi1_file_close()
644 	hfi1_user_sdma_free_queues(fdata, uctxt);  in hfi1_file_close()
656 	fdata->uctxt = NULL;  in hfi1_file_close()
657 	hfi1_rcd_put(uctxt);  in hfi1_file_close()
663 	ev = dd->events + uctxt_offset(uctxt) + fdata->subctxt;  in hfi1_file_close()
667 	__clear_bit(fdata->subctxt, uctxt->in_use_ctxts);  in hfi1_file_close()
668 	if (!bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {  in hfi1_file_close()
684 		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);  in hfi1_file_close()
686 	hfi1_clear_ctxt_jkey(dd, uctxt);  in hfi1_file_close()
691 	if (uctxt->sc) {  in hfi1_file_close()
692 		sc_disable(uctxt->sc);  in hfi1_file_close()
693 		set_pio_integrity(uctxt->sc);  in hfi1_file_close()
696 	hfi1_free_ctxt_rcv_groups(uctxt);  in hfi1_file_close()
697 	hfi1_clear_ctxt_pkey(dd, uctxt);  in hfi1_file_close()
699 	uctxt->event_flags = 0;  in hfi1_file_close()
701 	deallocate_ctxt(uctxt);  in hfi1_file_close()
751 		fd->uctxt->wait,  in complete_subctxt()
752 		!test_bit(HFI1_CTXT_BASE_UNINIT, &fd->uctxt->event_flags));  in complete_subctxt()
754 	if (test_bit(HFI1_CTXT_BASE_FAILED, &fd->uctxt->event_flags))  in complete_subctxt()
759 		fd->rec_cpu_num = hfi1_get_proc_affinity(fd->uctxt->numa_id);  in complete_subctxt()
760 		ret = init_user_ctxt(fd, fd->uctxt);  in complete_subctxt()
765 		__clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts);  in complete_subctxt()
767 		hfi1_rcd_put(fd->uctxt);  in complete_subctxt()
768 		fd->uctxt = NULL;  in complete_subctxt()
778 	struct hfi1_ctxtdata *uctxt = NULL;  in assign_ctxt()  local
781 	if (fd->uctxt)  in assign_ctxt()
813 		ret = allocate_ctxt(fd, fd->dd, &uinfo, &uctxt);  in assign_ctxt()
820 		ret = setup_base_ctxt(fd, uctxt);  in assign_ctxt()
822 			deallocate_ctxt(uctxt);  in assign_ctxt()
845 		      struct hfi1_ctxtdata *uctxt)  in match_ctxt()  argument
852 	if (uctxt->sc && (uctxt->sc->type == SC_KERNEL))  in match_ctxt()
856 	if (memcmp(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid)) ||  in match_ctxt()
857 	    uctxt->jkey != generate_jkey(current_uid()) ||  in match_ctxt()
858 	    uctxt->subctxt_id != uinfo->subctxt_id ||  in match_ctxt()
859 	    uctxt->subctxt_cnt != uinfo->subctxt_cnt)  in match_ctxt()
863 	if (uctxt->userversion != uinfo->userversion)  in match_ctxt()
868 	if (bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {  in match_ctxt()
874 	subctxt = find_first_zero_bit(uctxt->in_use_ctxts,  in match_ctxt()
876 	if (subctxt >= uctxt->subctxt_cnt) {  in match_ctxt()
882 	__set_bit(fd->subctxt, uctxt->in_use_ctxts);  in match_ctxt()
885 	fd->uctxt = uctxt;  in match_ctxt()
886 	hfi1_rcd_get(uctxt);  in match_ctxt()
908 	struct hfi1_ctxtdata *uctxt;  in find_sub_ctxt()  local
917 		uctxt = hfi1_rcd_get_by_index(dd, i);  in find_sub_ctxt()
918 		if (uctxt) {  in find_sub_ctxt()
919 			ret = match_ctxt(fd, uinfo, uctxt);  in find_sub_ctxt()
920 			hfi1_rcd_put(uctxt);  in find_sub_ctxt()
934 	struct hfi1_ctxtdata *uctxt;  in allocate_ctxt()  local
960 	ret = hfi1_create_ctxtdata(dd->pport, numa, &uctxt);  in allocate_ctxt()
966 		  uctxt->ctxt, fd->subctxt, current->pid, fd->rec_cpu_num,  in allocate_ctxt()
967 		  uctxt->numa_id);  in allocate_ctxt()
972 	uctxt->sc = sc_alloc(dd, SC_USER, uctxt->rcvhdrqentsize, dd->node);  in allocate_ctxt()
973 	if (!uctxt->sc) {  in allocate_ctxt()
977 	hfi1_cdbg(PROC, "allocated send context %u(%u)\n", uctxt->sc->sw_index,  in allocate_ctxt()
978 		  uctxt->sc->hw_context);  in allocate_ctxt()
979 	ret = sc_enable(uctxt->sc);  in allocate_ctxt()
992 	__set_bit(0, uctxt->in_use_ctxts);  in allocate_ctxt()
994 		init_subctxts(uctxt, uinfo);  in allocate_ctxt()
995 	uctxt->userversion = uinfo->userversion;  in allocate_ctxt()
996 	uctxt->flags = hfi1_cap_mask; /* save current flag state */  in allocate_ctxt()
997 	init_waitqueue_head(&uctxt->wait);  in allocate_ctxt()
998 	strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));  in allocate_ctxt()
999 	memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));  in allocate_ctxt()
1000 	uctxt->jkey = generate_jkey(current_uid());  in allocate_ctxt()
1009 	*rcd = uctxt;  in allocate_ctxt()
1014 	hfi1_free_ctxt(uctxt);  in allocate_ctxt()
1018 static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt)  in deallocate_ctxt()  argument
1022 	if (++uctxt->dd->freectxts == uctxt->dd->num_user_contexts)  in deallocate_ctxt()
1023 		aspm_enable_all(uctxt->dd);  in deallocate_ctxt()
1026 	hfi1_free_ctxt(uctxt);  in deallocate_ctxt()
1029 static void init_subctxts(struct hfi1_ctxtdata *uctxt,  in init_subctxts()  argument
1032 	uctxt->subctxt_cnt = uinfo->subctxt_cnt;  in init_subctxts()
1033 	uctxt->subctxt_id = uinfo->subctxt_id;  in init_subctxts()
1034 	set_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);  in init_subctxts()
1037 static int setup_subctxt(struct hfi1_ctxtdata *uctxt)  in setup_subctxt()  argument
1040 	u16 num_subctxts = uctxt->subctxt_cnt;  in setup_subctxt()
1042 	uctxt->subctxt_uregbase = vmalloc_user(PAGE_SIZE);  in setup_subctxt()
1043 	if (!uctxt->subctxt_uregbase)  in setup_subctxt()
1047 	uctxt->subctxt_rcvhdr_base = vmalloc_user(rcvhdrq_size(uctxt) *  in setup_subctxt()
1049 	if (!uctxt->subctxt_rcvhdr_base) {  in setup_subctxt()
1054 	uctxt->subctxt_rcvegrbuf = vmalloc_user(uctxt->egrbufs.size *  in setup_subctxt()
1056 	if (!uctxt->subctxt_rcvegrbuf) {  in setup_subctxt()
1064 	vfree(uctxt->subctxt_rcvhdr_base);  in setup_subctxt()
1065 	uctxt->subctxt_rcvhdr_base = NULL;  in setup_subctxt()
1067 	vfree(uctxt->subctxt_uregbase);  in setup_subctxt()
1068 	uctxt->subctxt_uregbase = NULL;  in setup_subctxt()
1073 static void user_init(struct hfi1_ctxtdata *uctxt)  in user_init()  argument
1078 	uctxt->urgent = 0;  in user_init()
1079 	uctxt->urgent_poll = 0;  in user_init()
1092 	if (uctxt->rcvhdrtail_kvaddr)  in user_init()
1093 		clear_rcvhdrtail(uctxt);  in user_init()
1096 	hfi1_set_ctxt_jkey(uctxt->dd, uctxt, uctxt->jkey);  in user_init()
1099 	if (HFI1_CAP_UGET_MASK(uctxt->flags, HDRSUPP))  in user_init()
1106 	if (!HFI1_CAP_UGET_MASK(uctxt->flags, MULTI_PKT_EGR))  in user_init()
1108 	if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_EGR_FULL))  in user_init()
1110 	if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_RHQ_FULL))  in user_init()
1118 	if (HFI1_CAP_UGET_MASK(uctxt->flags, DMA_RTAIL))  in user_init()
1122 	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);  in user_init()
1128 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in get_ctxt_info()  local
1134 	cinfo.runtime_flags = (((uctxt->flags >> HFI1_CAP_MISC_SHIFT) &  in get_ctxt_info()
1136 			HFI1_CAP_UGET_MASK(uctxt->flags, MASK) |  in get_ctxt_info()
1137 			HFI1_CAP_KGET_MASK(uctxt->flags, K2U);  in get_ctxt_info()
1143 	cinfo.unit = uctxt->dd->unit;  in get_ctxt_info()
1144 	cinfo.ctxt = uctxt->ctxt;  in get_ctxt_info()
1146 	cinfo.rcvtids = roundup(uctxt->egrbufs.alloced,  in get_ctxt_info()
1147 				uctxt->dd->rcv_entries.group_size) +  in get_ctxt_info()
1148 		uctxt->expected_count;  in get_ctxt_info()
1149 	cinfo.credits = uctxt->sc->credits;  in get_ctxt_info()
1150 	cinfo.numa_node = uctxt->numa_id;  in get_ctxt_info()
1152 	cinfo.send_ctxt = uctxt->sc->hw_context;  in get_ctxt_info()
1154 	cinfo.egrtids = uctxt->egrbufs.alloced;  in get_ctxt_info()
1155 	cinfo.rcvhdrq_cnt = uctxt->rcvhdrq_cnt;  in get_ctxt_info()
1156 	cinfo.rcvhdrq_entsize = uctxt->rcvhdrqentsize << 2;  in get_ctxt_info()
1158 	cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size;  in get_ctxt_info()
1160 	trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo);  in get_ctxt_info()
1168 			  struct hfi1_ctxtdata *uctxt)  in init_user_ctxt()  argument
1172 	ret = hfi1_user_sdma_alloc_queues(uctxt, fd);  in init_user_ctxt()
1176 	ret = hfi1_user_exp_rcv_init(fd, uctxt);  in init_user_ctxt()
1178 		hfi1_user_sdma_free_queues(fd, uctxt);  in init_user_ctxt()
1184 			   struct hfi1_ctxtdata *uctxt)  in setup_base_ctxt()  argument
1186 	struct hfi1_devdata *dd = uctxt->dd;  in setup_base_ctxt()
1189 	hfi1_init_ctxt(uctxt->sc);  in setup_base_ctxt()
1192 	ret = hfi1_create_rcvhdrq(dd, uctxt);  in setup_base_ctxt()
1196 	ret = hfi1_setup_eagerbufs(uctxt);  in setup_base_ctxt()
1201 	if (uctxt->subctxt_cnt)  in setup_base_ctxt()
1202 		ret = setup_subctxt(uctxt);  in setup_base_ctxt()
1206 	ret = hfi1_alloc_ctxt_rcv_groups(uctxt);  in setup_base_ctxt()
1210 	ret = init_user_ctxt(fd, uctxt);  in setup_base_ctxt()
1214 	user_init(uctxt);  in setup_base_ctxt()
1217 	fd->uctxt = uctxt;  in setup_base_ctxt()
1218 	hfi1_rcd_get(uctxt);  in setup_base_ctxt()
1221 	if (uctxt->subctxt_cnt) {  in setup_base_ctxt()
1227 			set_bit(HFI1_CTXT_BASE_FAILED, &uctxt->event_flags);  in setup_base_ctxt()
1233 		clear_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);  in setup_base_ctxt()
1234 		wake_up(&uctxt->wait);  in setup_base_ctxt()
1243 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in get_base_info()  local
1244 	struct hfi1_devdata *dd = uctxt->dd;  in get_base_info()
1247 	trace_hfi1_uctxtdata(uctxt->dd, uctxt, fd->subctxt);  in get_base_info()
1256 	binfo.jkey = uctxt->jkey;  in get_base_info()
1263 	offset = ((u64)uctxt->sc->hw_free -  in get_base_info()
1264 		  (u64)dd->cr_base[uctxt->numa_id].va) % PAGE_SIZE;  in get_base_info()
1265 	binfo.sc_credits_addr = HFI1_MMAP_TOKEN(PIO_CRED, uctxt->ctxt,  in get_base_info()
1267 	binfo.pio_bufbase = HFI1_MMAP_TOKEN(PIO_BUFS, uctxt->ctxt,  in get_base_info()
1269 					    uctxt->sc->base_addr);  in get_base_info()
1271 						uctxt->ctxt,  in get_base_info()
1273 						uctxt->sc->base_addr);  in get_base_info()
1274 	binfo.rcvhdr_bufbase = HFI1_MMAP_TOKEN(RCV_HDRQ, uctxt->ctxt,  in get_base_info()
1276 					       uctxt->rcvhdrq);  in get_base_info()
1277 	binfo.rcvegr_bufbase = HFI1_MMAP_TOKEN(RCV_EGRBUF, uctxt->ctxt,  in get_base_info()
1279 					       uctxt->egrbufs.rcvtids[0].dma);  in get_base_info()
1280 	binfo.sdma_comp_bufbase = HFI1_MMAP_TOKEN(SDMA_COMP, uctxt->ctxt,  in get_base_info()
1286 	binfo.user_regbase = HFI1_MMAP_TOKEN(UREGS, uctxt->ctxt,  in get_base_info()
1288 	offset = offset_in_page((uctxt_offset(uctxt) + fd->subctxt) *  in get_base_info()
1290 	binfo.events_bufbase = HFI1_MMAP_TOKEN(EVENTS, uctxt->ctxt,  in get_base_info()
1293 	binfo.status_bufbase = HFI1_MMAP_TOKEN(STATUS, uctxt->ctxt,  in get_base_info()
1297 		binfo.rcvhdrtail_base = HFI1_MMAP_TOKEN(RTAIL, uctxt->ctxt,  in get_base_info()
1299 	if (uctxt->subctxt_cnt) {  in get_base_info()
1301 							 uctxt->ctxt,  in get_base_info()
1304 							  uctxt->ctxt,  in get_base_info()
1307 							  uctxt->ctxt,  in get_base_info()
1434 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in poll_urgent()  local
1435 	struct hfi1_devdata *dd = uctxt->dd;  in poll_urgent()
1438 	poll_wait(fp, &uctxt->wait, pt);  in poll_urgent()
1441 	if (uctxt->urgent != uctxt->urgent_poll) {  in poll_urgent()
1443 		uctxt->urgent_poll = uctxt->urgent;  in poll_urgent()
1446 		set_bit(HFI1_CTXT_WAITING_URG, &uctxt->event_flags);  in poll_urgent()
1457 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in poll_next()  local
1458 	struct hfi1_devdata *dd = uctxt->dd;  in poll_next()
1461 	poll_wait(fp, &uctxt->wait, pt);  in poll_next()
1464 	if (hdrqempty(uctxt)) {  in poll_next()
1465 		set_bit(HFI1_CTXT_WAITING_RCV, &uctxt->event_flags);  in poll_next()
1466 		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt);  in poll_next()
1483 	struct hfi1_ctxtdata *uctxt;  in hfi1_set_uevent_bits()  local
1492 		uctxt = hfi1_rcd_get_by_index(dd, ctxt);  in hfi1_set_uevent_bits()
1493 		if (uctxt) {  in hfi1_set_uevent_bits()
1500 			evs = dd->events + uctxt_offset(uctxt);  in hfi1_set_uevent_bits()
1502 			for (i = 1; i < uctxt->subctxt_cnt; i++)  in hfi1_set_uevent_bits()
1504 			hfi1_rcd_put(uctxt);  in hfi1_set_uevent_bits()
1521 static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,  in manage_rcvq()  argument
1524 	struct hfi1_devdata *dd = uctxt->dd;  in manage_rcvq()
1544 		if (uctxt->rcvhdrtail_kvaddr)  in manage_rcvq()
1545 			clear_rcvhdrtail(uctxt);  in manage_rcvq()
1550 	hfi1_rcvctrl(dd, rcvctrl_op, uctxt);  in manage_rcvq()
1561 static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,  in user_event_ack()  argument
1565 	struct hfi1_devdata *dd = uctxt->dd;  in user_event_ack()
1575 	evs = dd->events + uctxt_offset(uctxt) + subctxt;  in user_event_ack()
1585 static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg)  in set_ctxt_pkey()  argument
1588 	struct hfi1_pportdata *ppd = uctxt->ppd;  in set_ctxt_pkey()
1589 	struct hfi1_devdata *dd = uctxt->dd;  in set_ctxt_pkey()
1603 			return hfi1_set_ctxt_pkey(dd, uctxt, pkey);  in set_ctxt_pkey()
1612 static int ctxt_reset(struct hfi1_ctxtdata *uctxt)  in ctxt_reset()  argument
1618 	if (!uctxt || !uctxt->dd || !uctxt->sc)  in ctxt_reset()
1627 	dd = uctxt->dd;  in ctxt_reset()
1628 	sc = uctxt->sc;  in ctxt_reset()
1661 		hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB, uctxt);  in ctxt_reset()