58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,  in uverbs_response()  argument
63 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in uverbs_response()
65 			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);  in uverbs_response()
67 	if (copy_to_user(attrs->ucore.outbuf, resp,  in uverbs_response()
68 			 min(attrs->ucore.outlen, resp_len)))  in uverbs_response()
71 	if (resp_len < attrs->ucore.outlen) {  in uverbs_response()
76 		ret = clear_user(attrs->ucore.outbuf + resp_len,  in uverbs_response()
77 				 attrs->ucore.outlen - resp_len);  in uverbs_response()
91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,  in uverbs_request()  argument
94 	if (copy_from_user(req, attrs->ucore.inbuf,  in uverbs_request()
95 			   min(attrs->ucore.inlen, req_len)))  in uverbs_request()
98 	if (attrs->ucore.inlen < req_len) {  in uverbs_request()
99 		memset(req + attrs->ucore.inlen, 0,  in uverbs_request()
100 		       req_len - attrs->ucore.inlen);  in uverbs_request()
101 	} else if (attrs->ucore.inlen > req_len) {  in uverbs_request()
102 		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,  in uverbs_request()
103 					  attrs->ucore.inlen - req_len))  in uverbs_request()
115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,  in uverbs_response_length()  argument
118 	return min_t(size_t, attrs->ucore.outlen, resp_len);  in uverbs_response_length()
130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,  in uverbs_request_start()  argument
135 	if (attrs->ucore.inlen < req_len)  in uverbs_request_start()
138 	if (copy_from_user(req, attrs->ucore.inbuf, req_len))  in uverbs_request_start()
141 	iter->cur = attrs->ucore.inbuf + req_len;  in uverbs_request_start()
142 	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;  in uverbs_request_start()
182 struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)  in uverbs_get_cleared_udata()  argument
184 	attrs->driver_udata = (struct ib_udata){};  in uverbs_get_cleared_udata()
185 	return &attrs->driver_udata;  in uverbs_get_cleared_udata()
189 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)  in _ib_uverbs_lookup_comp_file()  argument
192 					       fd, attrs);  in _ib_uverbs_lookup_comp_file()
206 int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)  in ib_alloc_ucontext()  argument
208 	struct ib_uverbs_file *ufile = attrs->ufile;  in ib_alloc_ucontext()
227 	attrs->context = ucontext;  in ib_alloc_ucontext()
231 int ib_init_ucontext(struct uverbs_attr_bundle *attrs)  in ib_init_ucontext()  argument
233 	struct ib_ucontext *ucontext = attrs->context;  in ib_init_ucontext()
234 	struct ib_uverbs_file *file = attrs->ufile;  in ib_init_ucontext()
251 						   &attrs->driver_udata);  in ib_init_ucontext()
276 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)  in ib_uverbs_get_context()  argument
284 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_get_context()
288 	ret = ib_alloc_ucontext(attrs);  in ib_uverbs_get_context()
292 	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);  in ib_uverbs_get_context()
299 		.num_comp_vectors = attrs->ufile->device->num_comp_vectors,  in ib_uverbs_get_context()
302 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_get_context()
306 	ret = ib_init_ucontext(attrs);  in ib_uverbs_get_context()
312 	rdma_alloc_commit_uobject(uobj, attrs);  in ib_uverbs_get_context()
316 	rdma_alloc_abort_uobject(uobj, attrs, false);  in ib_uverbs_get_context()
318 	rdma_restrack_put(&attrs->context->res);  in ib_uverbs_get_context()
319 	kfree(attrs->context);  in ib_uverbs_get_context()
320 	attrs->context = NULL;  in ib_uverbs_get_context()
370 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_device()  argument
377 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_device()
381 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_device()
386 	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);  in ib_uverbs_query_device()
388 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_device()
391 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_port()  argument
400 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_port()
405 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_port()
416 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_port()
419 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_pd()  argument
428 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_pd()
432 	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);  in ib_uverbs_alloc_pd()
449 	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);  in ib_uverbs_alloc_pd()
455 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_alloc_pd()
458 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_pd()
464 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_pd()
468 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_pd()  argument
473 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_pd()
477 	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_dealloc_pd()
565 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_xrcd()  argument
567 	struct ib_uverbs_device *ibudev = attrs->ufile->device;  in ib_uverbs_open_xrcd()
578 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_xrcd()
606 	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,  in ib_uverbs_open_xrcd()
614 		xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);  in ib_uverbs_open_xrcd()
639 	uobj_finalize_uobj_create(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
642 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_xrcd()
645 	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));  in ib_uverbs_open_xrcd()
648 	uobj_alloc_abort(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
659 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_close_xrcd()  argument
664 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_close_xrcd()
668 	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);  in ib_uverbs_close_xrcd()
673 			   struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_xrcd()  argument
677 	struct ib_uverbs_device *dev = attrs->ufile->device;  in ib_uverbs_dealloc_xrcd()
683 	ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);  in ib_uverbs_dealloc_xrcd()
695 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_reg_mr()  argument
705 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_reg_mr()
712 	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);  in ib_uverbs_reg_mr()
720 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_reg_mr()
728 					 &attrs->driver_udata);  in ib_uverbs_reg_mr()
750 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_reg_mr()
755 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_reg_mr()
760 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_reg_mr()
764 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_rereg_mr()  argument
777 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_rereg_mr()
791 	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_rereg_mr()
811 					   attrs);  in ib_uverbs_rereg_mr()
824 	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);  in ib_uverbs_rereg_mr()
832 					   &attrs->driver_udata);  in ib_uverbs_rereg_mr()
853 		rdma_assign_uobject(uobj, new_uobj, attrs);  in ib_uverbs_rereg_mr()
854 		rdma_alloc_commit_uobject(new_uobj, attrs);  in ib_uverbs_rereg_mr()
875 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_rereg_mr()
879 		uobj_alloc_abort(new_uobj, attrs);  in ib_uverbs_rereg_mr()
891 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dereg_mr()  argument
896 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dereg_mr()
900 	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_dereg_mr()
903 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_mw()  argument
913 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_mw()
917 	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);  in ib_uverbs_alloc_mw()
921 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_alloc_mw()
943 	ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);  in ib_uverbs_alloc_mw()
951 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_alloc_mw()
955 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_mw()
962 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_mw()
966 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_mw()  argument
971 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_mw()
975 	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);  in ib_uverbs_dealloc_mw()
978 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_comp_channel()  argument
987 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_comp_channel()
991 	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);  in ib_uverbs_create_comp_channel()
998 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_create_comp_channel()
1001 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_comp_channel()
1004 static int create_cq(struct uverbs_attr_bundle *attrs,  in create_cq()  argument
1015 	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)  in create_cq()
1018 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,  in create_cq()
1024 		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);  in create_cq()
1054 	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);  in create_cq()
1060 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in create_cq()
1063 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in create_cq()
1067 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_cq()
1068 	return uverbs_response(attrs, &resp, sizeof(resp));  in create_cq()
1077 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in create_cq()
1081 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_cq()  argument
1087 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_cq()
1097 	return create_cq(attrs, &cmd_ex);  in ib_uverbs_create_cq()
1100 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_cq()  argument
1105 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_cq()
1115 	return create_cq(attrs, &cmd);  in ib_uverbs_ex_create_cq()
1118 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_resize_cq()  argument
1125 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_resize_cq()
1129 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_resize_cq()
1133 	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);  in ib_uverbs_resize_cq()
1139 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_resize_cq()
1177 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_poll_cq()  argument
1187 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_poll_cq()
1191 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_poll_cq()
1196 	header_ptr = attrs->ucore.outbuf;  in ib_uverbs_poll_cq()
1221 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in ib_uverbs_poll_cq()
1222 		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);  in ib_uverbs_poll_cq()
1230 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_req_notify_cq()  argument
1236 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_req_notify_cq()
1240 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_req_notify_cq()
1252 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_cq()  argument
1260 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_cq()
1264 	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_destroy_cq()
1275 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_cq()
1278 static int create_qp(struct uverbs_attr_bundle *attrs,  in create_qp()  argument
1312 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in create_qp()
1323 					    cmd->rwq_ind_tbl_handle, attrs);  in create_qp()
1342 					  attrs);  in create_qp()
1362 							cmd->srq_handle, attrs);  in create_qp()
1373 						cmd->recv_cq_handle, attrs);  in create_qp()
1384 						cmd->send_cq_handle, attrs);  in create_qp()
1388 				       attrs);  in create_qp()
1437 	qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,  in create_qp()
1446 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in create_qp()
1470 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in create_qp()
1479 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_qp()
1480 	return uverbs_response(attrs, &resp, sizeof(resp));  in create_qp()
1499 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in create_qp()
1503 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_qp()  argument
1509 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_qp()
1528 	return create_qp(attrs, &cmd_ex);  in ib_uverbs_create_qp()
1531 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_qp()  argument
1536 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_qp()
1546 	return create_qp(attrs, &cmd);  in ib_uverbs_ex_create_qp()
1549 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_qp()  argument
1561 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_qp()
1565 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in ib_uverbs_open_qp()
1570 	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);  in ib_uverbs_open_qp()
1602 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1606 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_qp()
1611 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1637 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_qp()  argument
1646 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_qp()
1657 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_query_qp()
1705 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_qp()
1751 static int modify_qp(struct uverbs_attr_bundle *attrs,  in modify_qp()  argument
1763 			       attrs);  in modify_qp()
1905 				      &attrs->driver_udata);  in modify_qp()
1916 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_qp()  argument
1921 	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));  in ib_uverbs_modify_qp()
1928 	return modify_qp(attrs, &cmd);  in ib_uverbs_modify_qp()
1931 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_qp()  argument
1935 		.response_length = uverbs_response_length(attrs, sizeof(resp))  in ib_uverbs_ex_modify_qp()
1939 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_qp()
1950 	ret = modify_qp(attrs, &cmd);  in ib_uverbs_ex_modify_qp()
1954 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_modify_qp()
1957 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_qp()  argument
1965 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_qp()
1969 	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_destroy_qp()
1979 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_qp()
1993 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_send()  argument
2009 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_send()
2027 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_send()
2065 						   user_wr->wr.ud.ah, attrs);  in ib_uverbs_post_send()
2165 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_send()
2285 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_recv()  argument
2295 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_recv()
2304 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_recv()
2323 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_recv()
2336 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_srq_recv()  argument
2346 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_srq_recv()
2355 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_post_srq_recv()
2374 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_srq_recv()
2388 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_ah()  argument
2399 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_ah()
2403 	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);  in ib_uverbs_create_ah()
2412 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_create_ah()
2436 	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);  in ib_uverbs_create_ah()
2446 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_create_ah()
2449 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_ah()
2454 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_create_ah()
2458 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_ah()  argument
2463 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_ah()
2467 	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);  in ib_uverbs_destroy_ah()
2470 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_attach_mcast()  argument
2478 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_attach_mcast()
2482 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_attach_mcast()
2519 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_detach_mcast()  argument
2528 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_detach_mcast()
2532 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_detach_mcast()
2638 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec_action()  argument
2667 							attrs);  in kern_spec_to_ib_spec_action()
2685 					  attrs);  in kern_spec_to_ib_spec_action()
2865 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec()  argument
2874 		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,  in kern_spec_to_ib_spec()
2880 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_wq()  argument
2892 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_wq()
2899 	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,  in ib_uverbs_ex_create_wq()
2904 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_ex_create_wq()
2910 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_create_wq()
2925 	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);  in ib_uverbs_ex_create_wq()
2940 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in ib_uverbs_ex_create_wq()
2947 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2953 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_wq()
2954 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_wq()
2962 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2967 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_wq()  argument
2975 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_wq()
2982 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
2983 	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_destroy_wq()
2992 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
2995 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_wq()  argument
3002 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_wq()
3012 	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_modify_wq()
3040 					&attrs->driver_udata);  in ib_uverbs_ex_modify_wq()
3046 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_rwq_ind_table()  argument
3062 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_rwq_ind_table()
3096 				       wqs_handles[num_read_wqs], attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3106 	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);  in ib_uverbs_ex_create_rwq_ind_table()
3129 					       &attrs->driver_udata);  in ib_uverbs_ex_create_rwq_ind_table()
3137 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3141 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3142 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3147 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3160 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_rwq_ind_table()  argument
3165 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_rwq_ind_table()
3173 				    cmd.ind_tbl_handle, attrs);  in ib_uverbs_ex_destroy_rwq_ind_table()
3176 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_flow()  argument
3193 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_flow()
3241 	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);  in ib_uverbs_ex_create_flow()
3252 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_ex_create_flow()
3289 				attrs, (struct ib_uverbs_flow_spec *)kern_spec,  in ib_uverbs_ex_create_flow()
3308 					      &attrs->driver_udata);  in ib_uverbs_ex_create_flow()
3323 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_ex_create_flow()
3326 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_flow()
3336 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_flow()
3343 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_flow()  argument
3348 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_flow()
3355 	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);  in ib_uverbs_ex_destroy_flow()
3358 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,  in __uverbs_create_xsrq()  argument
3371 	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,  in __uverbs_create_xsrq()
3381 					  attrs);  in __uverbs_create_xsrq()
3399 						cmd->cq_handle, attrs);  in __uverbs_create_xsrq()
3406 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);  in __uverbs_create_xsrq()
3429 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in __uverbs_create_xsrq()
3444 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3449 	return uverbs_response(attrs, &resp, sizeof(resp));  in __uverbs_create_xsrq()
3465 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3469 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_srq()  argument
3475 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_srq()
3488 	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);  in ib_uverbs_create_srq()
3491 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_xsrq()  argument
3496 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_xsrq()
3500 	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);  in ib_uverbs_create_xsrq()
3503 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_srq()  argument
3510 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_modify_srq()
3514 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_modify_srq()
3522 					  &attrs->driver_udata);  in ib_uverbs_modify_srq()
3530 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_srq()  argument
3538 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_srq()
3542 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_query_srq()
3560 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_srq()
3563 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_srq()  argument
3571 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_srq()
3575 	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_destroy_srq()
3585 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_srq()
3588 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_query_device()  argument
3597 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_ex_query_device()
3602 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_query_device()
3612 	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);  in ib_uverbs_ex_query_device()
3647 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_query_device()
3649 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_query_device()
3652 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_cq()  argument
3658 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_cq()
3668 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_modify_cq()