58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,  in uverbs_response()  argument
63 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in uverbs_response()
65 			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);  in uverbs_response()
67 	if (copy_to_user(attrs->ucore.outbuf, resp,  in uverbs_response()
68 			 min(attrs->ucore.outlen, resp_len)))  in uverbs_response()
71 	if (resp_len < attrs->ucore.outlen) {  in uverbs_response()
76 		ret = clear_user(attrs->ucore.outbuf + resp_len,  in uverbs_response()
77 				 attrs->ucore.outlen - resp_len);  in uverbs_response()
91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,  in uverbs_request()  argument
94 	if (copy_from_user(req, attrs->ucore.inbuf,  in uverbs_request()
95 			   min(attrs->ucore.inlen, req_len)))  in uverbs_request()
98 	if (attrs->ucore.inlen < req_len) {  in uverbs_request()
99 		memset(req + attrs->ucore.inlen, 0,  in uverbs_request()
100 		       req_len - attrs->ucore.inlen);  in uverbs_request()
101 	} else if (attrs->ucore.inlen > req_len) {  in uverbs_request()
102 		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,  in uverbs_request()
103 					  attrs->ucore.inlen - req_len))  in uverbs_request()
115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,  in uverbs_response_length()  argument
118 	return min_t(size_t, attrs->ucore.outlen, resp_len);  in uverbs_response_length()
130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,  in uverbs_request_start()  argument
135 	if (attrs->ucore.inlen < req_len)  in uverbs_request_start()
138 	if (copy_from_user(req, attrs->ucore.inbuf, req_len))  in uverbs_request_start()
141 	iter->cur = attrs->ucore.inbuf + req_len;  in uverbs_request_start()
142 	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;  in uverbs_request_start()
182 struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)  in uverbs_get_cleared_udata()  argument
184 	attrs->driver_udata = (struct ib_udata){};  in uverbs_get_cleared_udata()
185 	return &attrs->driver_udata;  in uverbs_get_cleared_udata()
189 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)  in _ib_uverbs_lookup_comp_file()  argument
192 					       fd, attrs);  in _ib_uverbs_lookup_comp_file()
206 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)  in ib_uverbs_get_context()  argument
208 	struct ib_uverbs_file *file = attrs->ufile;  in ib_uverbs_get_context()
217 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_get_context()
244 	attrs->context = ucontext;  in ib_uverbs_get_context()
268 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_get_context()
272 	ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata);  in ib_uverbs_get_context()
356 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_device()  argument
363 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_device()
367 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_device()
372 	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);  in ib_uverbs_query_device()
374 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_device()
377 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_port()  argument
386 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_port()
391 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_port()
402 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_port()
405 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_pd()  argument
414 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_pd()
418 	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);  in ib_uverbs_alloc_pd()
434 	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);  in ib_uverbs_alloc_pd()
443 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_pd()
447 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_alloc_pd()
450 	ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs));  in ib_uverbs_alloc_pd()
455 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_pd()
459 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_pd()  argument
464 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_pd()
468 	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_dealloc_pd()
556 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_xrcd()  argument
558 	struct ib_uverbs_device *ibudev = attrs->ufile->device;  in ib_uverbs_open_xrcd()
569 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_xrcd()
597 	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,  in ib_uverbs_open_xrcd()
605 		xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);  in ib_uverbs_open_xrcd()
634 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_xrcd()
643 	return uobj_alloc_commit(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
653 	ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs));  in ib_uverbs_open_xrcd()
656 	uobj_alloc_abort(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
667 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_close_xrcd()  argument
672 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_close_xrcd()
676 	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);  in ib_uverbs_close_xrcd()
681 			   struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_xrcd()  argument
685 	struct ib_uverbs_device *dev = attrs->ufile->device;  in ib_uverbs_dealloc_xrcd()
691 	ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);  in ib_uverbs_dealloc_xrcd()
704 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_reg_mr()  argument
714 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_reg_mr()
725 	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);  in ib_uverbs_reg_mr()
729 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_reg_mr()
736 		if (!(pd->device->attrs.device_cap_flags &  in ib_uverbs_reg_mr()
746 					 &attrs->driver_udata);  in ib_uverbs_reg_mr()
769 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_reg_mr()
775 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_reg_mr()
778 	ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));  in ib_uverbs_reg_mr()
784 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_reg_mr()
788 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_rereg_mr()  argument
798 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_rereg_mr()
810 	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_rereg_mr()
829 				       attrs);  in ib_uverbs_rereg_mr()
840 					    &attrs->driver_udata);  in ib_uverbs_rereg_mr()
854 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_rereg_mr()
866 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dereg_mr()  argument
871 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dereg_mr()
875 	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_dereg_mr()
878 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_mw()  argument
888 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_mw()
892 	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);  in ib_uverbs_alloc_mw()
896 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_alloc_mw()
907 	mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata);  in ib_uverbs_alloc_mw()
924 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_mw()
929 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_alloc_mw()
936 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_mw()
940 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_mw()  argument
945 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_mw()
949 	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);  in ib_uverbs_dealloc_mw()
952 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_comp_channel()  argument
961 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_comp_channel()
965 	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);  in ib_uverbs_create_comp_channel()
975 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_comp_channel()
977 		uobj_alloc_abort(uobj, attrs);  in ib_uverbs_create_comp_channel()
981 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_create_comp_channel()
984 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,  in create_cq()  argument
995 	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)  in create_cq()
998 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,  in create_cq()
1004 		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);  in create_cq()
1033 	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);  in create_cq()
1041 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_cq()
1046 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in create_cq()
1050 	ret = uobj_alloc_commit(&obj->uobject, attrs);  in create_cq()
1056 	ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));  in create_cq()
1062 		ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);  in create_cq()
1065 	uobj_alloc_abort(&obj->uobject, attrs);  in create_cq()
1070 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_cq()  argument
1077 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_cq()
1087 	obj = create_cq(attrs, &cmd_ex);  in ib_uverbs_create_cq()
1091 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_cq()  argument
1097 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_cq()
1107 	obj = create_cq(attrs, &cmd);  in ib_uverbs_ex_create_cq()
1111 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_resize_cq()  argument
1118 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_resize_cq()
1122 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_resize_cq()
1126 	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);  in ib_uverbs_resize_cq()
1132 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_resize_cq()
1169 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_poll_cq()  argument
1179 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_poll_cq()
1183 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_poll_cq()
1188 	header_ptr = attrs->ucore.outbuf;  in ib_uverbs_poll_cq()
1213 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in ib_uverbs_poll_cq()
1214 		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);  in ib_uverbs_poll_cq()
1221 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_req_notify_cq()  argument
1227 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_req_notify_cq()
1231 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_req_notify_cq()
1243 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_cq()  argument
1251 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_cq()
1255 	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_destroy_cq()
1266 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_cq()
1269 static int create_qp(struct uverbs_attr_bundle *attrs,  in create_qp()  argument
1290 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in create_qp()
1301 					    cmd->rwq_ind_tbl_handle, attrs);  in create_qp()
1320 					  attrs);  in create_qp()
1340 							cmd->srq_handle, attrs);  in create_qp()
1351 						cmd->recv_cq_handle, attrs);  in create_qp()
1362 						cmd->send_cq_handle, attrs);  in create_qp()
1366 				       attrs);  in create_qp()
1376 	attr.qp_context    = attrs->ufile;  in create_qp()
1421 		qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,  in create_qp()
1468 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_qp()
1470 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in create_qp()
1492 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);  in create_qp()
1494 	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));  in create_qp()
1510 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in create_qp()
1514 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_qp()  argument
1520 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_qp()
1539 	return create_qp(attrs, &cmd_ex);  in ib_uverbs_create_qp()
1542 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_qp()  argument
1547 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_qp()
1557 	return create_qp(attrs, &cmd);  in ib_uverbs_ex_create_qp()
1560 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_qp()  argument
1572 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_qp()
1576 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in ib_uverbs_open_qp()
1581 	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);  in ib_uverbs_open_qp()
1594 	attr.qp_context    = attrs->ufile;  in ib_uverbs_open_qp()
1615 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_qp()
1624 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1627 	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));  in ib_uverbs_open_qp()
1631 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1657 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_qp()  argument
1666 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_qp()
1677 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_query_qp()
1724 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_qp()
1770 static int modify_qp(struct uverbs_attr_bundle *attrs,  in modify_qp()  argument
1782 			       attrs);  in modify_qp()
1919 				      &attrs->driver_udata);  in modify_qp()
1929 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_qp()  argument
1934 	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));  in ib_uverbs_modify_qp()
1942 	return modify_qp(attrs, &cmd);  in ib_uverbs_modify_qp()
1945 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_qp()  argument
1949 		.response_length = uverbs_response_length(attrs, sizeof(resp))  in ib_uverbs_ex_modify_qp()
1953 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_qp()
1967 	ret = modify_qp(attrs, &cmd);  in ib_uverbs_ex_modify_qp()
1971 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_modify_qp()
1974 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_qp()  argument
1982 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_qp()
1986 	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_destroy_qp()
1996 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_qp()
2009 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_send()  argument
2025 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_send()
2043 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_send()
2081 						   user_wr->wr.ud.ah, attrs);  in ib_uverbs_post_send()
2181 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_send()
2300 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_recv()  argument
2310 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_recv()
2319 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_recv()
2337 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_recv()
2350 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_srq_recv()  argument
2360 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_srq_recv()
2369 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_post_srq_recv()
2387 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_srq_recv()
2401 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_ah()  argument
2412 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_ah()
2416 	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);  in ib_uverbs_create_ah()
2425 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_create_ah()
2449 	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);  in ib_uverbs_create_ah()
2461 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_ah()
2466 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_create_ah()
2470 			     uverbs_get_cleared_udata(attrs));  in ib_uverbs_create_ah()
2476 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_create_ah()
2480 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_ah()  argument
2485 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_ah()
2489 	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);  in ib_uverbs_destroy_ah()
2492 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_attach_mcast()  argument
2500 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_attach_mcast()
2504 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_attach_mcast()
2540 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_detach_mcast()  argument
2549 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_detach_mcast()
2553 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_detach_mcast()
2658 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec_action()  argument
2687 							attrs);  in kern_spec_to_ib_spec_action()
2705 					  attrs);  in kern_spec_to_ib_spec_action()
2886 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec()  argument
2895 		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,  in kern_spec_to_ib_spec()
2901 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_wq()  argument
2913 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_wq()
2920 	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,  in ib_uverbs_ex_create_wq()
2925 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_ex_create_wq()
2931 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_create_wq()
2940 	wq_init_attr.wq_context = attrs->ufile;  in ib_uverbs_ex_create_wq()
2947 	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);  in ib_uverbs_ex_create_wq()
2971 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_wq()
2972 	err = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_wq()
2978 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2981 	ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs));  in ib_uverbs_ex_create_wq()
2987 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2992 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_wq()  argument
3000 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_wq()
3007 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
3008 	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_destroy_wq()
3017 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
3020 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_wq()  argument
3027 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_wq()
3037 	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_modify_wq()
3048 					&attrs->driver_udata);  in ib_uverbs_ex_modify_wq()
3053 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_rwq_ind_table()  argument
3069 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_rwq_ind_table()
3103 				       wqs_handles[num_read_wqs], attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3112 	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);  in ib_uverbs_ex_create_rwq_ind_table()
3122 						       &attrs->driver_udata);  in ib_uverbs_ex_create_rwq_ind_table()
3141 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3143 	err = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3152 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3157 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3167 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_rwq_ind_table()  argument
3172 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_rwq_ind_table()
3180 				    cmd.ind_tbl_handle, attrs);  in ib_uverbs_ex_destroy_rwq_ind_table()
3183 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_flow()  argument
3200 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_flow()
3248 	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);  in ib_uverbs_ex_create_flow()
3254 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_ex_create_flow()
3291 				attrs, (struct ib_uverbs_flow_spec *)kern_spec,  in ib_uverbs_ex_create_flow()
3310 		qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata);  in ib_uverbs_ex_create_flow()
3322 	err = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_flow()
3330 	return uobj_alloc_commit(uobj, attrs);  in ib_uverbs_ex_create_flow()
3341 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_flow()
3348 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_flow()  argument
3353 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_flow()
3360 	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);  in ib_uverbs_ex_destroy_flow()
3363 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,  in __uverbs_create_xsrq()  argument
3376 	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,  in __uverbs_create_xsrq()
3386 					  attrs);  in __uverbs_create_xsrq()
3404 						cmd->cq_handle, attrs);  in __uverbs_create_xsrq()
3411 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);  in __uverbs_create_xsrq()
3418 	attr.srq_context    = attrs->ufile;  in __uverbs_create_xsrq()
3467 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in __uverbs_create_xsrq()
3478 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3481 	ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs));  in __uverbs_create_xsrq()
3500 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3504 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_srq()  argument
3510 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_srq()
3523 	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);  in ib_uverbs_create_srq()
3526 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_xsrq()  argument
3531 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_xsrq()
3535 	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);  in ib_uverbs_create_xsrq()
3538 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_srq()  argument
3545 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_modify_srq()
3549 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_modify_srq()
3557 					  &attrs->driver_udata);  in ib_uverbs_modify_srq()
3564 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_srq()  argument
3572 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_srq()
3576 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_query_srq()
3593 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_srq()
3596 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_srq()  argument
3604 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_srq()
3608 	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_destroy_srq()
3618 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_srq()
3621 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_query_device()  argument
3630 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_ex_query_device()
3635 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_query_device()
3645 	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);  in ib_uverbs_ex_query_device()
3680 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_query_device()
3682 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_query_device()
3685 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_cq()  argument
3691 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_cq()
3701 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_modify_cq()