245 		       int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)  in set_rq_size()  argument
261 		if (ucmd) {  in set_rq_size()
262 			qp->rq.wqe_cnt = ucmd->rq_wqe_count;  in set_rq_size()
263 			if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))  in set_rq_size()
265 			qp->rq.wqe_shift = ucmd->rq_wqe_shift;  in set_rq_size()
440 			    struct mlx5_ib_create_qp *ucmd,  in set_user_buf_size()  argument
452 	if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) {  in set_user_buf_size()
454 			     ucmd->sq_wqe_count, ucmd->sq_wqe_count);  in set_user_buf_size()
458 	qp->sq.wqe_cnt = ucmd->sq_wqe_count;  in set_user_buf_size()
713 			  struct mlx5_ib_create_wq *ucmd)  in create_user_rq()  argument
722 	if (!ucmd->buf_addr)  in create_user_rq()
726 	rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,  in create_user_rq()
734 	mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,  in create_user_rq()
736 	err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,  in create_user_rq()
746 	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);  in create_user_rq()
749 		    (unsigned long long)ucmd->buf_addr, rwq->buf_size,  in create_user_rq()
752 	err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);  in create_user_rq()
781 	struct mlx5_ib_create_qp ucmd;  in create_user_qp()  local
793 	err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));  in create_user_qp()
800 	if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {  in create_user_qp()
802 						ucmd.bfreg_index, true);  in create_user_qp()
829 	err = set_user_buf_size(dev, qp, &ucmd, base, attr);  in create_user_qp()
833 	if (ucmd.buf_addr && ubuffer->buf_size) {  in create_user_qp()
834 		ubuffer->buf_addr = ucmd.buf_addr;  in create_user_qp()
869 	err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);  in create_user_qp()
1411 	struct mlx5_ib_create_qp_rss ucmd = {};  in create_rss_raw_qp_tir()  local
1424 	required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);  in create_rss_raw_qp_tir()
1430 	if (udata->inlen > sizeof(ucmd) &&  in create_rss_raw_qp_tir()
1431 	    !ib_is_udata_cleared(udata, sizeof(ucmd),  in create_rss_raw_qp_tir()
1432 				 udata->inlen - sizeof(ucmd))) {  in create_rss_raw_qp_tir()
1437 	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {  in create_rss_raw_qp_tir()
1442 	if (ucmd.comp_mask) {  in create_rss_raw_qp_tir()
1447 	if (ucmd.flags & ~MLX5_QP_FLAG_TUNNEL_OFFLOADS) {  in create_rss_raw_qp_tir()
1452 	if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS &&  in create_rss_raw_qp_tir()
1458 	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER &&  in create_rss_raw_qp_tir()
1459 	    !(ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {  in create_rss_raw_qp_tir()
1484 	if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)  in create_rss_raw_qp_tir()
1487 	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER)  in create_rss_raw_qp_tir()
1492 	switch (ucmd.rx_hash_function) {  in create_rss_raw_qp_tir()
1498 		if (len != ucmd.rx_key_len) {  in create_rss_raw_qp_tir()
1505 		memcpy(rss_key, ucmd.rx_hash_key, len);  in create_rss_raw_qp_tir()
1513 	if (!ucmd.rx_hash_fields_mask) {  in create_rss_raw_qp_tir()
1521 	if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||  in create_rss_raw_qp_tir()
1522 	     (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&  in create_rss_raw_qp_tir()
1523 	     ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||  in create_rss_raw_qp_tir()
1524 	     (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {  in create_rss_raw_qp_tir()
1530 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||  in create_rss_raw_qp_tir()
1531 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))  in create_rss_raw_qp_tir()
1534 	else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||  in create_rss_raw_qp_tir()
1535 		 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))  in create_rss_raw_qp_tir()
1539 	outer_l4 = ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||  in create_rss_raw_qp_tir()
1540 		    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) << 0 |  in create_rss_raw_qp_tir()
1541 		   ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||  in create_rss_raw_qp_tir()
1542 		    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) << 1 |  in create_rss_raw_qp_tir()
1543 		   (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;  in create_rss_raw_qp_tir()
1552 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||  in create_rss_raw_qp_tir()
1553 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))  in create_rss_raw_qp_tir()
1556 	else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||  in create_rss_raw_qp_tir()
1557 		 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))  in create_rss_raw_qp_tir()
1561 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||  in create_rss_raw_qp_tir()
1562 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))  in create_rss_raw_qp_tir()
1565 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||  in create_rss_raw_qp_tir()
1566 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))  in create_rss_raw_qp_tir()
1569 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||  in create_rss_raw_qp_tir()
1570 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))  in create_rss_raw_qp_tir()
1573 	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||  in create_rss_raw_qp_tir()
1574 	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))  in create_rss_raw_qp_tir()
1577 	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)  in create_rss_raw_qp_tir()
1615 	struct mlx5_ib_create_qp ucmd;  in create_qp_common()  local
1695 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {  in create_qp_common()
1701 					&ucmd, udata->inlen, &uidx);  in create_qp_common()
1705 		qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);  in create_qp_common()
1706 		qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);  in create_qp_common()
1707 		if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS) {  in create_qp_common()
1739 			  qp, (pd && pd->uobject) ? &ucmd : NULL);  in create_qp_common()
1749 			mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);  in create_qp_common()
1750 			if (ucmd.rq_wqe_shift != qp->rq.wqe_shift ||  in create_qp_common()
1751 			    ucmd.rq_wqe_count != qp->rq.wqe_cnt) {  in create_qp_common()
1755 			if (ucmd.sq_wqe_count > max_wqes) {  in create_qp_common()
1757 					    ucmd.sq_wqe_count, max_wqes);  in create_qp_common()
1912 		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;  in create_qp_common()
2170 					struct mlx5_ib_create_qp *ucmd)  in mlx5_ib_create_dct()  argument
2181 				ucmd, sizeof(*ucmd), &uidx);  in mlx5_ib_create_dct()
2200 	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);  in mlx5_ib_create_dct()
2213 			   struct mlx5_ib_create_qp *ucmd,  in set_mlx_qp_type()  argument
2222 	if (udata->inlen < sizeof(*ucmd)) {  in set_mlx_qp_type()
2226 	err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));  in set_mlx_qp_type()
2230 	if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {  in set_mlx_qp_type()
2233 		if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {  in set_mlx_qp_type()
2284 		struct mlx5_ib_create_qp ucmd;  in mlx5_ib_create_qp()  local
2288 		err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);  in mlx5_ib_create_qp()
2299 			return mlx5_ib_create_dct(pd, init_attr, &ucmd);  in mlx5_ib_create_qp()
2915 			       const struct mlx5_ib_modify_qp *ucmd)  in __mlx5_ib_modify_qp()  argument
3155 			if (ucmd->burst_info.max_burst_sz) {  in __mlx5_ib_modify_qp()
3159 						ucmd->burst_info.max_burst_sz;  in __mlx5_ib_modify_qp()
3166 			if (ucmd->burst_info.typical_pkt_sz) {  in __mlx5_ib_modify_qp()
3170 						ucmd->burst_info.typical_pkt_sz;  in __mlx5_ib_modify_qp()
3364 	struct mlx5_ib_modify_qp ucmd = {};  in mlx5_ib_modify_qp()  local
3376 		required_cmd_sz = offsetof(typeof(ucmd), reserved) +  in mlx5_ib_modify_qp()
3377 			sizeof(ucmd.reserved);  in mlx5_ib_modify_qp()
3381 		if (udata->inlen > sizeof(ucmd) &&  in mlx5_ib_modify_qp()
3382 		    !ib_is_udata_cleared(udata, sizeof(ucmd),  in mlx5_ib_modify_qp()
3383 					 udata->inlen - sizeof(ucmd)))  in mlx5_ib_modify_qp()
3386 		if (ib_copy_from_udata(&ucmd, udata,  in mlx5_ib_modify_qp()
3387 				       min(udata->inlen, sizeof(ucmd))))  in mlx5_ib_modify_qp()
3390 		if (ucmd.comp_mask ||  in mlx5_ib_modify_qp()
3391 		    memchr_inv(&ucmd.reserved, 0, sizeof(ucmd.reserved)) ||  in mlx5_ib_modify_qp()
3392 		    memchr_inv(&ucmd.burst_info.reserved, 0,  in mlx5_ib_modify_qp()
3393 			       sizeof(ucmd.burst_info.reserved)))  in mlx5_ib_modify_qp()
3478 				  new_state, &ucmd);  in mlx5_ib_modify_qp()
5353 			    struct mlx5_ib_create_wq *ucmd,  in set_user_rq_size()  argument
5360 	if (!ucmd->rq_wqe_count)  in set_user_rq_size()
5363 	rwq->wqe_count = ucmd->rq_wqe_count;  in set_user_rq_size()
5364 	rwq->wqe_shift = ucmd->rq_wqe_shift;  in set_user_rq_size()
5379 	struct mlx5_ib_create_wq ucmd = {};  in prepare_user_rq()  local
5383 	required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)  in prepare_user_rq()
5384 		+ sizeof(ucmd.single_stride_log_num_of_bytes);  in prepare_user_rq()
5390 	if (udata->inlen > sizeof(ucmd) &&  in prepare_user_rq()
5391 	    !ib_is_udata_cleared(udata, sizeof(ucmd),  in prepare_user_rq()
5392 				 udata->inlen - sizeof(ucmd))) {  in prepare_user_rq()
5397 	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {  in prepare_user_rq()
5402 	if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {  in prepare_user_rq()
5405 	} else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {  in prepare_user_rq()
5410 		if ((ucmd.single_stride_log_num_of_bytes <  in prepare_user_rq()
5412 		    (ucmd.single_stride_log_num_of_bytes >  in prepare_user_rq()
5415 				    ucmd.single_stride_log_num_of_bytes,  in prepare_user_rq()
5420 		if ((ucmd.single_wqe_log_num_of_strides >  in prepare_user_rq()
5422 		     (ucmd.single_wqe_log_num_of_strides <  in prepare_user_rq()
5425 				    ucmd.single_wqe_log_num_of_strides,  in prepare_user_rq()
5431 			ucmd.single_stride_log_num_of_bytes;  in prepare_user_rq()
5432 		rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;  in prepare_user_rq()
5433 		rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;  in prepare_user_rq()
5437 	err = set_user_rq_size(dev, init_attr, &ucmd, rwq);  in prepare_user_rq()
5443 	err = create_user_rq(dev, pd, rwq, &ucmd);  in prepare_user_rq()
5450 	rwq->user_index = ucmd.user_index;  in prepare_user_rq()
5616 	struct mlx5_ib_modify_wq ucmd = {};  in mlx5_ib_modify_wq()  local
5625 	required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);  in mlx5_ib_modify_wq()
5629 	if (udata->inlen > sizeof(ucmd) &&  in mlx5_ib_modify_wq()
5630 	    !ib_is_udata_cleared(udata, sizeof(ucmd),  in mlx5_ib_modify_wq()
5631 				 udata->inlen - sizeof(ucmd)))  in mlx5_ib_modify_wq()
5634 	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))  in mlx5_ib_modify_wq()
5637 	if (ucmd.comp_mask || ucmd.reserved)  in mlx5_ib_modify_wq()