432 	struct irdma_cqp_request *cqp_request = NULL;  in irdma_alloc_and_get_cqp_request()  local
437 		cqp_request = list_first_entry(&cqp->cqp_avail_reqs,  in irdma_alloc_and_get_cqp_request()
439 		list_del_init(&cqp_request->list);  in irdma_alloc_and_get_cqp_request()
442 	if (!cqp_request) {  in irdma_alloc_and_get_cqp_request()
443 		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);  in irdma_alloc_and_get_cqp_request()
444 		if (cqp_request) {  in irdma_alloc_and_get_cqp_request()
445 			cqp_request->dynamic = true;  in irdma_alloc_and_get_cqp_request()
447 				init_waitqueue_head(&cqp_request->waitq);  in irdma_alloc_and_get_cqp_request()
450 	if (!cqp_request) {  in irdma_alloc_and_get_cqp_request()
455 	cqp_request->waiting = wait;  in irdma_alloc_and_get_cqp_request()
456 	refcount_set(&cqp_request->refcnt, 1);  in irdma_alloc_and_get_cqp_request()
457 	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));  in irdma_alloc_and_get_cqp_request()
459 	return cqp_request;  in irdma_alloc_and_get_cqp_request()
466 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)  in irdma_get_cqp_request()  argument
468 	refcount_inc(&cqp_request->refcnt);  in irdma_get_cqp_request()
477 			    struct irdma_cqp_request *cqp_request)  in irdma_free_cqp_request()  argument
481 	if (cqp_request->dynamic) {  in irdma_free_cqp_request()
482 		kfree(cqp_request);  in irdma_free_cqp_request()
484 		cqp_request->request_done = false;  in irdma_free_cqp_request()
485 		cqp_request->callback_fcn = NULL;  in irdma_free_cqp_request()
486 		cqp_request->waiting = false;  in irdma_free_cqp_request()
489 		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);  in irdma_free_cqp_request()
501 			   struct irdma_cqp_request *cqp_request)  in irdma_put_cqp_request()  argument
503 	if (refcount_dec_and_test(&cqp_request->refcnt))  in irdma_put_cqp_request()
504 		irdma_free_cqp_request(cqp, cqp_request);  in irdma_put_cqp_request()
514 			       struct irdma_cqp_request *cqp_request)  in irdma_free_pending_cqp_request()  argument
516 	if (cqp_request->waiting) {  in irdma_free_pending_cqp_request()
517 		cqp_request->compl_info.error = true;  in irdma_free_pending_cqp_request()
518 		cqp_request->request_done = true;  in irdma_free_pending_cqp_request()
519 		wake_up(&cqp_request->waitq);  in irdma_free_pending_cqp_request()
522 			   refcount_read(&cqp_request->refcnt) == 1, 1000);  in irdma_free_pending_cqp_request()
523 	irdma_put_cqp_request(cqp, cqp_request);  in irdma_free_pending_cqp_request()
535 	struct irdma_cqp_request *cqp_request = NULL;  in irdma_cleanup_pending_cqp_op()  local
542 		cqp_request = (struct irdma_cqp_request *)(unsigned long)  in irdma_cleanup_pending_cqp_op()
544 		if (cqp_request)  in irdma_cleanup_pending_cqp_op()
545 			irdma_free_pending_cqp_request(cqp, cqp_request);  in irdma_cleanup_pending_cqp_op()
551 		cqp_request =  in irdma_cleanup_pending_cqp_op()
553 		if (cqp_request)  in irdma_cleanup_pending_cqp_op()
554 			irdma_free_pending_cqp_request(cqp, cqp_request);  in irdma_cleanup_pending_cqp_op()
564 			    struct irdma_cqp_request *cqp_request)  in irdma_wait_event()  argument
573 		if (wait_event_timeout(cqp_request->waitq,  in irdma_wait_event()
574 				       cqp_request->request_done,  in irdma_wait_event()
590 	cqp_error = cqp_request->compl_info.error;  in irdma_wait_event()
593 		if (cqp_request->compl_info.maj_err_code == 0xFFFF) {  in irdma_wait_event()
594 			if (cqp_request->compl_info.min_err_code == 0x8002)  in irdma_wait_event()
596 			else if (cqp_request->compl_info.min_err_code == 0x8029) {  in irdma_wait_event()
697 			struct irdma_cqp_request *cqp_request)  in irdma_handle_cqp_op()  argument
700 	struct cqp_cmds_info *info = &cqp_request->info;  in irdma_handle_cqp_op()
707 	irdma_get_cqp_request(cqp_request);  in irdma_handle_cqp_op()
712 	if (cqp_request->waiting) {  in irdma_handle_cqp_op()
714 		status = irdma_wait_event(rf, cqp_request);  in irdma_handle_cqp_op()
723 			       cqp_request->compl_info.maj_err_code,  in irdma_handle_cqp_op()
724 			       cqp_request->compl_info.min_err_code))  in irdma_handle_cqp_op()
727 			  irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,  in irdma_handle_cqp_op()
728 			  cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,  in irdma_handle_cqp_op()
729 			  cqp_request->compl_info.min_err_code);  in irdma_handle_cqp_op()
732 		irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_handle_cqp_op()
810 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_sds_cmd()  local
815 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_sds_cmd()
816 	if (!cqp_request)  in irdma_cqp_sds_cmd()
819 	cqp_info = &cqp_request->info;  in irdma_cqp_sds_cmd()
825 	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;  in irdma_cqp_sds_cmd()
827 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_sds_cmd()
828 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_sds_cmd()
841 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_qp_suspend_resume()  local
847 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);  in irdma_cqp_qp_suspend_resume()
848 	if (!cqp_request)  in irdma_cqp_qp_suspend_resume()
851 	cqp_info = &cqp_request->info;  in irdma_cqp_qp_suspend_resume()
855 	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;  in irdma_cqp_qp_suspend_resume()
857 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_qp_suspend_resume()
858 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_qp_suspend_resume()
958 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_query_fpm_val_cmd()  local
963 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_query_fpm_val_cmd()
964 	if (!cqp_request)  in irdma_cqp_query_fpm_val_cmd()
967 	cqp_info = &cqp_request->info;  in irdma_cqp_query_fpm_val_cmd()
968 	cqp_request->param = NULL;  in irdma_cqp_query_fpm_val_cmd()
975 	cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;  in irdma_cqp_query_fpm_val_cmd()
977 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_query_fpm_val_cmd()
978 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_query_fpm_val_cmd()
992 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_commit_fpm_val_cmd()  local
997 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_commit_fpm_val_cmd()
998 	if (!cqp_request)  in irdma_cqp_commit_fpm_val_cmd()
1001 	cqp_info = &cqp_request->info;  in irdma_cqp_commit_fpm_val_cmd()
1002 	cqp_request->param = NULL;  in irdma_cqp_commit_fpm_val_cmd()
1009 	cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;  in irdma_cqp_commit_fpm_val_cmd()
1011 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_commit_fpm_val_cmd()
1012 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_commit_fpm_val_cmd()
1026 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_cq_create_cmd()  local
1030 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);  in irdma_cqp_cq_create_cmd()
1031 	if (!cqp_request)  in irdma_cqp_cq_create_cmd()
1034 	cqp_info = &cqp_request->info;  in irdma_cqp_cq_create_cmd()
1038 	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;  in irdma_cqp_cq_create_cmd()
1040 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_cq_create_cmd()
1041 	irdma_put_cqp_request(iwcqp, cqp_request);  in irdma_cqp_cq_create_cmd()
1055 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_qp_create_cmd()  local
1060 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);  in irdma_cqp_qp_create_cmd()
1061 	if (!cqp_request)  in irdma_cqp_qp_create_cmd()
1064 	cqp_info = &cqp_request->info;  in irdma_cqp_qp_create_cmd()
1065 	qp_info = &cqp_request->info.in.u.qp_create.info;  in irdma_cqp_qp_create_cmd()
1072 	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;  in irdma_cqp_qp_create_cmd()
1074 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_qp_create_cmd()
1075 	irdma_put_cqp_request(iwcqp, cqp_request);  in irdma_cqp_qp_create_cmd()
1088 	struct irdma_cqp_request *cqp_request;  in irdma_dealloc_push_page()  local
1095 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);  in irdma_dealloc_push_page()
1096 	if (!cqp_request)  in irdma_dealloc_push_page()
1099 	cqp_info = &cqp_request->info;  in irdma_dealloc_push_page()
1107 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;  in irdma_dealloc_push_page()
1108 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_dealloc_push_page()
1111 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_dealloc_push_page()
1151 	struct irdma_cqp_request *cqp_request;  in irdma_cq_wq_destroy()  local
1154 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cq_wq_destroy()
1155 	if (!cqp_request)  in irdma_cq_wq_destroy()
1158 	cqp_info = &cqp_request->info;  in irdma_cq_wq_destroy()
1162 	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;  in irdma_cq_wq_destroy()
1164 	irdma_handle_cqp_op(rf, cqp_request);  in irdma_cq_wq_destroy()
1165 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cq_wq_destroy()
1172 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)  in irdma_hw_modify_qp_callback()  argument
1177 	cqp_info = &cqp_request->info;  in irdma_hw_modify_qp_callback()
1195 	struct irdma_cqp_request *cqp_request;  in irdma_hw_modify_qp()  local
1199 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);  in irdma_hw_modify_qp()
1200 	if (!cqp_request)  in irdma_hw_modify_qp()
1204 		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;  in irdma_hw_modify_qp()
1207 	cqp_info = &cqp_request->info;  in irdma_hw_modify_qp()
1213 	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;  in irdma_hw_modify_qp()
1214 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_hw_modify_qp()
1215 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_hw_modify_qp()
1236 				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,  in irdma_hw_modify_qp()
1238 				if (!cqp_request)  in irdma_hw_modify_qp()
1241 				cqp_info = &cqp_request->info;  in irdma_hw_modify_qp()
1247 				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;  in irdma_hw_modify_qp()
1250 				irdma_handle_cqp_op(rf, cqp_request);  in irdma_hw_modify_qp()
1251 				irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_hw_modify_qp()
1284 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_qp_destroy_cmd()  local
1288 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);  in irdma_cqp_qp_destroy_cmd()
1289 	if (!cqp_request)  in irdma_cqp_qp_destroy_cmd()
1292 	cqp_info = &cqp_request->info;  in irdma_cqp_qp_destroy_cmd()
1297 	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;  in irdma_cqp_qp_destroy_cmd()
1300 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_qp_destroy_cmd()
1301 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_qp_destroy_cmd()
1855 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)  in irdma_process_cqp_stats()  argument
1857 	struct irdma_vsi_pestat *pestat = cqp_request->param;  in irdma_process_cqp_stats()
1874 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_gather_stats_cmd()  local
1878 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);  in irdma_cqp_gather_stats_cmd()
1879 	if (!cqp_request)  in irdma_cqp_gather_stats_cmd()
1882 	cqp_info = &cqp_request->info;  in irdma_cqp_gather_stats_cmd()
1887 	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;  in irdma_cqp_gather_stats_cmd()
1889 	cqp_request->param = pestat;  in irdma_cqp_gather_stats_cmd()
1891 		cqp_request->callback_fcn = irdma_process_cqp_stats;  in irdma_cqp_gather_stats_cmd()
1892 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_gather_stats_cmd()
1895 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_gather_stats_cmd()
1911 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_stats_inst_cmd()  local
1918 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);  in irdma_cqp_stats_inst_cmd()
1919 	if (!cqp_request)  in irdma_cqp_stats_inst_cmd()
1922 	cqp_info = &cqp_request->info;  in irdma_cqp_stats_inst_cmd()
1927 	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;  in irdma_cqp_stats_inst_cmd()
1929 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_stats_inst_cmd()
1931 		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;  in irdma_cqp_stats_inst_cmd()
1932 	irdma_put_cqp_request(iwcqp, cqp_request);  in irdma_cqp_stats_inst_cmd()
1946 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_ceq_cmd()  local
1951 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_ceq_cmd()
1952 	if (!cqp_request)  in irdma_cqp_ceq_cmd()
1955 	cqp_info = &cqp_request->info;  in irdma_cqp_ceq_cmd()
1959 	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;  in irdma_cqp_ceq_cmd()
1961 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_ceq_cmd()
1962 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_ceq_cmd()
1976 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_aeq_cmd()  local
1981 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_aeq_cmd()
1982 	if (!cqp_request)  in irdma_cqp_aeq_cmd()
1985 	cqp_info = &cqp_request->info;  in irdma_cqp_aeq_cmd()
1989 	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;  in irdma_cqp_aeq_cmd()
1991 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_aeq_cmd()
1992 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_aeq_cmd()
2009 	struct irdma_cqp_request *cqp_request;  in irdma_cqp_ws_node_cmd()  local
2019 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);  in irdma_cqp_ws_node_cmd()
2020 	if (!cqp_request)  in irdma_cqp_ws_node_cmd()
2023 	cqp_info = &cqp_request->info;  in irdma_cqp_ws_node_cmd()
2029 	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;  in irdma_cqp_ws_node_cmd()
2030 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_ws_node_cmd()
2043 		node_info->qs_handle = cqp_request->compl_info.op_ret_val;  in irdma_cqp_ws_node_cmd()
2047 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_ws_node_cmd()
2068 	struct irdma_cqp_request *cqp_request;  in irdma_ah_cqp_op()  local
2075 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);  in irdma_ah_cqp_op()
2076 	if (!cqp_request)  in irdma_ah_cqp_op()
2079 	cqp_info = &cqp_request->info;  in irdma_ah_cqp_op()
2084 		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;  in irdma_ah_cqp_op()
2088 		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;  in irdma_ah_cqp_op()
2093 		cqp_request->callback_fcn = callback_fcn;  in irdma_ah_cqp_op()
2094 		cqp_request->param = cb_param;  in irdma_ah_cqp_op()
2096 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_ah_cqp_op()
2097 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_ah_cqp_op()
2112 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)  in irdma_ieq_ah_cb()  argument
2114 	struct irdma_sc_qp *qp = cqp_request->param;  in irdma_ieq_ah_cb()
2119 	if (!cqp_request->compl_info.op_ret_val) {  in irdma_ieq_ah_cb()
2133 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)  in irdma_ilq_ah_cb()  argument
2135 	struct irdma_cm_node *cm_node = cqp_request->param;  in irdma_ilq_ah_cb()
2138 	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;  in irdma_ilq_ah_cb()
2217 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)  in irdma_gsi_ud_qp_ah_cb()  argument
2219 	struct irdma_sc_ah *sc_ah = cqp_request->param;  in irdma_gsi_ud_qp_ah_cb()
2221 	if (!cqp_request->compl_info.op_ret_val)  in irdma_gsi_ud_qp_ah_cb()