63 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_init()  local
67 	efx->mcdi = kzalloc(sizeof(*efx->mcdi), GFP_KERNEL);  in efx_mcdi_init()
68 	if (!efx->mcdi)  in efx_mcdi_init()
71 	mcdi = efx_mcdi(efx);  in efx_mcdi_init()
72 	mcdi->efx = efx;  in efx_mcdi_init()
75 	mcdi->logging_buffer = (char *)__get_free_page(GFP_KERNEL);  in efx_mcdi_init()
76 	if (!mcdi->logging_buffer)  in efx_mcdi_init()
78 	mcdi->logging_enabled = mcdi_logging_default;  in efx_mcdi_init()
80 	init_waitqueue_head(&mcdi->wq);  in efx_mcdi_init()
81 	init_waitqueue_head(&mcdi->proxy_rx_wq);  in efx_mcdi_init()
82 	spin_lock_init(&mcdi->iface_lock);  in efx_mcdi_init()
83 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_init()
84 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_init()
85 	spin_lock_init(&mcdi->async_lock);  in efx_mcdi_init()
86 	INIT_LIST_HEAD(&mcdi->async_list);  in efx_mcdi_init()
87 	timer_setup(&mcdi->async_timer, efx_mcdi_timeout_async, 0);  in efx_mcdi_init()
90 	mcdi->new_epoch = true;  in efx_mcdi_init()
109 	if (efx->mcdi->fn_flags &  in efx_mcdi_init()
116 	free_page((unsigned long)mcdi->logging_buffer);  in efx_mcdi_init()
119 	kfree(efx->mcdi);  in efx_mcdi_init()
120 	efx->mcdi = NULL;  in efx_mcdi_init()
127 	if (!efx->mcdi)  in efx_mcdi_detach()
130 	BUG_ON(efx->mcdi->iface.state != MCDI_STATE_QUIESCENT);  in efx_mcdi_detach()
138 	if (!efx->mcdi)  in efx_mcdi_fini()
142 	free_page((unsigned long)efx->mcdi->iface.logging_buffer);  in efx_mcdi_fini()
145 	kfree(efx->mcdi);  in efx_mcdi_fini()
151 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_send_request()  local
153 	char *buf = mcdi->logging_buffer; /* page-sized */  in efx_mcdi_send_request()
159 	BUG_ON(mcdi->state == MCDI_STATE_QUIESCENT);  in efx_mcdi_send_request()
162 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
163 	++mcdi->seqno;  in efx_mcdi_send_request()
164 	seqno = mcdi->seqno & SEQ_MASK;  in efx_mcdi_send_request()
165 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
168 	if (mcdi->mode == MCDI_MODE_EVENTS)  in efx_mcdi_send_request()
180 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
192 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
200 	if (mcdi->logging_enabled && !WARN_ON_ONCE(!buf)) {  in efx_mcdi_send_request()
228 	mcdi->new_epoch = false;  in efx_mcdi_send_request()
265 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_read_response_header()  local
268 	char *buf = mcdi->logging_buffer; /* page-sized */  in efx_mcdi_read_response_header()
278 		mcdi->resp_hdr_len = 4;  in efx_mcdi_read_response_header()
279 		mcdi->resp_data_len = EFX_DWORD_FIELD(hdr, MCDI_HEADER_DATALEN);  in efx_mcdi_read_response_header()
282 		mcdi->resp_hdr_len = 8;  in efx_mcdi_read_response_header()
283 		mcdi->resp_data_len =  in efx_mcdi_read_response_header()
288 	if (mcdi->logging_enabled && !WARN_ON_ONCE(!buf)) {  in efx_mcdi_read_response_header()
293 		WARN_ON_ONCE(mcdi->resp_hdr_len % 4);  in efx_mcdi_read_response_header()
294 		hdr_len = mcdi->resp_hdr_len / 4;  in efx_mcdi_read_response_header()
298 		data_len = DIV_ROUND_UP(mcdi->resp_data_len, 4);  in efx_mcdi_read_response_header()
311 					mcdi->resp_hdr_len + (i * 4), 4);  in efx_mcdi_read_response_header()
320 	mcdi->resprc_raw = 0;  in efx_mcdi_read_response_header()
321 	if (error && mcdi->resp_data_len == 0) {  in efx_mcdi_read_response_header()
323 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
324 	} else if ((respseq ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_read_response_header()
327 			  respseq, mcdi->seqno);  in efx_mcdi_read_response_header()
328 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
330 		efx->type->mcdi_read_response(efx, &hdr, mcdi->resp_hdr_len, 4);  in efx_mcdi_read_response_header()
331 		mcdi->resprc_raw = EFX_DWORD_FIELD(hdr, EFX_DWORD_0);  in efx_mcdi_read_response_header()
332 		mcdi->resprc = efx_mcdi_errno(mcdi->resprc_raw);  in efx_mcdi_read_response_header()
334 		mcdi->resprc = 0;  in efx_mcdi_read_response_header()
340 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll_once()  local
346 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
348 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
355 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll()  local
363 		spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
364 		mcdi->resprc = rc;  in efx_mcdi_poll()
365 		mcdi->resp_hdr_len = 0;  in efx_mcdi_poll()
366 		mcdi->resp_data_len = 0;  in efx_mcdi_poll()
367 		spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
404 	if (!efx->mcdi)  in efx_mcdi_poll_reboot()
410 static bool efx_mcdi_acquire_async(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_async()  argument
412 	return cmpxchg(&mcdi->state,  in efx_mcdi_acquire_async()
417 static void efx_mcdi_acquire_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_sync()  argument
422 	wait_event(mcdi->wq,  in efx_mcdi_acquire_sync()
423 		   cmpxchg(&mcdi->state,  in efx_mcdi_acquire_sync()
430 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_await_completion()  local
432 	if (wait_event_timeout(mcdi->wq, mcdi->state == MCDI_STATE_COMPLETED,  in efx_mcdi_await_completion()
444 	if (mcdi->mode == MCDI_MODE_POLL)  in efx_mcdi_await_completion()
453 static bool efx_mcdi_complete_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_complete_sync()  argument
455 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_sync()
458 		wake_up(&mcdi->wq);  in efx_mcdi_complete_sync()
465 static void efx_mcdi_release(struct efx_mcdi_iface *mcdi)  in efx_mcdi_release()  argument
467 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_release()
469 		struct efx_nic *efx = mcdi->efx;  in efx_mcdi_release()
472 		spin_lock_bh(&mcdi->async_lock);  in efx_mcdi_release()
474 			&mcdi->async_list, struct efx_mcdi_async_param, list);  in efx_mcdi_release()
476 			mcdi->state = MCDI_STATE_RUNNING_ASYNC;  in efx_mcdi_release()
480 			mod_timer(&mcdi->async_timer,  in efx_mcdi_release()
483 		spin_unlock_bh(&mcdi->async_lock);  in efx_mcdi_release()
489 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_release()
490 	wake_up(&mcdi->wq);  in efx_mcdi_release()
498 static bool efx_mcdi_complete_async(struct efx_mcdi_iface *mcdi, bool timeout)  in efx_mcdi_complete_async()  argument
500 	struct efx_nic *efx = mcdi->efx;  in efx_mcdi_complete_async()
507 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_async()
512 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
517 		++mcdi->seqno;  in efx_mcdi_complete_async()
518 		++mcdi->credits;  in efx_mcdi_complete_async()
523 		rc = mcdi->resprc;  in efx_mcdi_complete_async()
524 		hdr_len = mcdi->resp_hdr_len;  in efx_mcdi_complete_async()
525 		data_len = mcdi->resp_data_len;  in efx_mcdi_complete_async()
527 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
534 		del_timer_sync(&mcdi->async_timer);  in efx_mcdi_complete_async()
536 	spin_lock(&mcdi->async_lock);  in efx_mcdi_complete_async()
537 	async = list_first_entry(&mcdi->async_list,  in efx_mcdi_complete_async()
540 	spin_unlock(&mcdi->async_lock);  in efx_mcdi_complete_async()
558 	efx_mcdi_release(mcdi);  in efx_mcdi_complete_async()
566 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_cpl()  local
569 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
571 	if ((seqno ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_ev_cpl()
572 		if (mcdi->credits)  in efx_mcdi_ev_cpl()
574 			--mcdi->credits;  in efx_mcdi_ev_cpl()
578 				  "seq 0x%x\n", seqno, mcdi->seqno);  in efx_mcdi_ev_cpl()
584 			mcdi->resprc = efx_mcdi_errno(mcdi_err);  in efx_mcdi_ev_cpl()
585 			mcdi->resp_hdr_len = 4;  in efx_mcdi_ev_cpl()
586 			mcdi->resp_data_len = datalen;  in efx_mcdi_ev_cpl()
592 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
595 		if (!efx_mcdi_complete_async(mcdi, false))  in efx_mcdi_ev_cpl()
596 			(void) efx_mcdi_complete_sync(mcdi);  in efx_mcdi_ev_cpl()
609 	struct efx_mcdi_iface *mcdi = from_timer(mcdi, t, async_timer);  in efx_mcdi_timeout_async()  local
611 	efx_mcdi_complete_async(mcdi, true);  in efx_mcdi_timeout_async()
655 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_finish()  local
659 	if (mcdi->mode == MCDI_MODE_POLL)  in _efx_mcdi_rpc_finish()
667 			  cmd, (int)inlen, mcdi->mode);  in _efx_mcdi_rpc_finish()
669 		if (mcdi->mode == MCDI_MODE_EVENTS && efx_mcdi_poll_once(efx)) {  in _efx_mcdi_rpc_finish()
681 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
682 		++mcdi->seqno;  in _efx_mcdi_rpc_finish()
683 		++mcdi->credits;  in _efx_mcdi_rpc_finish()
684 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
700 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
701 		rc = mcdi->resprc;  in _efx_mcdi_rpc_finish()
703 			*raw_rc = mcdi->resprc_raw;  in _efx_mcdi_rpc_finish()
704 		hdr_len = mcdi->resp_hdr_len;  in _efx_mcdi_rpc_finish()
705 		data_len = mcdi->resp_data_len;  in _efx_mcdi_rpc_finish()
707 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
730 			mcdi->proxy_rx_status = 0;  in _efx_mcdi_rpc_finish()
731 			mcdi->proxy_rx_handle = 0;  in _efx_mcdi_rpc_finish()
732 			mcdi->state = MCDI_STATE_PROXY_WAIT;  in _efx_mcdi_rpc_finish()
741 			mcdi->new_epoch = true;  in _efx_mcdi_rpc_finish()
746 		efx_mcdi_release(mcdi);  in _efx_mcdi_rpc_finish()
750 static void efx_mcdi_proxy_abort(struct efx_mcdi_iface *mcdi)  in efx_mcdi_proxy_abort()  argument
752 	if (mcdi->state == MCDI_STATE_PROXY_WAIT) {  in efx_mcdi_proxy_abort()
754 		mcdi->proxy_rx_status = -EINTR;  in efx_mcdi_proxy_abort()
755 		wake_up(&mcdi->proxy_rx_wq);  in efx_mcdi_proxy_abort()
762 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_proxy_response()  local
764 	WARN_ON(mcdi->state != MCDI_STATE_PROXY_WAIT);  in efx_mcdi_ev_proxy_response()
766 	mcdi->proxy_rx_status = efx_mcdi_errno(status);  in efx_mcdi_ev_proxy_response()
771 	mcdi->proxy_rx_handle = handle;  in efx_mcdi_ev_proxy_response()
772 	wake_up(&mcdi->proxy_rx_wq);  in efx_mcdi_ev_proxy_response()
777 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_proxy_wait()  local
781 	rc = wait_event_timeout(mcdi->proxy_rx_wq,  in efx_mcdi_proxy_wait()
782 				mcdi->proxy_rx_handle != 0 ||  in efx_mcdi_proxy_wait()
783 				mcdi->proxy_rx_status == -EINTR,  in efx_mcdi_proxy_wait()
790 	} else if (mcdi->proxy_rx_handle != handle) {  in efx_mcdi_proxy_wait()
793 			   mcdi->proxy_rx_handle, handle);  in efx_mcdi_proxy_wait()
797 	return mcdi->proxy_rx_status;  in efx_mcdi_proxy_wait()
826 		struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc()  local
838 			mcdi->state = MCDI_STATE_RUNNING_SYNC;  in _efx_mcdi_rpc()
851 			efx_mcdi_release(mcdi);  in _efx_mcdi_rpc()
954 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_rpc_start()  local
964 	if (mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_rpc_start()
967 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_rpc_start()
978 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_async()  local
1002 	spin_lock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
1004 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in _efx_mcdi_rpc_async()
1005 		list_add_tail(&async->list, &mcdi->async_list);  in _efx_mcdi_rpc_async()
1010 		if (mcdi->async_list.next == &async->list &&  in _efx_mcdi_rpc_async()
1011 		    efx_mcdi_acquire_async(mcdi)) {  in _efx_mcdi_rpc_async()
1013 			mod_timer(&mcdi->async_timer,  in _efx_mcdi_rpc_async()
1021 	spin_unlock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
1101 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_poll()  local
1103 	if (!efx->mcdi)  in efx_mcdi_mode_poll()
1106 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_poll()
1111 	if (mcdi->mode == MCDI_MODE_POLL || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_poll()
1122 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_mode_poll()
1124 	efx_mcdi_complete_sync(mcdi);  in efx_mcdi_mode_poll()
1133 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_flush_async()  local
1135 	if (!efx->mcdi)  in efx_mcdi_flush_async()
1138 	mcdi = efx_mcdi(efx);  in efx_mcdi_flush_async()
1141 	BUG_ON(mcdi->mode == MCDI_MODE_EVENTS);  in efx_mcdi_flush_async()
1143 	del_timer_sync(&mcdi->async_timer);  in efx_mcdi_flush_async()
1149 	if (mcdi->state == MCDI_STATE_RUNNING_ASYNC) {  in efx_mcdi_flush_async()
1151 		mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_flush_async()
1159 	list_for_each_entry_safe(async, next, &mcdi->async_list, list) {  in efx_mcdi_flush_async()
1169 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_event()  local
1171 	if (!efx->mcdi)  in efx_mcdi_mode_event()
1174 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_event()
1179 	if (mcdi->mode == MCDI_MODE_EVENTS || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_event()
1189 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_mode_event()
1190 	mcdi->mode = MCDI_MODE_EVENTS;  in efx_mcdi_mode_event()
1191 	efx_mcdi_release(mcdi);  in efx_mcdi_mode_event()
1196 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_death()  local
1222 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
1223 	efx_mcdi_proxy_abort(mcdi);  in efx_mcdi_ev_death()
1225 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_death()
1226 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_death()
1227 			mcdi->resprc = rc;  in efx_mcdi_ev_death()
1228 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_death()
1229 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_death()
1230 			++mcdi->credits;  in efx_mcdi_ev_death()
1252 		mcdi->new_epoch = true;  in efx_mcdi_ev_death()
1258 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
1268 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_bist()  local
1270 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
1272 	efx_mcdi_proxy_abort(mcdi);  in efx_mcdi_ev_bist()
1274 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_bist()
1275 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_bist()
1276 			mcdi->resprc = -EIO;  in efx_mcdi_ev_bist()
1277 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_bist()
1278 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_bist()
1279 			++mcdi->credits;  in efx_mcdi_ev_bist()
1282 	mcdi->new_epoch = true;  in efx_mcdi_ev_bist()
1284 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
1292 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_abandon()  local
1294 	if (xchg(&mcdi->mode, MCDI_MODE_FAIL) == MCDI_MODE_FAIL)  in efx_mcdi_abandon()
1493 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1498 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1930 		if (efx->mcdi) {  in efx_mcdi_reset()
1931 			struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_reset()  local
1932 			mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_reset()