150 	union nvme_tcp_pdu	pdu;  member
274 		void *pdu, size_t len)  in nvmet_tcp_hdgst()  argument
278 	sg_init_one(&sg, pdu, len);  in nvmet_tcp_hdgst()
279 	ahash_request_set_crypt(hash, &sg, pdu + len, len);  in nvmet_tcp_hdgst()
284 	void *pdu, size_t len)  in nvmet_tcp_verify_hdgst()  argument
286 	struct nvme_tcp_hdr *hdr = pdu;  in nvmet_tcp_verify_hdgst()
296 	recv_digest = *(__le32 *)(pdu + hdr->hlen);  in nvmet_tcp_verify_hdgst()
297 	nvmet_tcp_hdgst(queue->rcv_hash, pdu, len);  in nvmet_tcp_verify_hdgst()
298 	exp_digest = *(__le32 *)(pdu + hdr->hlen);  in nvmet_tcp_verify_hdgst()
309 static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)  in nvmet_tcp_check_ddgst()  argument
311 	struct nvme_tcp_hdr *hdr = pdu;  in nvmet_tcp_check_ddgst()
429 	struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;  in nvmet_setup_c2h_data_pdu()  local
437 	pdu->hdr.type = nvme_tcp_c2h_data;  in nvmet_setup_c2h_data_pdu()
438 	pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ?  in nvmet_setup_c2h_data_pdu()
440 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_c2h_data_pdu()
441 	pdu->hdr.pdo = pdu->hdr.hlen + hdgst;  in nvmet_setup_c2h_data_pdu()
442 	pdu->hdr.plen =  in nvmet_setup_c2h_data_pdu()
443 		cpu_to_le32(pdu->hdr.hlen + hdgst +  in nvmet_setup_c2h_data_pdu()
445 	pdu->command_id = cmd->req.cqe->command_id;  in nvmet_setup_c2h_data_pdu()
446 	pdu->data_length = cpu_to_le32(cmd->req.transfer_len);  in nvmet_setup_c2h_data_pdu()
447 	pdu->data_offset = cpu_to_le32(cmd->wbytes_done);  in nvmet_setup_c2h_data_pdu()
450 		pdu->hdr.flags |= NVME_TCP_F_DDGST;  in nvmet_setup_c2h_data_pdu()
455 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_c2h_data_pdu()
456 		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvmet_setup_c2h_data_pdu()
462 	struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;  in nvmet_setup_r2t_pdu()  local
469 	pdu->hdr.type = nvme_tcp_r2t;  in nvmet_setup_r2t_pdu()
470 	pdu->hdr.flags = 0;  in nvmet_setup_r2t_pdu()
471 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_r2t_pdu()
472 	pdu->hdr.pdo = 0;  in nvmet_setup_r2t_pdu()
473 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvmet_setup_r2t_pdu()
475 	pdu->command_id = cmd->req.cmd->common.command_id;  in nvmet_setup_r2t_pdu()
476 	pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd);  in nvmet_setup_r2t_pdu()
477 	pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);  in nvmet_setup_r2t_pdu()
478 	pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);  in nvmet_setup_r2t_pdu()
480 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_r2t_pdu()
481 		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvmet_setup_r2t_pdu()
487 	struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;  in nvmet_setup_response_pdu()  local
494 	pdu->hdr.type = nvme_tcp_rsp;  in nvmet_setup_response_pdu()
495 	pdu->hdr.flags = 0;  in nvmet_setup_response_pdu()
496 	pdu->hdr.hlen = sizeof(*pdu);  in nvmet_setup_response_pdu()
497 	pdu->hdr.pdo = 0;  in nvmet_setup_response_pdu()
498 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvmet_setup_response_pdu()
500 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvmet_setup_response_pdu()
501 		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvmet_setup_response_pdu()
866 	struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq;  in nvmet_tcp_handle_icreq()
867 	struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp;  in nvmet_tcp_handle_icreq()
873 		pr_err("bad nvme-tcp pdu length (%d)\n",  in nvmet_tcp_handle_icreq()
930 	 * we don't, we can simply prepare for the next pdu and bail out,  in nvmet_tcp_handle_req_failure()
954 	struct nvme_tcp_data_pdu *data = &queue->pdu.data;  in nvmet_tcp_handle_h2c_data_pdu()
990 	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;  in nvmet_tcp_done_recv_pdu()
991 	struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;  in nvmet_tcp_done_recv_pdu()
997 			pr_err("unexpected pdu type (%d) before icreq\n",  in nvmet_tcp_done_recv_pdu()
1006 		pr_err("queue %d: received icreq pdu in state %d\n",  in nvmet_tcp_done_recv_pdu()
1101 	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;  in nvmet_tcp_try_recv_pdu()
1107 	iov.iov_base = (void *)&queue->pdu + queue->offset;  in nvmet_tcp_try_recv_pdu()
1123 			pr_err("unexpected pdu type %d\n", hdr->type);  in nvmet_tcp_try_recv_pdu()
1129 			pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);  in nvmet_tcp_try_recv_pdu()
1138 	    nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) {  in nvmet_tcp_try_recv_pdu()
1144 	    nvmet_tcp_check_ddgst(queue, &queue->pdu)) {  in nvmet_tcp_try_recv_pdu()
1210 		pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",  in nvmet_tcp_try_recv_ddgst()
1212 			queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),  in nvmet_tcp_try_recv_ddgst()