42 	void			*pdu;  member
87 	void			*pdu;  member
358 		void *pdu, size_t len)  in nvme_tcp_hdgst()  argument
362 	sg_init_one(&sg, pdu, len);  in nvme_tcp_hdgst()
363 	ahash_request_set_crypt(hash, &sg, pdu + len, len);  in nvme_tcp_hdgst()
368 		void *pdu, size_t pdu_len)  in nvme_tcp_verify_hdgst()  argument
370 	struct nvme_tcp_hdr *hdr = pdu;  in nvme_tcp_verify_hdgst()
381 	recv_digest = *(__le32 *)(pdu + hdr->hlen);  in nvme_tcp_verify_hdgst()
382 	nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);  in nvme_tcp_verify_hdgst()
383 	exp_digest = *(__le32 *)(pdu + hdr->hlen);  in nvme_tcp_verify_hdgst()
394 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)  in nvme_tcp_check_ddgst()  argument
396 	struct nvme_tcp_hdr *hdr = pdu;  in nvme_tcp_check_ddgst()
419 	page_frag_free(req->pdu);  in nvme_tcp_exit_request()
428 	struct nvme_tcp_cmd_pdu *pdu;  in nvme_tcp_init_request()  local
433 	req->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_init_request()
436 	if (!req->pdu)  in nvme_tcp_init_request()
439 	pdu = req->pdu;  in nvme_tcp_init_request()
442 	nvme_req(rq)->cmd = &pdu->cmd;  in nvme_tcp_init_request()
520 		struct nvme_tcp_data_pdu *pdu)  in nvme_tcp_handle_c2h_data()  argument
524 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_c2h_data()
528 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_c2h_data()
539 	queue->data_remaining = le32_to_cpu(pdu->data_length);  in nvme_tcp_handle_c2h_data()
541 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&  in nvme_tcp_handle_c2h_data()
542 	    unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {  in nvme_tcp_handle_c2h_data()
554 		struct nvme_tcp_rsp_pdu *pdu)  in nvme_tcp_handle_comp()  argument
556 	struct nvme_completion *cqe = &pdu->cqe;  in nvme_tcp_handle_comp()
576 		struct nvme_tcp_r2t_pdu *pdu)  in nvme_tcp_setup_h2c_data_pdu()  argument
578 	struct nvme_tcp_data_pdu *data = req->pdu;  in nvme_tcp_setup_h2c_data_pdu()
584 	req->pdu_len = le32_to_cpu(pdu->r2t_length);  in nvme_tcp_setup_h2c_data_pdu()
602 	if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {  in nvme_tcp_setup_h2c_data_pdu()
605 			rq->tag, le32_to_cpu(pdu->r2t_offset),  in nvme_tcp_setup_h2c_data_pdu()
621 	data->ttag = pdu->ttag;  in nvme_tcp_setup_h2c_data_pdu()
623 	data->data_offset = pdu->r2t_offset;  in nvme_tcp_setup_h2c_data_pdu()
629 		struct nvme_tcp_r2t_pdu *pdu)  in nvme_tcp_handle_r2t()  argument
635 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_r2t()
639 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_r2t()
644 	ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);  in nvme_tcp_handle_r2t()
660 	char *pdu = queue->pdu;  in nvme_tcp_recv_pdu()  local
665 		&pdu[queue->pdu_offset], rcv_len);  in nvme_tcp_recv_pdu()
676 	hdr = queue->pdu;  in nvme_tcp_recv_pdu()
678 		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);  in nvme_tcp_recv_pdu()
685 		ret = nvme_tcp_check_ddgst(queue, queue->pdu);  in nvme_tcp_recv_pdu()
692 		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
695 		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
698 		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
717 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_data()  local
719 		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_recv_data()
773 			if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {  in nvme_tcp_recv_data()
788 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_ddgst()  local
806 					pdu->command_id);  in nvme_tcp_recv_ddgst()
817 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {  in nvme_tcp_recv_ddgst()
819 					pdu->command_id);  in nvme_tcp_recv_ddgst()
985 	struct nvme_tcp_cmd_pdu *pdu = req->pdu;  in nvme_tcp_try_send_cmd_pdu()  local
988 	int len = sizeof(*pdu) + hdgst - req->offset;  in nvme_tcp_try_send_cmd_pdu()
998 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvme_tcp_try_send_cmd_pdu()
1000 	ret = kernel_sendpage(queue->sock, virt_to_page(pdu),  in nvme_tcp_try_send_cmd_pdu()
1001 			offset_in_page(pdu) + req->offset, len,  flags);  in nvme_tcp_try_send_cmd_pdu()
1024 	struct nvme_tcp_data_pdu *pdu = req->pdu;  in nvme_tcp_try_send_data_pdu()  local
1026 	int len = sizeof(*pdu) - req->offset + hdgst;  in nvme_tcp_try_send_data_pdu()
1030 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvme_tcp_try_send_data_pdu()
1032 	ret = kernel_sendpage(queue->sock, virt_to_page(pdu),  in nvme_tcp_try_send_data_pdu()
1033 			offset_in_page(pdu) + req->offset, len,  in nvme_tcp_try_send_data_pdu()
1214 	page_frag_free(async->pdu);  in nvme_tcp_free_async_req()
1223 	async->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_alloc_async_req()
1226 	if (!async->pdu)  in nvme_tcp_alloc_async_req()
1245 	kfree(queue->pdu);  in nvme_tcp_free_queue()
1501 	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);  in nvme_tcp_alloc_queue()
1502 	if (!queue->pdu) {  in nvme_tcp_alloc_queue()
1544 	kfree(queue->pdu);  in nvme_tcp_alloc_queue()
1651 	if (to_tcp_ctrl(ctrl)->async_req.pdu) {  in nvme_tcp_free_admin_queue()
1654 		to_tcp_ctrl(ctrl)->async_req.pdu = NULL;  in nvme_tcp_free_admin_queue()
2212 	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;  in nvme_tcp_submit_async_event()  local
2213 	struct nvme_command *cmd = &pdu->cmd;  in nvme_tcp_submit_async_event()
2216 	memset(pdu, 0, sizeof(*pdu));  in nvme_tcp_submit_async_event()
2217 	pdu->hdr.type = nvme_tcp_cmd;  in nvme_tcp_submit_async_event()
2219 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvme_tcp_submit_async_event()
2220 	pdu->hdr.hlen = sizeof(*pdu);  in nvme_tcp_submit_async_event()
2221 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvme_tcp_submit_async_event()
2253 	struct nvme_tcp_cmd_pdu *pdu = req->pdu;  in nvme_tcp_timeout()  local
2257 		nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);  in nvme_tcp_timeout()
2289 	struct nvme_tcp_cmd_pdu *pdu = req->pdu;  in nvme_tcp_map_data()  local
2290 	struct nvme_command *c = &pdu->cmd;  in nvme_tcp_map_data()
2309 	struct nvme_tcp_cmd_pdu *pdu = req->pdu;  in nvme_tcp_setup_cmd_pdu()  local
2334 	pdu->hdr.type = nvme_tcp_cmd;  in nvme_tcp_setup_cmd_pdu()
2335 	pdu->hdr.flags = 0;  in nvme_tcp_setup_cmd_pdu()
2337 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvme_tcp_setup_cmd_pdu()
2339 		pdu->hdr.flags |= NVME_TCP_F_DDGST;  in nvme_tcp_setup_cmd_pdu()
2342 	pdu->hdr.hlen = sizeof(*pdu);  in nvme_tcp_setup_cmd_pdu()
2343 	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;  in nvme_tcp_setup_cmd_pdu()
2344 	pdu->hdr.plen =  in nvme_tcp_setup_cmd_pdu()
2345 		cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);  in nvme_tcp_setup_cmd_pdu()