186 	struct tls_context *tls_ctx;  in tls_decrypt_done()  local
192 	tls_ctx = tls_get_ctx(sk);  in tls_decrypt_done()
193 	ctx = tls_sw_ctx_rx(tls_ctx);  in tls_decrypt_done()
229 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_do_decryption()  local
230 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_do_decryption()
231 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_do_decryption()
265 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_trim_both_msgs()  local
266 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_trim_both_msgs()
267 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_trim_both_msgs()
278 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_alloc_encrypted_msg()  local
279 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_alloc_encrypted_msg()
288 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_clone_plaintext_msg()  local
289 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_clone_plaintext_msg()
290 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_clone_plaintext_msg()
312 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_get_rec()  local
313 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_get_rec()
314 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_get_rec()
351 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_free_open_rec()  local
352 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_free_open_rec()
363 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_tx_records()  local
364 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_tx_records()
369 	if (tls_is_partially_sent_record(tls_ctx)) {  in tls_tx_records()
378 		rc = tls_push_partial_record(sk, tls_ctx, tx_flags);  in tls_tx_records()
399 			rc = tls_push_sg(sk, tls_ctx,  in tls_tx_records()
424 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_encrypt_done()  local
425 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_encrypt_done()
426 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_encrypt_done()
482 			     struct tls_context *tls_ctx,  in tls_do_encryption()  argument
487 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_do_encryption()
505 	memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,  in tls_do_encryption()
509 			    tls_ctx->tx.rec_seq);  in tls_do_encryption()
545 	tls_advance_record_sn(sk, prot, &tls_ctx->tx);  in tls_do_encryption()
670 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_push_record()  local
671 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_push_record()
672 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_push_record()
752 		     tls_ctx->tx.rec_seq, record_type, prot);  in tls_push_record()
754 	tls_fill_prepend(tls_ctx,  in tls_push_record()
760 	tls_ctx->pending_open_record_frags = false;  in tls_push_record()
762 	rc = tls_do_encryption(sk, tls_ctx, ctx, req,  in tls_push_record()
768 				tls_ctx->pending_open_record_frags = true;  in tls_push_record()
778 		tls_ctx->pending_open_record_frags = true;  in tls_push_record()
789 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in bpf_exec_tx_verdict()  local
790 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in bpf_exec_tx_verdict()
897 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_push_pending_record()  local
898 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_push_pending_record()
918 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_sendmsg()  local
919 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_sw_sendmsg()
920 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_sendmsg()
942 	mutex_lock(&tls_ctx->tx_lock);  in tls_sw_sendmsg()
1064 		tls_ctx->pending_open_record_frags = true;  in tls_sw_sendmsg()
1135 	mutex_unlock(&tls_ctx->tx_lock);  in tls_sw_sendmsg()
1143 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_do_sendpage()  local
1144 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_do_sendpage()
1145 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_sw_do_sendpage()
1212 		tls_ctx->pending_open_record_frags = true;  in tls_sw_do_sendpage()
1269 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_sendpage()  local
1276 	mutex_lock(&tls_ctx->tx_lock);  in tls_sw_sendpage()
1280 	mutex_unlock(&tls_ctx->tx_lock);  in tls_sw_sendpage()
1288 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_rx_rec_wait()  local
1289 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_rx_rec_wait()
1438 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_decrypt_sg()  local
1439 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_decrypt_sg()
1440 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_decrypt_sg()
1514 		memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv,  in tls_decrypt_sg()
1522 		memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv, prot->salt_size);  in tls_decrypt_sg()
1524 	tls_xor_iv_with_seq(prot, &dctx->iv[iv_offset], tls_ctx->rx.rec_seq);  in tls_decrypt_sg()
1529 		     tls_ctx->rx.rec_seq, tlm->control, prot);  in tls_decrypt_sg()
1598 tls_decrypt_sw(struct sock *sk, struct tls_context *tls_ctx,  in tls_decrypt_sw()  argument
1601 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_decrypt_sw()
1602 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_decrypt_sw()
1621 		return tls_decrypt_sw(sk, tls_ctx, msg, darg);  in tls_decrypt_sw()
1639 		   struct tls_context *tls_ctx, struct tls_decrypt_arg *darg)  in tls_decrypt_device()  argument
1641 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_decrypt_device()
1642 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_decrypt_device()
1646 	if (tls_ctx->rx_conf != TLS_HW)  in tls_decrypt_device()
1649 	err = tls_device_decrypted(sk, tls_ctx);  in tls_decrypt_device()
1690 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_rx_one_record()  local
1691 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_rx_one_record()
1695 	err = tls_decrypt_device(sk, msg, tls_ctx, darg);  in tls_rx_one_record()
1697 		err = tls_decrypt_sw(sk, tls_ctx, msg, darg);  in tls_rx_one_record()
1704 	tls_advance_record_sn(sk, prot, &tls_ctx->rx);  in tls_rx_one_record()
1906 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_recvmsg()  local
1907 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_recvmsg()
1908 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_sw_recvmsg()
2127 	struct tls_context *tls_ctx = tls_get_ctx(sock->sk);  in tls_sw_splice_read()  local
2128 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_splice_read()
2196 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_sock_is_readable()  local
2197 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_sock_is_readable()
2213 	struct tls_context *tls_ctx = tls_get_ctx(strp->sk);  in tls_rx_msg_size()  local
2214 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_rx_msg_size()
2281 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_data_ready()  local
2282 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_data_ready()
2295 void tls_sw_cancel_work_tx(struct tls_context *tls_ctx)  in tls_sw_cancel_work_tx()  argument
2297 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_cancel_work_tx()
2306 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_release_resources_tx()  local
2307 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_release_resources_tx()
2325 	if (tls_ctx->partially_sent_record) {  in tls_sw_release_resources_tx()
2326 		tls_free_partial_record(sk, tls_ctx);  in tls_sw_release_resources_tx()
2345 void tls_sw_free_ctx_tx(struct tls_context *tls_ctx)  in tls_sw_free_ctx_tx()  argument
2347 	struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);  in tls_sw_free_ctx_tx()
2354 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_release_resources_rx()  local
2355 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_release_resources_rx()
2357 	kfree(tls_ctx->rx.rec_seq);  in tls_sw_release_resources_rx()
2358 	kfree(tls_ctx->rx.iv);  in tls_sw_release_resources_rx()
2376 void tls_sw_strparser_done(struct tls_context *tls_ctx)  in tls_sw_strparser_done()  argument
2378 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_strparser_done()
2383 void tls_sw_free_ctx_rx(struct tls_context *tls_ctx)  in tls_sw_free_ctx_rx()  argument
2385 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_free_ctx_rx()
2392 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_sw_free_resources_rx()  local
2395 	tls_sw_free_ctx_rx(tls_ctx);  in tls_sw_free_resources_rx()
2405 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tx_work_handler()  local
2408 	if (unlikely(!tls_ctx))  in tx_work_handler()
2411 	ctx = tls_sw_ctx_tx(tls_ctx);  in tx_work_handler()
2417 	mutex_lock(&tls_ctx->tx_lock);  in tx_work_handler()
2421 	mutex_unlock(&tls_ctx->tx_lock);  in tx_work_handler()
2445 void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx)  in tls_sw_strparser_arm()  argument
2447 	struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx);  in tls_sw_strparser_arm()
2455 void tls_update_rx_zc_capable(struct tls_context *tls_ctx)  in tls_update_rx_zc_capable()  argument
2457 	struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx);  in tls_update_rx_zc_capable()
2459 	rx_ctx->zc_capable = tls_ctx->rx_no_pad ||  in tls_update_rx_zc_capable()
2460 		tls_ctx->prot_info.version != TLS_1_3_VERSION;  in tls_update_rx_zc_capable()
2465 	struct tls_context *tls_ctx = tls_get_ctx(sk);  in tls_set_sw_offload()  local
2466 	struct tls_prot_info *prot = &tls_ctx->prot_info;  in tls_set_sw_offload()