Lines Matching refs:psock
56 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, in kcm_abort_tx_psock() argument
59 struct sock *csk = psock->sk; in kcm_abort_tx_psock()
60 struct kcm_mux *mux = psock->mux; in kcm_abort_tx_psock()
66 if (psock->tx_stopped) { in kcm_abort_tx_psock()
71 psock->tx_stopped = 1; in kcm_abort_tx_psock()
72 KCM_STATS_INCR(psock->stats.tx_aborts); in kcm_abort_tx_psock()
74 if (!psock->tx_kcm) { in kcm_abort_tx_psock()
76 list_del(&psock->psock_avail_list); in kcm_abort_tx_psock()
85 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_abort_tx_psock()
96 struct kcm_psock *psock) in kcm_update_rx_mux_stats() argument
99 psock->strp.stats.bytes - in kcm_update_rx_mux_stats()
100 psock->saved_rx_bytes); in kcm_update_rx_mux_stats()
102 psock->strp.stats.msgs - psock->saved_rx_msgs; in kcm_update_rx_mux_stats()
103 psock->saved_rx_msgs = psock->strp.stats.msgs; in kcm_update_rx_mux_stats()
104 psock->saved_rx_bytes = psock->strp.stats.bytes; in kcm_update_rx_mux_stats()
108 struct kcm_psock *psock) in kcm_update_tx_mux_stats() argument
111 psock->stats.tx_bytes - psock->saved_tx_bytes); in kcm_update_tx_mux_stats()
113 psock->stats.tx_msgs - psock->saved_tx_msgs; in kcm_update_tx_mux_stats()
114 psock->saved_tx_msgs = psock->stats.tx_msgs; in kcm_update_tx_mux_stats()
115 psock->saved_tx_bytes = psock->stats.tx_bytes; in kcm_update_tx_mux_stats()
127 struct kcm_psock *psock; in kcm_rcv_ready() local
143 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, in kcm_rcv_ready()
146 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { in kcm_rcv_ready()
155 list_del(&psock->psock_ready_list); in kcm_rcv_ready()
156 psock->ready_rx_msg = NULL; in kcm_rcv_ready()
160 strp_unpause(&psock->strp); in kcm_rcv_ready()
161 strp_check_rcv(&psock->strp); in kcm_rcv_ready()
253 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, in reserve_rx_kcm() argument
256 struct kcm_mux *mux = psock->mux; in reserve_rx_kcm()
259 WARN_ON(psock->ready_rx_msg); in reserve_rx_kcm()
261 if (psock->rx_kcm) in reserve_rx_kcm()
262 return psock->rx_kcm; in reserve_rx_kcm()
266 if (psock->rx_kcm) { in reserve_rx_kcm()
268 return psock->rx_kcm; in reserve_rx_kcm()
271 kcm_update_rx_mux_stats(mux, psock); in reserve_rx_kcm()
274 psock->ready_rx_msg = head; in reserve_rx_kcm()
275 strp_pause(&psock->strp); in reserve_rx_kcm()
276 list_add_tail(&psock->psock_ready_list, in reserve_rx_kcm()
287 psock->rx_kcm = kcm; in reserve_rx_kcm()
288 kcm->rx_psock = psock; in reserve_rx_kcm()
303 static void unreserve_rx_kcm(struct kcm_psock *psock, in unreserve_rx_kcm() argument
306 struct kcm_sock *kcm = psock->rx_kcm; in unreserve_rx_kcm()
307 struct kcm_mux *mux = psock->mux; in unreserve_rx_kcm()
314 psock->rx_kcm = NULL; in unreserve_rx_kcm()
347 struct kcm_psock *psock; in psock_data_ready() local
351 psock = (struct kcm_psock *)sk->sk_user_data; in psock_data_ready()
352 if (likely(psock)) in psock_data_ready()
353 strp_data_ready(&psock->strp); in psock_data_ready()
361 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_rcv_strparser() local
365 kcm = reserve_rx_kcm(psock, skb); in kcm_rcv_strparser()
375 unreserve_rx_kcm(psock, false); in kcm_rcv_strparser()
382 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_parse_func_strparser() local
383 struct bpf_prog *prog = psock->bpf_prog; in kcm_parse_func_strparser()
390 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_read_sock_done() local
392 unreserve_rx_kcm(psock, true); in kcm_read_sock_done()
409 struct kcm_psock *psock; in psock_write_space() local
415 psock = (struct kcm_psock *)sk->sk_user_data; in psock_write_space()
416 if (unlikely(!psock)) in psock_write_space()
418 mux = psock->mux; in psock_write_space()
423 kcm = psock->tx_kcm; in psock_write_space()
438 struct kcm_psock *psock; in reserve_psock() local
440 psock = kcm->tx_psock; in reserve_psock()
444 if (psock) { in reserve_psock()
446 if (unlikely(psock->tx_stopped)) in reserve_psock()
457 psock = kcm->tx_psock; in reserve_psock()
458 if (unlikely(psock)) { in reserve_psock()
465 psock = list_first_entry(&mux->psocks_avail, in reserve_psock()
468 list_del(&psock->psock_avail_list); in reserve_psock()
473 kcm->tx_psock = psock; in reserve_psock()
474 psock->tx_kcm = kcm; in reserve_psock()
475 KCM_STATS_INCR(psock->stats.reserved); in reserve_psock()
484 return psock; in reserve_psock()
488 static void psock_now_avail(struct kcm_psock *psock) in psock_now_avail() argument
490 struct kcm_mux *mux = psock->mux; in psock_now_avail()
494 list_add_tail(&psock->psock_avail_list, in psock_now_avail()
502 psock->tx_kcm = kcm; in psock_now_avail()
509 kcm->tx_psock = psock; in psock_now_avail()
510 KCM_STATS_INCR(psock->stats.reserved); in psock_now_avail()
518 struct kcm_psock *psock; in unreserve_psock() local
523 psock = kcm->tx_psock; in unreserve_psock()
525 if (WARN_ON(!psock)) { in unreserve_psock()
532 kcm_update_tx_mux_stats(mux, psock); in unreserve_psock()
537 psock->tx_kcm = NULL; in unreserve_psock()
538 KCM_STATS_INCR(psock->stats.unreserved); in unreserve_psock()
540 if (unlikely(psock->tx_stopped)) { in unreserve_psock()
541 if (psock->done) { in unreserve_psock()
543 list_del(&psock->psock_list); in unreserve_psock()
545 sock_put(psock->sk); in unreserve_psock()
546 fput(psock->sk->sk_socket->file); in unreserve_psock()
547 kmem_cache_free(kcm_psockp, psock); in unreserve_psock()
557 psock_now_avail(psock); in unreserve_psock()
577 struct kcm_psock *psock; in kcm_write_msgs() local
585 psock = kcm->tx_psock; in kcm_write_msgs()
586 if (unlikely(psock && psock->tx_stopped)) { in kcm_write_msgs()
606 if (WARN_ON(!psock)) { in kcm_write_msgs()
619 psock = reserve_psock(kcm); in kcm_write_msgs()
620 if (!psock) in kcm_write_msgs()
646 ret = kernel_sendpage(psock->sk->sk_socket, in kcm_write_msgs()
670 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, in kcm_write_msgs()
683 KCM_STATS_ADD(psock->stats.tx_bytes, ret); in kcm_write_msgs()
705 KCM_STATS_INCR(psock->stats.tx_msgs); in kcm_write_msgs()
1376 struct kcm_psock *psock = NULL, *tpsock; in kcm_attach() local
1405 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); in kcm_attach()
1406 if (!psock) { in kcm_attach()
1411 psock->mux = mux; in kcm_attach()
1412 psock->sk = csk; in kcm_attach()
1413 psock->bpf_prog = prog; in kcm_attach()
1415 err = strp_init(&psock->strp, csk, &cb); in kcm_attach()
1417 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1428 strp_stop(&psock->strp); in kcm_attach()
1429 strp_done(&psock->strp); in kcm_attach()
1430 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1435 psock->save_data_ready = csk->sk_data_ready; in kcm_attach()
1436 psock->save_write_space = csk->sk_write_space; in kcm_attach()
1437 psock->save_state_change = csk->sk_state_change; in kcm_attach()
1438 csk->sk_user_data = psock; in kcm_attach()
1457 list_add(&psock->psock_list, head); in kcm_attach()
1458 psock->index = index; in kcm_attach()
1462 psock_now_avail(psock); in kcm_attach()
1466 strp_check_rcv(&psock->strp); in kcm_attach()
1504 static void kcm_unattach(struct kcm_psock *psock) in kcm_unattach() argument
1506 struct sock *csk = psock->sk; in kcm_unattach()
1507 struct kcm_mux *mux = psock->mux; in kcm_unattach()
1516 csk->sk_data_ready = psock->save_data_ready; in kcm_unattach()
1517 csk->sk_write_space = psock->save_write_space; in kcm_unattach()
1518 csk->sk_state_change = psock->save_state_change; in kcm_unattach()
1519 strp_stop(&psock->strp); in kcm_unattach()
1521 if (WARN_ON(psock->rx_kcm)) { in kcm_unattach()
1532 if (psock->ready_rx_msg) { in kcm_unattach()
1533 list_del(&psock->psock_ready_list); in kcm_unattach()
1534 kfree_skb(psock->ready_rx_msg); in kcm_unattach()
1535 psock->ready_rx_msg = NULL; in kcm_unattach()
1545 strp_done(&psock->strp); in kcm_unattach()
1548 bpf_prog_put(psock->bpf_prog); in kcm_unattach()
1552 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats); in kcm_unattach()
1553 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats); in kcm_unattach()
1557 if (psock->tx_kcm) { in kcm_unattach()
1569 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_unattach()
1572 if (!psock->tx_kcm) { in kcm_unattach()
1576 psock->done = 1; in kcm_unattach()
1582 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_unattach()
1586 if (!psock->tx_stopped) in kcm_unattach()
1587 list_del(&psock->psock_avail_list); in kcm_unattach()
1588 list_del(&psock->psock_list); in kcm_unattach()
1594 kmem_cache_free(kcm_psockp, psock); in kcm_unattach()
1604 struct kcm_psock *psock; in kcm_unattach_ioctl() local
1623 list_for_each_entry(psock, &mux->psocks, psock_list) { in kcm_unattach_ioctl()
1624 if (psock->sk != csk) in kcm_unattach_ioctl()
1629 if (psock->unattaching || WARN_ON(psock->done)) { in kcm_unattach_ioctl()
1634 psock->unattaching = 1; in kcm_unattach_ioctl()
1639 kcm_unattach(psock); in kcm_unattach_ioctl()
1752 struct kcm_psock *psock, *tmp_psock; in release_mux() local
1755 list_for_each_entry_safe(psock, tmp_psock, in release_mux()
1757 if (!WARN_ON(psock->unattaching)) in release_mux()
1758 kcm_unattach(psock); in release_mux()
1834 struct kcm_psock *psock; in kcm_release() local
1876 psock = kcm->tx_psock; in kcm_release()
1877 if (psock) { in kcm_release()
1882 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_release()