Lines Matching refs:psock
422 struct sk_psock *psock, in sk_psock_skb_ingress_enqueue() argument
447 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress_enqueue()
448 sk_psock_data_ready(sk, psock); in sk_psock_skb_ingress_enqueue()
452 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
454 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress() argument
456 struct sock *sk = psock->sk; in sk_psock_skb_ingress()
464 return sk_psock_skb_ingress_self(psock, skb); in sk_psock_skb_ingress()
476 return sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress()
483 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress_self() argument
486 struct sock *sk = psock->sk; in sk_psock_skb_ingress_self()
491 return sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress_self()
494 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_handle_skb() argument
498 if (!sock_writeable(psock->sk)) in sk_psock_handle_skb()
500 return skb_send_sock_locked(psock->sk, skb, off, len); in sk_psock_handle_skb()
502 return sk_psock_skb_ingress(psock, skb); in sk_psock_handle_skb()
507 struct sk_psock *psock = container_of(work, struct sk_psock, work); in sk_psock_backlog() local
508 struct sk_psock_work_state *state = &psock->work_state; in sk_psock_backlog()
515 lock_sock(psock->sk); in sk_psock_backlog()
524 while ((skb = skb_dequeue(&psock->ingress_skb))) { in sk_psock_backlog()
531 if (likely(psock->sk->sk_socket)) in sk_psock_backlog()
532 ret = sk_psock_handle_skb(psock, skb, off, in sk_psock_backlog()
542 sk_psock_report_error(psock, ret ? -ret : EPIPE); in sk_psock_backlog()
543 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_backlog()
555 release_sock(psock->sk); in sk_psock_backlog()
560 struct sk_psock *psock; in sk_psock_init() local
566 psock = ERR_PTR(-EINVAL); in sk_psock_init()
571 psock = ERR_PTR(-EBUSY); in sk_psock_init()
575 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node); in sk_psock_init()
576 if (!psock) { in sk_psock_init()
577 psock = ERR_PTR(-ENOMEM); in sk_psock_init()
582 psock->sk = sk; in sk_psock_init()
583 psock->eval = __SK_NONE; in sk_psock_init()
584 psock->sk_proto = prot; in sk_psock_init()
585 psock->saved_unhash = prot->unhash; in sk_psock_init()
586 psock->saved_close = prot->close; in sk_psock_init()
587 psock->saved_write_space = sk->sk_write_space; in sk_psock_init()
589 INIT_LIST_HEAD(&psock->link); in sk_psock_init()
590 spin_lock_init(&psock->link_lock); in sk_psock_init()
592 INIT_WORK(&psock->work, sk_psock_backlog); in sk_psock_init()
593 INIT_LIST_HEAD(&psock->ingress_msg); in sk_psock_init()
594 skb_queue_head_init(&psock->ingress_skb); in sk_psock_init()
596 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_init()
597 refcount_set(&psock->refcnt, 1); in sk_psock_init()
599 rcu_assign_sk_user_data_nocopy(sk, psock); in sk_psock_init()
604 return psock; in sk_psock_init()
608 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock) in sk_psock_link_pop() argument
612 spin_lock_bh(&psock->link_lock); in sk_psock_link_pop()
613 link = list_first_entry_or_null(&psock->link, struct sk_psock_link, in sk_psock_link_pop()
617 spin_unlock_bh(&psock->link_lock); in sk_psock_link_pop()
621 void __sk_psock_purge_ingress_msg(struct sk_psock *psock) in __sk_psock_purge_ingress_msg() argument
625 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
627 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
632 static void sk_psock_zap_ingress(struct sk_psock *psock) in sk_psock_zap_ingress() argument
634 __skb_queue_purge(&psock->ingress_skb); in sk_psock_zap_ingress()
635 __sk_psock_purge_ingress_msg(psock); in sk_psock_zap_ingress()
638 static void sk_psock_link_destroy(struct sk_psock *psock) in sk_psock_link_destroy() argument
642 list_for_each_entry_safe(link, tmp, &psock->link, list) { in sk_psock_link_destroy()
650 struct sk_psock *psock = container_of(gc, struct sk_psock, gc); in sk_psock_destroy_deferred() local
655 if (psock->progs.skb_parser) in sk_psock_destroy_deferred()
656 strp_done(&psock->parser.strp); in sk_psock_destroy_deferred()
658 cancel_work_sync(&psock->work); in sk_psock_destroy_deferred()
660 psock_progs_drop(&psock->progs); in sk_psock_destroy_deferred()
662 sk_psock_link_destroy(psock); in sk_psock_destroy_deferred()
663 sk_psock_cork_free(psock); in sk_psock_destroy_deferred()
664 sk_psock_zap_ingress(psock); in sk_psock_destroy_deferred()
666 if (psock->sk_redir) in sk_psock_destroy_deferred()
667 sock_put(psock->sk_redir); in sk_psock_destroy_deferred()
668 sock_put(psock->sk); in sk_psock_destroy_deferred()
669 kfree(psock); in sk_psock_destroy_deferred()
674 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu); in sk_psock_destroy() local
676 INIT_WORK(&psock->gc, sk_psock_destroy_deferred); in sk_psock_destroy()
677 schedule_work(&psock->gc); in sk_psock_destroy()
681 void sk_psock_drop(struct sock *sk, struct sk_psock *psock) in sk_psock_drop() argument
683 sk_psock_cork_free(psock); in sk_psock_drop()
684 sk_psock_zap_ingress(psock); in sk_psock_drop()
687 sk_psock_restore_proto(sk, psock); in sk_psock_drop()
689 if (psock->progs.skb_parser) in sk_psock_drop()
690 sk_psock_stop_strp(sk, psock); in sk_psock_drop()
691 else if (psock->progs.skb_verdict) in sk_psock_drop()
692 sk_psock_stop_verdict(sk, psock); in sk_psock_drop()
694 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_drop()
696 call_rcu(&psock->rcu, sk_psock_destroy); in sk_psock_drop()
713 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, in sk_psock_msg_verdict() argument
720 prog = READ_ONCE(psock->progs.msg_parser); in sk_psock_msg_verdict()
730 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
732 if (psock->sk_redir) in sk_psock_msg_verdict()
733 sock_put(psock->sk_redir); in sk_psock_msg_verdict()
734 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()
735 if (!psock->sk_redir) { in sk_psock_msg_verdict()
739 sock_hold(psock->sk_redir); in sk_psock_msg_verdict()
747 static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog, in sk_psock_bpf_run() argument
804 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_tls_strp_read() argument
810 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_tls_strp_read()
816 skb->sk = psock->sk; in sk_psock_tls_strp_read()
818 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_tls_strp_read()
822 sk_psock_tls_verdict_apply(skb, psock->sk, ret); in sk_psock_tls_strp_read()
828 static void sk_psock_verdict_apply(struct sk_psock *psock, in sk_psock_verdict_apply() argument
837 sk_other = psock->sk; in sk_psock_verdict_apply()
839 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
852 if (skb_queue_empty(&psock->ingress_skb)) { in sk_psock_verdict_apply()
853 err = sk_psock_skb_ingress_self(psock, skb); in sk_psock_verdict_apply()
856 skb_queue_tail(&psock->ingress_skb, skb); in sk_psock_verdict_apply()
857 schedule_work(&psock->work); in sk_psock_verdict_apply()
872 struct sk_psock *psock; in sk_psock_strp_read() local
879 psock = sk_psock(sk); in sk_psock_strp_read()
880 if (unlikely(!psock)) { in sk_psock_strp_read()
885 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_strp_read()
888 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_strp_read()
891 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_strp_read()
903 struct sk_psock *psock = sk_psock_from_strp(strp); in sk_psock_strp_parse() local
908 prog = READ_ONCE(psock->progs.skb_parser); in sk_psock_strp_parse()
910 skb->sk = psock->sk; in sk_psock_strp_parse()
911 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_strp_parse()
921 struct sk_psock *psock; in sk_psock_strp_data_ready() local
924 psock = sk_psock(sk); in sk_psock_strp_data_ready()
925 if (likely(psock)) { in sk_psock_strp_data_ready()
927 psock->parser.saved_data_ready(sk); in sk_psock_strp_data_ready()
930 strp_data_ready(&psock->parser.strp); in sk_psock_strp_data_ready()
941 struct sk_psock *psock; in sk_psock_verdict_recv() local
954 psock = sk_psock(sk); in sk_psock_verdict_recv()
955 if (unlikely(!psock)) { in sk_psock_verdict_recv()
961 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_verdict_recv()
964 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_verdict_recv()
967 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_verdict_recv()
990 struct sk_psock *psock; in sk_psock_write_space() local
994 psock = sk_psock(sk); in sk_psock_write_space()
995 if (likely(psock)) { in sk_psock_write_space()
996 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_write_space()
997 schedule_work(&psock->work); in sk_psock_write_space()
998 write_space = psock->saved_write_space; in sk_psock_write_space()
1005 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_init_strp() argument
1013 psock->parser.enabled = false; in sk_psock_init_strp()
1014 return strp_init(&psock->parser.strp, sk, &cb); in sk_psock_init_strp()
1017 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_start_verdict() argument
1019 struct sk_psock_parser *parser = &psock->parser; in sk_psock_start_verdict()
1030 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_start_strp() argument
1032 struct sk_psock_parser *parser = &psock->parser; in sk_psock_start_strp()
1043 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_strp() argument
1045 struct sk_psock_parser *parser = &psock->parser; in sk_psock_stop_strp()
1056 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_verdict() argument
1058 struct sk_psock_parser *parser = &psock->parser; in sk_psock_stop_verdict()