Lines Matching refs:tfile
227 struct tun_file *tfile = container_of(napi, struct tun_file, napi); in tun_napi_receive() local
228 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_napi_receive()
265 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, in tun_napi_init() argument
268 tfile->napi_enabled = napi_en; in tun_napi_init()
269 tfile->napi_frags_enabled = napi_en && napi_frags; in tun_napi_init()
271 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll); in tun_napi_init()
272 napi_enable(&tfile->napi); in tun_napi_init()
276 static void tun_napi_enable(struct tun_file *tfile) in tun_napi_enable() argument
278 if (tfile->napi_enabled) in tun_napi_enable()
279 napi_enable(&tfile->napi); in tun_napi_enable()
282 static void tun_napi_disable(struct tun_file *tfile) in tun_napi_disable() argument
284 if (tfile->napi_enabled) in tun_napi_disable()
285 napi_disable(&tfile->napi); in tun_napi_disable()
288 static void tun_napi_del(struct tun_file *tfile) in tun_napi_del() argument
290 if (tfile->napi_enabled) in tun_napi_del()
291 netif_napi_del(&tfile->napi); in tun_napi_del()
294 static bool tun_napi_frags_enabled(const struct tun_file *tfile) in tun_napi_frags_enabled() argument
296 return tfile->napi_frags_enabled; in tun_napi_frags_enabled()
474 struct tun_file *tfile) in tun_flow_update() argument
479 u16 queue_index = tfile->queue_index; in tun_flow_update()
593 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) in tun_disable_queue() argument
595 tfile->detached = tun; in tun_disable_queue()
596 list_add_tail(&tfile->next, &tun->disabled); in tun_disable_queue()
600 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) in tun_enable_queue() argument
602 struct tun_struct *tun = tfile->detached; in tun_enable_queue()
604 tfile->detached = NULL; in tun_enable_queue()
605 list_del_init(&tfile->next); in tun_enable_queue()
624 static void tun_queue_purge(struct tun_file *tfile) in tun_queue_purge() argument
628 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) in tun_queue_purge()
631 skb_queue_purge(&tfile->sk.sk_write_queue); in tun_queue_purge()
632 skb_queue_purge(&tfile->sk.sk_error_queue); in tun_queue_purge()
635 static void __tun_detach(struct tun_file *tfile, bool clean) in __tun_detach() argument
640 tun = rtnl_dereference(tfile->tun); in __tun_detach()
643 if (!tfile->detached) in __tun_detach()
644 tun_napi_disable(tfile); in __tun_detach()
645 tun_napi_del(tfile); in __tun_detach()
648 if (tun && !tfile->detached) { in __tun_detach()
649 u16 index = tfile->queue_index; in __tun_detach()
661 RCU_INIT_POINTER(tfile->tun, NULL); in __tun_detach()
662 sock_put(&tfile->sk); in __tun_detach()
664 tun_disable_queue(tun, tfile); in __tun_detach()
665 tun_napi_disable(tfile); in __tun_detach()
671 tun_queue_purge(tfile); in __tun_detach()
673 } else if (tfile->detached && clean) { in __tun_detach()
674 tun = tun_enable_queue(tfile); in __tun_detach()
675 sock_put(&tfile->sk); in __tun_detach()
687 xdp_rxq_info_unreg(&tfile->xdp_rxq); in __tun_detach()
688 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); in __tun_detach()
692 static void tun_detach(struct tun_file *tfile, bool clean) in tun_detach() argument
698 tun = rtnl_dereference(tfile->tun); in tun_detach()
700 __tun_detach(tfile, clean); in tun_detach()
706 sock_put(&tfile->sk); in tun_detach()
712 struct tun_file *tfile, *tmp; in tun_detach_all() local
716 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
717 BUG_ON(!tfile); in tun_detach_all()
718 tun_napi_disable(tfile); in tun_detach_all()
719 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; in tun_detach_all()
720 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_detach_all()
721 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
724 list_for_each_entry(tfile, &tun->disabled, next) { in tun_detach_all()
725 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; in tun_detach_all()
726 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_detach_all()
727 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
733 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
734 tun_napi_del(tfile); in tun_detach_all()
736 tun_queue_purge(tfile); in tun_detach_all()
737 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_detach_all()
738 sock_put(&tfile->sk); in tun_detach_all()
740 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { in tun_detach_all()
741 tun_napi_del(tfile); in tun_detach_all()
742 tun_enable_queue(tfile); in tun_detach_all()
743 tun_queue_purge(tfile); in tun_detach_all()
744 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_detach_all()
745 sock_put(&tfile->sk); in tun_detach_all()
757 struct tun_file *tfile = file->private_data; in tun_attach() local
761 err = security_tun_dev_attach(tfile->socket.sk, tun->security); in tun_attach()
766 if (rtnl_dereference(tfile->tun) && !tfile->detached) in tun_attach()
774 if (!tfile->detached && in tun_attach()
782 lock_sock(tfile->socket.sk); in tun_attach()
783 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach()
784 release_sock(tfile->socket.sk); in tun_attach()
789 if (!tfile->detached && in tun_attach()
790 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, in tun_attach()
796 tfile->queue_index = tun->numqueues; in tun_attach()
797 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; in tun_attach()
799 if (tfile->detached) { in tun_attach()
801 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); in tun_attach()
803 if (tfile->xdp_rxq.queue_index != tfile->queue_index) in tun_attach()
804 tfile->xdp_rxq.queue_index = tfile->queue_index; in tun_attach()
807 err = xdp_rxq_info_reg(&tfile->xdp_rxq, in tun_attach()
808 tun->dev, tfile->queue_index, 0); in tun_attach()
811 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, in tun_attach()
814 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_attach()
820 if (tfile->detached) { in tun_attach()
821 tun_enable_queue(tfile); in tun_attach()
822 tun_napi_enable(tfile); in tun_attach()
824 sock_hold(&tfile->sk); in tun_attach()
825 tun_napi_init(tun, tfile, napi, napi_frags); in tun_attach()
829 sock_set_flag(&tfile->sk, SOCK_XDP); in tun_attach()
840 rcu_assign_pointer(tfile->tun, tun); in tun_attach()
841 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); in tun_attach()
848 static struct tun_struct *tun_get(struct tun_file *tfile) in tun_get() argument
853 tun = rcu_dereference(tfile->tun); in tun_get()
1076 struct tun_file *tfile; in tun_net_xmit() local
1080 tfile = rcu_dereference(tun->tfiles[txq]); in tun_net_xmit()
1083 if (!tfile) { in tun_net_xmit()
1101 if (tfile->socket.sk->sk_filter && in tun_net_xmit()
1102 sk_filter(tfile->socket.sk, skb)) { in tun_net_xmit()
1132 if (ptr_ring_produce(&tfile->tx_ring, skb)) { in tun_net_xmit()
1142 if (tfile->flags & TUN_FASYNC) in tun_net_xmit()
1143 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); in tun_net_xmit()
1144 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_net_xmit()
1199 struct tun_file *tfile; in tun_xdp_set() local
1209 tfile = rtnl_dereference(tun->tfiles[i]); in tun_xdp_set()
1211 sock_set_flag(&tfile->sk, SOCK_XDP); in tun_xdp_set()
1213 sock_reset_flag(&tfile->sk, SOCK_XDP); in tun_xdp_set()
1215 list_for_each_entry(tfile, &tun->disabled, next) { in tun_xdp_set()
1217 sock_set_flag(&tfile->sk, SOCK_XDP); in tun_xdp_set()
1219 sock_reset_flag(&tfile->sk, SOCK_XDP); in tun_xdp_set()
1263 static void __tun_xdp_flush_tfile(struct tun_file *tfile) in __tun_xdp_flush_tfile() argument
1266 if (tfile->flags & TUN_FASYNC) in __tun_xdp_flush_tfile()
1267 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); in __tun_xdp_flush_tfile()
1268 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in __tun_xdp_flush_tfile()
1275 struct tun_file *tfile; in tun_xdp_xmit() local
1292 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % in tun_xdp_xmit()
1294 if (unlikely(!tfile)) in tun_xdp_xmit()
1297 spin_lock(&tfile->tx_ring.producer_lock); in tun_xdp_xmit()
1305 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { in tun_xdp_xmit()
1311 spin_unlock(&tfile->tx_ring.producer_lock); in tun_xdp_xmit()
1314 __tun_xdp_flush_tfile(tfile); in tun_xdp_xmit()
1411 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) in tun_sock_writeable() argument
1413 struct sock *sk = tfile->socket.sk; in tun_sock_writeable()
1423 struct tun_file *tfile = file->private_data; in tun_chr_poll() local
1424 struct tun_struct *tun = tun_get(tfile); in tun_chr_poll()
1431 sk = tfile->socket.sk; in tun_chr_poll()
1435 if (!ptr_ring_empty(&tfile->tx_ring)) in tun_chr_poll()
1443 if (tun_sock_writeable(tun, tfile) || in tun_chr_poll()
1445 tun_sock_writeable(tun, tfile))) in tun_chr_poll()
1455 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, in tun_napi_alloc_frags() argument
1469 skb = napi_get_frags(&tfile->napi); in tun_napi_alloc_frags()
1505 napi_free_frags(&tfile->napi); in tun_napi_alloc_frags()
1511 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, in tun_alloc_skb() argument
1515 struct sock *sk = tfile->socket.sk; in tun_alloc_skb()
1536 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, in tun_rx_batched() argument
1539 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_rx_batched()
1546 skb_record_rx_queue(skb, tfile->queue_index); in tun_rx_batched()
1567 skb_record_rx_queue(nskb, tfile->queue_index); in tun_rx_batched()
1570 skb_record_rx_queue(skb, tfile->queue_index); in tun_rx_batched()
1576 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, in tun_can_build_skb() argument
1582 if (tfile->socket.sk->sk_sndbuf != INT_MAX) in tun_can_build_skb()
1598 static struct sk_buff *__tun_build_skb(struct tun_file *tfile, in __tun_build_skb() argument
1609 skb_set_owner_w(skb, tfile->socket.sk); in __tun_build_skb()
1650 struct tun_file *tfile, in tun_build_skb() argument
1687 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, in tun_build_skb()
1700 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq); in tun_build_skb()
1726 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad); in tun_build_skb()
1735 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, in tun_get_user() argument
1750 bool frags = tun_napi_frags_enabled(tfile); in tun_get_user()
1806 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { in tun_get_user()
1811 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); in tun_get_user()
1828 mutex_lock(&tfile->napi_mutex); in tun_get_user()
1829 skb = tun_napi_alloc_frags(tfile, copylen, from); in tun_get_user()
1836 skb = tun_alloc_skb(tfile, align, copylen, linear, in tun_get_user()
1844 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1860 tfile->napi.skb = NULL; in tun_get_user()
1861 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1872 tfile->napi.skb = NULL; in tun_get_user()
1873 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1922 skb_record_rx_queue(skb, tfile->queue_index); in tun_get_user()
1937 tfile->napi.skb = NULL; in tun_get_user()
1938 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1952 !tfile->detached) in tun_get_user()
1976 napi_free_frags(&tfile->napi); in tun_get_user()
1978 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1982 if (likely(napi_schedule_prep(&tfile->napi))) { in tun_get_user()
1984 napi_gro_frags(&tfile->napi); in tun_get_user()
1985 napi_complete(&tfile->napi); in tun_get_user()
1991 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1992 } else if (tfile->napi_enabled) { in tun_get_user()
1993 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_get_user()
2002 napi_schedule(&tfile->napi); in tun_get_user()
2006 tun_rx_batched(tun, tfile, skb, more); in tun_get_user()
2017 tun_flow_update(tun, rxhash, tfile); in tun_get_user()
2025 struct tun_file *tfile = file->private_data; in tun_chr_write_iter() local
2026 struct tun_struct *tun = tun_get(tfile); in tun_chr_write_iter()
2036 result = tun_get_user(tun, tfile, NULL, from, noblock, false); in tun_chr_write_iter()
2043 struct tun_file *tfile, in tun_put_user_xdp() argument
2074 struct tun_file *tfile, in tun_put_user() argument
2163 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) in tun_ring_recv() argument
2169 ptr = ptr_ring_consume(&tfile->tx_ring); in tun_ring_recv()
2177 add_wait_queue(&tfile->socket.wq.wait, &wait); in tun_ring_recv()
2181 ptr = ptr_ring_consume(&tfile->tx_ring); in tun_ring_recv()
2188 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { in tun_ring_recv()
2197 remove_wait_queue(&tfile->socket.wq.wait, &wait); in tun_ring_recv()
2204 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, in tun_do_read() argument
2218 ptr = tun_ring_recv(tfile, noblock, &err); in tun_do_read()
2226 ret = tun_put_user_xdp(tun, tfile, xdpf, to); in tun_do_read()
2231 ret = tun_put_user(tun, tfile, skb, to); in tun_do_read()
2244 struct tun_file *tfile = file->private_data; in tun_chr_read_iter() local
2245 struct tun_struct *tun = tun_get(tfile); in tun_chr_read_iter()
2255 ret = tun_do_read(tun, tfile, to, noblock, NULL); in tun_chr_read_iter()
2400 struct tun_file *tfile; in tun_sock_write_space() local
2414 tfile = container_of(sk, struct tun_file, sk); in tun_sock_write_space()
2415 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); in tun_sock_write_space()
2425 struct tun_file *tfile, in tun_xdp_one() argument
2448 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq); in tun_xdp_one()
2499 skb_record_rx_queue(skb, tfile->queue_index); in tun_xdp_one()
2510 !tfile->detached) in tun_xdp_one()
2513 if (tfile->napi_enabled) { in tun_xdp_one()
2514 queue = &tfile->sk.sk_write_queue; in tun_xdp_one()
2530 tun_flow_update(tun, rxhash, tfile); in tun_xdp_one()
2539 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_sendmsg() local
2540 struct tun_struct *tun = tun_get(tfile); in tun_sendmsg()
2560 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage); in tun_sendmsg()
2568 if (tfile->napi_enabled && queued > 0) in tun_sendmsg()
2569 napi_schedule(&tfile->napi); in tun_sendmsg()
2580 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, in tun_sendmsg()
2591 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_recvmsg() local
2592 struct tun_struct *tun = tun_get(tfile); in tun_recvmsg()
2610 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); in tun_recvmsg()
2642 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_peek_len() local
2646 tun = tun_get(tfile); in tun_peek_len()
2650 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); in tun_peek_len()
2719 struct tun_file *tfile = file->private_data; in tun_set_iff() local
2723 if (tfile->detached) in tun_set_iff()
2811 dev->ifindex = tfile->ifindex; in tun_set_iff()
2822 tun->sndbuf = tfile->socket.sk->sk_sndbuf; in tun_set_iff()
2839 rcu_assign_pointer(tfile->tun, tun); in tun_set_iff()
2906 struct tun_file *tfile; in tun_detach_filter() local
2909 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_filter()
2910 lock_sock(tfile->socket.sk); in tun_detach_filter()
2911 sk_detach_filter(tfile->socket.sk); in tun_detach_filter()
2912 release_sock(tfile->socket.sk); in tun_detach_filter()
2921 struct tun_file *tfile; in tun_attach_filter() local
2924 tfile = rtnl_dereference(tun->tfiles[i]); in tun_attach_filter()
2925 lock_sock(tfile->socket.sk); in tun_attach_filter()
2926 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach_filter()
2927 release_sock(tfile->socket.sk); in tun_attach_filter()
2940 struct tun_file *tfile; in tun_set_sndbuf() local
2944 tfile = rtnl_dereference(tun->tfiles[i]); in tun_set_sndbuf()
2945 tfile->socket.sk->sk_sndbuf = tun->sndbuf; in tun_set_sndbuf()
2951 struct tun_file *tfile = file->private_data; in tun_set_queue() local
2958 tun = tfile->detached; in tun_set_queue()
2969 tun = rtnl_dereference(tfile->tun); in tun_set_queue()
2970 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) in tun_set_queue()
2973 __tun_detach(tfile, false); in tun_set_queue()
3047 struct tun_file *tfile = file->private_data; in __tun_chr_ioctl() local
3048 struct net *net = sock_net(&tfile->sk); in __tun_chr_ioctl()
3085 tun = tun_get(tfile); in __tun_chr_ioctl()
3112 tfile->ifindex = ifindex; in __tun_chr_ioctl()
3128 if (tfile->detached) in __tun_chr_ioctl()
3130 if (!tfile->socket.sk->sk_filter) in __tun_chr_ioctl()
3243 sndbuf = tfile->socket.sk->sk_sndbuf; in __tun_chr_ioctl()
3412 struct tun_file *tfile = file->private_data; in tun_chr_fasync() local
3415 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) in tun_chr_fasync()
3420 tfile->flags |= TUN_FASYNC; in tun_chr_fasync()
3422 tfile->flags &= ~TUN_FASYNC; in tun_chr_fasync()
3431 struct tun_file *tfile; in tun_chr_open() local
3433 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, in tun_chr_open()
3435 if (!tfile) in tun_chr_open()
3437 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { in tun_chr_open()
3438 sk_free(&tfile->sk); in tun_chr_open()
3442 mutex_init(&tfile->napi_mutex); in tun_chr_open()
3443 RCU_INIT_POINTER(tfile->tun, NULL); in tun_chr_open()
3444 tfile->flags = 0; in tun_chr_open()
3445 tfile->ifindex = 0; in tun_chr_open()
3447 init_waitqueue_head(&tfile->socket.wq.wait); in tun_chr_open()
3449 tfile->socket.file = file; in tun_chr_open()
3450 tfile->socket.ops = &tun_socket_ops; in tun_chr_open()
3452 sock_init_data(&tfile->socket, &tfile->sk); in tun_chr_open()
3454 tfile->sk.sk_write_space = tun_sock_write_space; in tun_chr_open()
3455 tfile->sk.sk_sndbuf = INT_MAX; in tun_chr_open()
3457 file->private_data = tfile; in tun_chr_open()
3458 INIT_LIST_HEAD(&tfile->next); in tun_chr_open()
3460 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); in tun_chr_open()
3467 struct tun_file *tfile = file->private_data; in tun_chr_close() local
3469 tun_detach(tfile, true); in tun_chr_close()
3477 struct tun_file *tfile = file->private_data; in tun_chr_show_fdinfo() local
3484 tun = tun_get(tfile); in tun_chr_show_fdinfo()
3627 struct tun_file *tfile; in tun_queue_resize() local
3637 tfile = rtnl_dereference(tun->tfiles[i]); in tun_queue_resize()
3638 rings[i] = &tfile->tx_ring; in tun_queue_resize()
3640 list_for_each_entry(tfile, &tun->disabled, next) in tun_queue_resize()
3641 rings[i++] = &tfile->tx_ring; in tun_queue_resize()
3668 struct tun_file *tfile; in tun_device_event() local
3670 tfile = rtnl_dereference(tun->tfiles[i]); in tun_device_event()
3671 tfile->socket.sk->sk_write_space(tfile->socket.sk); in tun_device_event()
3732 struct tun_file *tfile; in tun_get_socket() local
3735 tfile = file->private_data; in tun_get_socket()
3736 if (!tfile) in tun_get_socket()
3738 return &tfile->socket; in tun_get_socket()
3744 struct tun_file *tfile; in tun_get_tx_ring() local
3748 tfile = file->private_data; in tun_get_tx_ring()
3749 if (!tfile) in tun_get_tx_ring()
3751 return &tfile->tx_ring; in tun_get_tx_ring()