Lines Matching refs:tfile
278 struct tun_file *tfile = container_of(napi, struct tun_file, napi); in tun_napi_receive() local
279 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_napi_receive()
316 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, in tun_napi_init() argument
319 tfile->napi_enabled = napi_en; in tun_napi_init()
320 tfile->napi_frags_enabled = napi_en && napi_frags; in tun_napi_init()
322 netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll, in tun_napi_init()
324 napi_enable(&tfile->napi); in tun_napi_init()
328 static void tun_napi_disable(struct tun_file *tfile) in tun_napi_disable() argument
330 if (tfile->napi_enabled) in tun_napi_disable()
331 napi_disable(&tfile->napi); in tun_napi_disable()
334 static void tun_napi_del(struct tun_file *tfile) in tun_napi_del() argument
336 if (tfile->napi_enabled) in tun_napi_del()
337 netif_napi_del(&tfile->napi); in tun_napi_del()
340 static bool tun_napi_frags_enabled(const struct tun_file *tfile) in tun_napi_frags_enabled() argument
342 return tfile->napi_frags_enabled; in tun_napi_frags_enabled()
521 struct tun_file *tfile) in tun_flow_update() argument
526 u16 queue_index = tfile->queue_index; in tun_flow_update()
644 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) in tun_disable_queue() argument
646 tfile->detached = tun; in tun_disable_queue()
647 list_add_tail(&tfile->next, &tun->disabled); in tun_disable_queue()
651 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) in tun_enable_queue() argument
653 struct tun_struct *tun = tfile->detached; in tun_enable_queue()
655 tfile->detached = NULL; in tun_enable_queue()
656 list_del_init(&tfile->next); in tun_enable_queue()
675 static void tun_queue_purge(struct tun_file *tfile) in tun_queue_purge() argument
679 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) in tun_queue_purge()
682 skb_queue_purge(&tfile->sk.sk_write_queue); in tun_queue_purge()
683 skb_queue_purge(&tfile->sk.sk_error_queue); in tun_queue_purge()
686 static void __tun_detach(struct tun_file *tfile, bool clean) in __tun_detach() argument
691 tun = rtnl_dereference(tfile->tun); in __tun_detach()
694 tun_napi_disable(tfile); in __tun_detach()
695 tun_napi_del(tfile); in __tun_detach()
698 if (tun && !tfile->detached) { in __tun_detach()
699 u16 index = tfile->queue_index; in __tun_detach()
709 RCU_INIT_POINTER(tfile->tun, NULL); in __tun_detach()
710 sock_put(&tfile->sk); in __tun_detach()
712 tun_disable_queue(tun, tfile); in __tun_detach()
717 tun_queue_purge(tfile); in __tun_detach()
719 } else if (tfile->detached && clean) { in __tun_detach()
720 tun = tun_enable_queue(tfile); in __tun_detach()
721 sock_put(&tfile->sk); in __tun_detach()
733 xdp_rxq_info_unreg(&tfile->xdp_rxq); in __tun_detach()
734 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); in __tun_detach()
735 sock_put(&tfile->sk); in __tun_detach()
739 static void tun_detach(struct tun_file *tfile, bool clean) in tun_detach() argument
745 tun = rtnl_dereference(tfile->tun); in tun_detach()
747 __tun_detach(tfile, clean); in tun_detach()
756 struct tun_file *tfile, *tmp; in tun_detach_all() local
760 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
761 BUG_ON(!tfile); in tun_detach_all()
762 tun_napi_disable(tfile); in tun_detach_all()
763 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; in tun_detach_all()
764 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_detach_all()
765 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
768 list_for_each_entry(tfile, &tun->disabled, next) { in tun_detach_all()
769 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; in tun_detach_all()
770 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_detach_all()
771 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
777 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
778 tun_napi_del(tfile); in tun_detach_all()
780 tun_queue_purge(tfile); in tun_detach_all()
781 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_detach_all()
782 sock_put(&tfile->sk); in tun_detach_all()
784 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { in tun_detach_all()
785 tun_enable_queue(tfile); in tun_detach_all()
786 tun_queue_purge(tfile); in tun_detach_all()
787 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_detach_all()
788 sock_put(&tfile->sk); in tun_detach_all()
799 struct tun_file *tfile = file->private_data; in tun_attach() local
803 err = security_tun_dev_attach(tfile->socket.sk, tun->security); in tun_attach()
808 if (rtnl_dereference(tfile->tun) && !tfile->detached) in tun_attach()
816 if (!tfile->detached && in tun_attach()
824 lock_sock(tfile->socket.sk); in tun_attach()
825 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach()
826 release_sock(tfile->socket.sk); in tun_attach()
831 if (!tfile->detached && in tun_attach()
832 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, in tun_attach()
838 tfile->queue_index = tun->numqueues; in tun_attach()
839 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; in tun_attach()
841 if (tfile->detached) { in tun_attach()
843 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); in tun_attach()
845 if (tfile->xdp_rxq.queue_index != tfile->queue_index) in tun_attach()
846 tfile->xdp_rxq.queue_index = tfile->queue_index; in tun_attach()
849 err = xdp_rxq_info_reg(&tfile->xdp_rxq, in tun_attach()
850 tun->dev, tfile->queue_index); in tun_attach()
853 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, in tun_attach()
856 xdp_rxq_info_unreg(&tfile->xdp_rxq); in tun_attach()
862 rcu_assign_pointer(tfile->tun, tun); in tun_attach()
863 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); in tun_attach()
866 if (tfile->detached) { in tun_attach()
867 tun_enable_queue(tfile); in tun_attach()
869 sock_hold(&tfile->sk); in tun_attach()
870 tun_napi_init(tun, tfile, napi, napi_frags); in tun_attach()
883 static struct tun_struct *tun_get(struct tun_file *tfile) in tun_get() argument
888 tun = rcu_dereference(tfile->tun); in tun_get()
1024 struct tun_file *tfile; in tun_net_open() local
1026 tfile = rtnl_dereference(tun->tfiles[i]); in tun_net_open()
1027 tfile->socket.sk->sk_write_space(tfile->socket.sk); in tun_net_open()
1079 struct tun_file *tfile; in tun_net_xmit() local
1083 tfile = rcu_dereference(tun->tfiles[txq]); in tun_net_xmit()
1094 BUG_ON(!tfile); in tun_net_xmit()
1102 if (tfile->socket.sk->sk_filter && in tun_net_xmit()
1103 sk_filter(tfile->socket.sk, skb)) in tun_net_xmit()
1122 if (ptr_ring_produce(&tfile->tx_ring, skb)) in tun_net_xmit()
1126 if (tfile->flags & TUN_FASYNC) in tun_net_xmit()
1127 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); in tun_net_xmit()
1128 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in tun_net_xmit()
1254 static void __tun_xdp_flush_tfile(struct tun_file *tfile) in __tun_xdp_flush_tfile() argument
1257 if (tfile->flags & TUN_FASYNC) in __tun_xdp_flush_tfile()
1258 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); in __tun_xdp_flush_tfile()
1259 tfile->socket.sk->sk_data_ready(tfile->socket.sk); in __tun_xdp_flush_tfile()
1266 struct tun_file *tfile; in tun_xdp_xmit() local
1283 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % in tun_xdp_xmit()
1286 spin_lock(&tfile->tx_ring.producer_lock); in tun_xdp_xmit()
1294 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { in tun_xdp_xmit()
1300 spin_unlock(&tfile->tx_ring.producer_lock); in tun_xdp_xmit()
1303 __tun_xdp_flush_tfile(tfile); in tun_xdp_xmit()
1393 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) in tun_sock_writeable() argument
1395 struct sock *sk = tfile->socket.sk; in tun_sock_writeable()
1405 struct tun_file *tfile = file->private_data; in tun_chr_poll() local
1406 struct tun_struct *tun = tun_get(tfile); in tun_chr_poll()
1413 sk = tfile->socket.sk; in tun_chr_poll()
1419 if (!ptr_ring_empty(&tfile->tx_ring)) in tun_chr_poll()
1427 if (tun_sock_writeable(tun, tfile) || in tun_chr_poll()
1429 tun_sock_writeable(tun, tfile))) in tun_chr_poll()
1439 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, in tun_napi_alloc_frags() argument
1452 skb = napi_get_frags(&tfile->napi); in tun_napi_alloc_frags()
1489 napi_free_frags(&tfile->napi); in tun_napi_alloc_frags()
1495 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, in tun_alloc_skb() argument
1499 struct sock *sk = tfile->socket.sk; in tun_alloc_skb()
1520 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, in tun_rx_batched() argument
1523 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_rx_batched()
1556 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, in tun_can_build_skb() argument
1562 if (tfile->socket.sk->sk_sndbuf != INT_MAX) in tun_can_build_skb()
1579 struct tun_file *tfile, in tun_build_skb() argument
1632 xdp.rxq = &tfile->xdp_rxq; in tun_build_skb()
1697 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, in tun_get_user() argument
1713 bool frags = tun_napi_frags_enabled(tfile); in tun_get_user()
1771 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { in tun_get_user()
1776 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); in tun_get_user()
1793 mutex_lock(&tfile->napi_mutex); in tun_get_user()
1794 skb = tun_napi_alloc_frags(tfile, copylen, from); in tun_get_user()
1801 skb = tun_alloc_skb(tfile, align, copylen, linear, in tun_get_user()
1809 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1822 tfile->napi.skb = NULL; in tun_get_user()
1823 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1834 tfile->napi.skb = NULL; in tun_get_user()
1835 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1907 !tfile->detached) in tun_get_user()
1916 napi_free_frags(&tfile->napi); in tun_get_user()
1917 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1923 napi_gro_frags(&tfile->napi); in tun_get_user()
1925 mutex_unlock(&tfile->napi_mutex); in tun_get_user()
1926 } else if (tfile->napi_enabled) { in tun_get_user()
1927 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; in tun_get_user()
1936 napi_schedule(&tfile->napi); in tun_get_user()
1940 tun_rx_batched(tun, tfile, skb, more); in tun_get_user()
1953 tun_flow_update(tun, rxhash, tfile); in tun_get_user()
1961 struct tun_file *tfile = file->private_data; in tun_chr_write_iter() local
1962 struct tun_struct *tun = tun_get(tfile); in tun_chr_write_iter()
1968 result = tun_get_user(tun, tfile, NULL, from, in tun_chr_write_iter()
1976 struct tun_file *tfile, in tun_put_user_xdp() argument
2011 struct tun_file *tfile, in tun_put_user() argument
2104 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) in tun_ring_recv() argument
2110 ptr = ptr_ring_consume(&tfile->tx_ring); in tun_ring_recv()
2118 add_wait_queue(&tfile->wq.wait, &wait); in tun_ring_recv()
2122 ptr = ptr_ring_consume(&tfile->tx_ring); in tun_ring_recv()
2129 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { in tun_ring_recv()
2138 remove_wait_queue(&tfile->wq.wait, &wait); in tun_ring_recv()
2145 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, in tun_do_read() argument
2161 ptr = tun_ring_recv(tfile, noblock, &err); in tun_do_read()
2169 ret = tun_put_user_xdp(tun, tfile, xdpf, to); in tun_do_read()
2174 ret = tun_put_user(tun, tfile, skb, to); in tun_do_read()
2187 struct tun_file *tfile = file->private_data; in tun_chr_read_iter() local
2188 struct tun_struct *tun = tun_get(tfile); in tun_chr_read_iter()
2193 ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK, NULL); in tun_chr_read_iter()
2335 struct tun_file *tfile; in tun_sock_write_space() local
2349 tfile = container_of(sk, struct tun_file, sk); in tun_sock_write_space()
2350 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); in tun_sock_write_space()
2356 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_sendmsg() local
2357 struct tun_struct *tun = tun_get(tfile); in tun_sendmsg()
2362 ret = tun_get_user(tun, tfile, m->msg_control, &m->msg_iter, in tun_sendmsg()
2372 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_recvmsg() local
2373 struct tun_struct *tun = tun_get(tfile); in tun_recvmsg()
2391 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); in tun_recvmsg()
2423 struct tun_file *tfile = container_of(sock, struct tun_file, socket); in tun_peek_len() local
2427 tun = tun_get(tfile); in tun_peek_len()
2431 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); in tun_peek_len()
2500 struct tun_file *tfile = file->private_data; in tun_set_iff() local
2504 if (tfile->detached) in tun_set_iff()
2595 dev->ifindex = tfile->ifindex; in tun_set_iff()
2606 tun->sndbuf = tfile->socket.sk->sk_sndbuf; in tun_set_iff()
2727 struct tun_file *tfile; in tun_detach_filter() local
2730 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_filter()
2731 lock_sock(tfile->socket.sk); in tun_detach_filter()
2732 sk_detach_filter(tfile->socket.sk); in tun_detach_filter()
2733 release_sock(tfile->socket.sk); in tun_detach_filter()
2742 struct tun_file *tfile; in tun_attach_filter() local
2745 tfile = rtnl_dereference(tun->tfiles[i]); in tun_attach_filter()
2746 lock_sock(tfile->socket.sk); in tun_attach_filter()
2747 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach_filter()
2748 release_sock(tfile->socket.sk); in tun_attach_filter()
2761 struct tun_file *tfile; in tun_set_sndbuf() local
2765 tfile = rtnl_dereference(tun->tfiles[i]); in tun_set_sndbuf()
2766 tfile->socket.sk->sk_sndbuf = tun->sndbuf; in tun_set_sndbuf()
2772 struct tun_file *tfile = file->private_data; in tun_set_queue() local
2779 tun = tfile->detached; in tun_set_queue()
2790 tun = rtnl_dereference(tfile->tun); in tun_set_queue()
2791 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) in tun_set_queue()
2794 __tun_detach(tfile, false); in tun_set_queue()
2829 struct tun_file *tfile = file->private_data; in __tun_chr_ioctl() local
2830 struct net *net = sock_net(&tfile->sk); in __tun_chr_ioctl()
2868 tun = tun_get(tfile); in __tun_chr_ioctl()
2895 tfile->ifindex = ifindex; in __tun_chr_ioctl()
2910 if (tfile->detached) in __tun_chr_ioctl()
2912 if (!tfile->socket.sk->sk_filter) in __tun_chr_ioctl()
3020 sndbuf = tfile->socket.sk->sk_sndbuf; in __tun_chr_ioctl()
3174 struct tun_file *tfile = file->private_data; in tun_chr_fasync() local
3177 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) in tun_chr_fasync()
3182 tfile->flags |= TUN_FASYNC; in tun_chr_fasync()
3184 tfile->flags &= ~TUN_FASYNC; in tun_chr_fasync()
3193 struct tun_file *tfile; in tun_chr_open() local
3197 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, in tun_chr_open()
3199 if (!tfile) in tun_chr_open()
3201 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { in tun_chr_open()
3202 sk_free(&tfile->sk); in tun_chr_open()
3206 mutex_init(&tfile->napi_mutex); in tun_chr_open()
3207 RCU_INIT_POINTER(tfile->tun, NULL); in tun_chr_open()
3208 tfile->flags = 0; in tun_chr_open()
3209 tfile->ifindex = 0; in tun_chr_open()
3211 init_waitqueue_head(&tfile->wq.wait); in tun_chr_open()
3212 RCU_INIT_POINTER(tfile->socket.wq, &tfile->wq); in tun_chr_open()
3214 tfile->socket.file = file; in tun_chr_open()
3215 tfile->socket.ops = &tun_socket_ops; in tun_chr_open()
3217 sock_init_data(&tfile->socket, &tfile->sk); in tun_chr_open()
3219 tfile->sk.sk_write_space = tun_sock_write_space; in tun_chr_open()
3220 tfile->sk.sk_sndbuf = INT_MAX; in tun_chr_open()
3222 file->private_data = tfile; in tun_chr_open()
3223 INIT_LIST_HEAD(&tfile->next); in tun_chr_open()
3225 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); in tun_chr_open()
3232 struct tun_file *tfile = file->private_data; in tun_chr_close() local
3234 tun_detach(tfile, true); in tun_chr_close()
3242 struct tun_file *tfile = file->private_data; in tun_chr_show_fdinfo() local
3249 tun = tun_get(tfile); in tun_chr_show_fdinfo()
3391 struct tun_file *tfile; in tun_queue_resize() local
3401 tfile = rtnl_dereference(tun->tfiles[i]); in tun_queue_resize()
3402 rings[i] = &tfile->tx_ring; in tun_queue_resize()
3404 list_for_each_entry(tfile, &tun->disabled, next) in tun_queue_resize()
3405 rings[i++] = &tfile->tx_ring; in tun_queue_resize()
3487 struct tun_file *tfile; in tun_get_socket() local
3490 tfile = file->private_data; in tun_get_socket()
3491 if (!tfile) in tun_get_socket()
3493 return &tfile->socket; in tun_get_socket()
3499 struct tun_file *tfile; in tun_get_tx_ring() local
3503 tfile = file->private_data; in tun_get_tx_ring()
3504 if (!tfile) in tun_get_tx_ring()
3506 return &tfile->tx_ring; in tun_get_tx_ring()